lsst.obs.base  20.0.0-13-gfbf2cf9+1
fitsExposure.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 __all__ = ("FitsExposureFormatter", )
23 
24 from astro_metadata_translator import fix_header
25 from lsst.daf.butler import Formatter
26 from lsst.afw.image import ExposureFitsReader
27 from lsst.daf.base import PropertySet
28 
29 
30 class FitsExposureFormatter(Formatter):
31  """Interface for reading and writing Exposures to and from FITS files.
32 
33  This Formatter supports write recipes.
34 
35  Each ``FitsExposureFormatter`` recipe for FITS compression should
36  define ``image``, ``mask`` and ``variance`` entries, each of which may
37  contain ``compression`` and ``scaling`` entries. Defaults will be
38  provided for any missing elements under ``compression`` and
39  ``scaling``.
40 
41  The allowed entries under ``compression`` are:
42 
43  * ``algorithm`` (`str`): compression algorithm to use
44  * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
45  * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
46  * ``quantizeLevel`` (`float`): cfitsio quantization level
47 
48  The allowed entries under ``scaling`` are:
49 
50  * ``algorithm`` (`str`): scaling algorithm to use
51  * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
52  * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
53  * ``seed`` (`int`): seed for random number generator when fuzzing
54  * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
55  statistics
56  * ``quantizeLevel`` (`float`): divisor of the standard deviation for
57  ``STDEV_*`` scaling
58  * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
59  ``STDEV_POSITIVE``/``NEGATIVE``)
60  * ``bscale`` (`float`): manually specified ``BSCALE``
61  (for ``MANUAL`` scaling)
62  * ``bzero`` (`float`): manually specified ``BSCALE``
63  (for ``MANUAL`` scaling)
64 
65  A very simple example YAML recipe:
66 
67  .. code-block:: yaml
68 
69  lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
70  default:
71  image: &default
72  compression:
73  algorithm: GZIP_SHUFFLE
74  mask: *default
75  variance: *default
76 
77  """
78  supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz"})
79  extension = ".fits"
80  _metadata = None
81  supportedWriteParameters = frozenset({"recipe"})
82 
83  @property
84  def metadata(self):
85  """The metadata read from this file. It will be stripped as
86  components are extracted from it
87  (`lsst.daf.base.PropertyList`).
88  """
89  if self._metadata is None:
90  self._metadata = self.readMetadata()
91  return self._metadata
92 
93  def readMetadata(self):
94  """Read all header metadata directly into a PropertyList.
95 
96  Returns
97  -------
98  metadata : `~lsst.daf.base.PropertyList`
99  Header metadata.
100  """
101  from lsst.afw.image import readMetadata
102  md = readMetadata(self.fileDescriptor.location.path)
103  fix_header(md)
104  return md
105 
106  def stripMetadata(self):
107  """Remove metadata entries that are parsed into components.
108 
109  This is only called when just the metadata is requested; stripping
110  entries there forces code that wants other components to ask for those
111  components directly rather than trying to extract them from the
112  metadata manually, which is fragile. This behavior is an intentional
113  change from Gen2.
114 
115  Parameters
116  ----------
117  metadata : `~lsst.daf.base.PropertyList`
118  Header metadata, to be modified in-place.
119  """
120  # TODO: make sure this covers everything, by delegating to something
121  # that doesn't yet exist in afw.image.ExposureInfo.
122  from lsst.afw.image import bboxFromMetadata
123  from lsst.afw.geom import makeSkyWcs
124  bboxFromMetadata(self.metadata) # always strips
125  makeSkyWcs(self.metadata, strip=True)
126 
127  def readComponent(self, component, parameters=None):
128  """Read a component held by the Exposure.
129 
130  Parameters
131  ----------
132  component : `str`, optional
133  Component to read from the file.
134  parameters : `dict`, optional
135  If specified, a dictionary of slicing parameters that
136  overrides those in ``fileDescriptor``.
137 
138  Returns
139  -------
140  obj : component-dependent
141  In-memory component object.
142 
143  Raises
144  ------
145  KeyError
146  Raised if the requested component cannot be handled.
147  """
148  componentMap = {'wcs': ('readWcs', False),
149  'coaddInputs': ('readCoaddInputs', False),
150  'psf': ('readPsf', False),
151  'image': ('readImage', True),
152  'mask': ('readMask', True),
153  'variance': ('readVariance', True),
154  'photoCalib': ('readPhotoCalib', False),
155  'bbox': ('readBBox', True),
156  'xy0': ('readXY0', True),
157  'metadata': ('readMetadata', False),
158  'filter': ('readFilter', False),
159  'polygon': ('readValidPolygon', False),
160  'apCorrMap': ('readApCorrMap', False),
161  'visitInfo': ('readVisitInfo', False),
162  'transmissionCurve': ('readTransmissionCurve', False),
163  'detector': ('readDetector', False),
164  'extras': ('readExtraComponents', False),
165  'exposureInfo': ('readExposureInfo', False),
166  }
167  method, hasParams = componentMap.get(component, None)
168 
169  if method:
170  reader = ExposureFitsReader(self.fileDescriptor.location.path)
171  caller = getattr(reader, method, None)
172 
173  if caller:
174  if parameters is None:
175  parameters = self.fileDescriptor.parameters
176  if parameters is None:
177  parameters = {}
178  self.fileDescriptor.storageClass.validateParameters(parameters)
179 
180  if hasParams and parameters:
181  return caller(**parameters)
182  else:
183  return caller()
184  else:
185  raise KeyError(f"Unknown component requested: {component}")
186 
187  def readFull(self, parameters=None):
188  """Read the full Exposure object.
189 
190  Parameters
191  ----------
192  parameters : `dict`, optional
193  If specified a dictionary of slicing parameters that overrides
194  those in ``fileDescriptor``.
195 
196  Returns
197  -------
198  exposure : `~lsst.afw.image.Exposure`
199  Complete in-memory exposure.
200  """
201  fileDescriptor = self.fileDescriptor
202  if parameters is None:
203  parameters = fileDescriptor.parameters
204  if parameters is None:
205  parameters = {}
206  fileDescriptor.storageClass.validateParameters(parameters)
207  try:
208  output = fileDescriptor.storageClass.pytype(fileDescriptor.location.path, **parameters)
209  except TypeError:
210  reader = ExposureFitsReader(fileDescriptor.location.path)
211  output = reader.read(**parameters)
212  return output
213 
214  def read(self, component=None, parameters=None):
215  """Read data from a file.
216 
217  Parameters
218  ----------
219  component : `str`, optional
220  Component to read from the file. Only used if the `StorageClass`
221  for reading differed from the `StorageClass` used to write the
222  file.
223  parameters : `dict`, optional
224  If specified, a dictionary of slicing parameters that
225  overrides those in ``fileDescriptor``.
226 
227  Returns
228  -------
229  inMemoryDataset : `object`
230  The requested data as a Python object. The type of object
231  is controlled by the specific formatter.
232 
233  Raises
234  ------
235  ValueError
236  Component requested but this file does not seem to be a concrete
237  composite.
238  KeyError
239  Raised when parameters passed with fileDescriptor are not
240  supported.
241  """
242  fileDescriptor = self.fileDescriptor
243  if fileDescriptor.readStorageClass != fileDescriptor.storageClass:
244  if component == "metadata":
245  self.stripMetadata()
246  return self.metadata
247  elif component is not None:
248  return self.readComponent(component, parameters)
249  else:
250  raise ValueError("Storage class inconsistency ({} vs {}) but no"
251  " component requested".format(fileDescriptor.readStorageClass.name,
252  fileDescriptor.storageClass.name))
253  return self.readFull(parameters=parameters)
254 
255  def write(self, inMemoryDataset):
256  """Write a Python object to a file.
257 
258  Parameters
259  ----------
260  inMemoryDataset : `object`
261  The Python object to store.
262 
263  Returns
264  -------
265  path : `str`
266  The `URI` where the primary file is stored.
267  """
268  # Update the location with the formatter-preferred file extension
269  self.fileDescriptor.location.updateExtension(self.extension)
270  outputPath = self.fileDescriptor.location.path
271 
272  # check to see if we have a recipe requested
273  recipeName = self.writeParameters.get("recipe")
274  recipe = self.getImageCompressionSettings(recipeName)
275  if recipe:
276  # Can not construct a PropertySet from a hierarchical
277  # dict but can update one.
278  ps = PropertySet()
279  ps.update(recipe)
280  inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
281  else:
282  inMemoryDataset.writeFits(outputPath)
283  return self.fileDescriptor.location.pathInStore
284 
285  def getImageCompressionSettings(self, recipeName):
286  """Retrieve the relevant compression settings for this recipe.
287 
288  Parameters
289  ----------
290  recipeName : `str`
291  Label associated with the collection of compression parameters
292  to select.
293 
294  Returns
295  -------
296  settings : `dict`
297  The selected settings.
298  """
299  # if no recipe has been provided and there is no default
300  # return immediately
301  if not recipeName:
302  if "default" not in self.writeRecipes:
303  return {}
304  recipeName = "default"
305 
306  if recipeName not in self.writeRecipes:
307  raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
308 
309  recipe = self.writeRecipes[recipeName]
310 
311  # Set the seed based on dataId
312  seed = hash(tuple(self.dataId.items())) % 2**31
313  for plane in ("image", "mask", "variance"):
314  if plane in recipe and "scaling" in recipe[plane]:
315  scaling = recipe[plane]["scaling"]
316  if "seed" in scaling and scaling["seed"] == 0:
317  scaling["seed"] = seed
318 
319  return recipe
320 
321  @classmethod
322  def validateWriteRecipes(cls, recipes):
323  """Validate supplied recipes for this formatter.
324 
325  The recipes are supplemented with default values where appropriate.
326 
327  TODO: replace this custom validation code with Cerberus (DM-11846)
328 
329  Parameters
330  ----------
331  recipes : `dict`
332  Recipes to validate. Can be empty dict or `None`.
333 
334  Returns
335  -------
336  validated : `dict`
337  Validated recipes. Returns what was given if there are no
338  recipes listed.
339 
340  Raises
341  ------
342  RuntimeError
343  Raised if validation fails.
344  """
345  # Schemas define what should be there, and the default values (and by the default
346  # value, the expected type).
347  compressionSchema = {
348  "algorithm": "NONE",
349  "rows": 1,
350  "columns": 0,
351  "quantizeLevel": 0.0,
352  }
353  scalingSchema = {
354  "algorithm": "NONE",
355  "bitpix": 0,
356  "maskPlanes": ["NO_DATA"],
357  "seed": 0,
358  "quantizeLevel": 4.0,
359  "quantizePad": 5.0,
360  "fuzz": True,
361  "bscale": 1.0,
362  "bzero": 0.0,
363  }
364 
365  if not recipes:
366  # We can not insist on recipes being specified
367  return recipes
368 
369  def checkUnrecognized(entry, allowed, description):
370  """Check to see if the entry contains unrecognised keywords"""
371  unrecognized = set(entry) - set(allowed)
372  if unrecognized:
373  raise RuntimeError(
374  f"Unrecognized entries when parsing image compression recipe {description}: "
375  f"{unrecognized}")
376 
377  validated = {}
378  for name in recipes:
379  checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
380  validated[name] = {}
381  for plane in ("image", "mask", "variance"):
382  checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
383  f"{name}->{plane}")
384 
385  np = {}
386  validated[name][plane] = np
387  for settings, schema in (("compression", compressionSchema),
388  ("scaling", scalingSchema)):
389  np[settings] = {}
390  if settings not in recipes[name][plane]:
391  for key in schema:
392  np[settings][key] = schema[key]
393  continue
394  entry = recipes[name][plane][settings]
395  checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
396  for key in schema:
397  value = type(schema[key])(entry[key]) if key in entry else schema[key]
398  np[settings][key] = value
399  return validated
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.read
def read(self, component=None, parameters=None)
Definition: fitsExposure.py:214
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.write
def write(self, inMemoryDataset)
Definition: fitsExposure.py:255
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.readFull
def readFull(self, parameters=None)
Definition: fitsExposure.py:187
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter._metadata
_metadata
Definition: fitsExposure.py:80
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.stripMetadata
def stripMetadata(self)
Definition: fitsExposure.py:106
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
Definition: fitsExposure.py:30
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.getImageCompressionSettings
def getImageCompressionSettings(self, recipeName)
Definition: fitsExposure.py:285
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.metadata
def metadata(self)
Definition: fitsExposure.py:84
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.validateWriteRecipes
def validateWriteRecipes(cls, recipes)
Definition: fitsExposure.py:322
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.extension
string extension
Definition: fitsExposure.py:79
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.readMetadata
def readMetadata(self)
Definition: fitsExposure.py:93
lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter.readComponent
def readComponent(self, component, parameters=None)
Definition: fitsExposure.py:127