lsst.obs.base  19.0.0-74-g769a877
fitsExposureFormatter.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 __all__ = ("FitsExposureFormatter", )
23 
24 from astro_metadata_translator import fix_header
25 from lsst.daf.butler import Formatter
26 from lsst.afw.image import ExposureFitsReader
27 from lsst.daf.base import PropertySet
28 
29 
30 class FitsExposureFormatter(Formatter):
31  """Interface for reading and writing Exposures to and from FITS files.
32 
33  This Formatter supports write recipes.
34 
35  Each ``FitsExposureFormatter`` recipe for FITS compression should
36  define ``image``, ``mask`` and ``variance`` entries, each of which may
37  contain ``compression`` and ``scaling`` entries. Defaults will be
38  provided for any missing elements under ``compression`` and
39  ``scaling``.
40 
41  The allowed entries under ``compression`` are:
42 
43  * ``algorithm`` (`str`): compression algorithm to use
44  * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
45  * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
46  * ``quantizeLevel`` (`float`): cfitsio quantization level
47 
48  The allowed entries under ``scaling`` are:
49 
50  * ``algorithm`` (`str`): scaling algorithm to use
51  * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
52  * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
53  * ``seed`` (`int`): seed for random number generator when fuzzing
54  * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
55  statistics
56  * ``quantizeLevel`` (`float`): divisor of the standard deviation for
57  ``STDEV_*`` scaling
58  * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
59  ``STDEV_POSITIVE``/``NEGATIVE``)
60  * ``bscale`` (`float`): manually specified ``BSCALE``
61  (for ``MANUAL`` scaling)
62  * ``bzero`` (`float`): manually specified ``BSCALE``
63  (for ``MANUAL`` scaling)
64 
65  A very simple example YAML recipe:
66 
67  .. code-block:: yaml
68 
69  lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
70  default:
71  image: &default
72  compression:
73  algorithm: GZIP_SHUFFLE
74  mask: *default
75  variance: *default
76 
77  """
78  extension = ".fits"
79  _metadata = None
80  supportedWriteParameters = frozenset({"recipe"})
81 
82  @property
83  def metadata(self):
84  """The metadata read from this file. It will be stripped as
85  components are extracted from it
86  (`lsst.daf.base.PropertyList`).
87  """
88  if self._metadata is None:
89  self._metadata = self.readMetadata()
90  return self._metadata
91 
92  def readMetadata(self):
93  """Read all header metadata directly into a PropertyList.
94 
95  Returns
96  -------
97  metadata : `~lsst.daf.base.PropertyList`
98  Header metadata.
99  """
100  from lsst.afw.image import readMetadata
101  md = readMetadata(self.fileDescriptor.location.path)
102  fix_header(md)
103  return md
104 
105  def stripMetadata(self):
106  """Remove metadata entries that are parsed into components.
107 
108  This is only called when just the metadata is requested; stripping
109  entries there forces code that wants other components to ask for those
110  components directly rather than trying to extract them from the
111  metadata manually, which is fragile. This behavior is an intentional
112  change from Gen2.
113 
114  Parameters
115  ----------
116  metadata : `~lsst.daf.base.PropertyList`
117  Header metadata, to be modified in-place.
118  """
119  # TODO: make sure this covers everything, by delegating to something
120  # that doesn't yet exist in afw.image.ExposureInfo.
121  from lsst.afw.image import bboxFromMetadata
122  from lsst.afw.geom import makeSkyWcs
123  bboxFromMetadata(self.metadata) # always strips
124  makeSkyWcs(self.metadata, strip=True)
125 
126  def readComponent(self, component, parameters=None):
127  """Read a component held by the Exposure.
128 
129  Parameters
130  ----------
131  component : `str`, optional
132  Component to read from the file.
133  parameters : `dict`, optional
134  If specified, a dictionary of slicing parameters that
135  overrides those in ``fileDescriptor``.
136 
137  Returns
138  -------
139  obj : component-dependent
140  In-memory component object.
141 
142  Raises
143  ------
144  KeyError
145  Raised if the requested component cannot be handled.
146  """
147  componentMap = {'wcs': ('readWcs', False),
148  'coaddInputs': ('readCoaddInputs', False),
149  'psf': ('readPsf', False),
150  'image': ('readImage', True),
151  'mask': ('readMask', True),
152  'variance': ('readVariance', True),
153  'photoCalib': ('readPhotoCalib', False),
154  'bbox': ('readBBox', True),
155  'xy0': ('readXY0', True),
156  'metadata': ('readMetadata', False),
157  'filter': ('readFilter', False),
158  'polygon': ('readValidPolygon', False),
159  'apCorrMap': ('readApCorrMap', False),
160  'visitInfo': ('readVisitInfo', False),
161  'transmissionCurve': ('readTransmissionCurve', False),
162  'detector': ('readDetector', False),
163  'extras': ('readExtraComponents', False),
164  'exposureInfo': ('readExposureInfo', False),
165  }
166  method, hasParams = componentMap.get(component, None)
167 
168  if method:
169  reader = ExposureFitsReader(self.fileDescriptor.location.path)
170  caller = getattr(reader, method, None)
171 
172  if caller:
173  if parameters is None:
174  parameters = self.fileDescriptor.parameters
175  if parameters is None:
176  parameters = {}
177  self.fileDescriptor.storageClass.validateParameters(parameters)
178 
179  if hasParams and parameters:
180  return caller(**parameters)
181  else:
182  return caller()
183  else:
184  raise KeyError(f"Unknown component requested: {component}")
185 
186  def readFull(self, parameters=None):
187  """Read the full Exposure object.
188 
189  Parameters
190  ----------
191  parameters : `dict`, optional
192  If specified a dictionary of slicing parameters that overrides
193  those in ``fileDescriptor``.
194 
195  Returns
196  -------
197  exposure : `~lsst.afw.image.Exposure`
198  Complete in-memory exposure.
199  """
200  fileDescriptor = self.fileDescriptor
201  if parameters is None:
202  parameters = fileDescriptor.parameters
203  if parameters is None:
204  parameters = {}
205  fileDescriptor.storageClass.validateParameters(parameters)
206  try:
207  output = fileDescriptor.storageClass.pytype(fileDescriptor.location.path, **parameters)
208  except TypeError:
209  reader = ExposureFitsReader(fileDescriptor.location.path)
210  output = reader.read(**parameters)
211  return output
212 
213  def read(self, component=None, parameters=None):
214  """Read data from a file.
215 
216  Parameters
217  ----------
218  component : `str`, optional
219  Component to read from the file. Only used if the `StorageClass`
220  for reading differed from the `StorageClass` used to write the
221  file.
222  parameters : `dict`, optional
223  If specified, a dictionary of slicing parameters that
224  overrides those in ``fileDescriptor``.
225 
226  Returns
227  -------
228  inMemoryDataset : `object`
229  The requested data as a Python object. The type of object
230  is controlled by the specific formatter.
231 
232  Raises
233  ------
234  ValueError
235  Component requested but this file does not seem to be a concrete
236  composite.
237  KeyError
238  Raised when parameters passed with fileDescriptor are not
239  supported.
240  """
241  fileDescriptor = self.fileDescriptor
242  if fileDescriptor.readStorageClass != fileDescriptor.storageClass:
243  if component == "metadata":
244  self.stripMetadata()
245  return self.metadata
246  elif component is not None:
247  return self.readComponent(component, parameters)
248  else:
249  raise ValueError("Storage class inconsistency ({} vs {}) but no"
250  " component requested".format(fileDescriptor.readStorageClass.name,
251  fileDescriptor.storageClass.name))
252  return self.readFull(parameters=parameters)
253 
254  def write(self, inMemoryDataset):
255  """Write a Python object to a file.
256 
257  Parameters
258  ----------
259  inMemoryDataset : `object`
260  The Python object to store.
261 
262  Returns
263  -------
264  path : `str`
265  The `URI` where the primary file is stored.
266  """
267  # Update the location with the formatter-preferred file extension
268  self.fileDescriptor.location.updateExtension(self.extension)
269  outputPath = self.fileDescriptor.location.path
270 
271  # check to see if we have a recipe requested
272  recipeName = self.writeParameters.get("recipe")
273  recipe = self.getImageCompressionSettings(recipeName)
274  if recipe:
275  # Can not construct a PropertySet from a hierarchical
276  # dict but can update one.
277  ps = PropertySet()
278  ps.update(recipe)
279  inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
280  else:
281  inMemoryDataset.writeFits(outputPath)
282  return self.fileDescriptor.location.pathInStore
283 
284  def getImageCompressionSettings(self, recipeName):
285  """Retrieve the relevant compression settings for this recipe.
286 
287  Parameters
288  ----------
289  recipeName : `str`
290  Label associated with the collection of compression parameters
291  to select.
292 
293  Returns
294  -------
295  settings : `dict`
296  The selected settings.
297  """
298  # if no recipe has been provided and there is no default
299  # return immediately
300  if not recipeName:
301  if "default" not in self.writeRecipes:
302  return {}
303  recipeName = "default"
304 
305  if recipeName not in self.writeRecipes:
306  raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
307 
308  recipe = self.writeRecipes[recipeName]
309 
310  # Set the seed based on dataId
311  seed = hash(tuple(self.dataId.items())) % 2**31
312  for plane in ("image", "mask", "variance"):
313  if plane in recipe and "scaling" in recipe[plane]:
314  scaling = recipe[plane]["scaling"]
315  if "seed" in scaling and scaling["seed"] == 0:
316  scaling["seed"] = seed
317 
318  return recipe
319 
320  @classmethod
321  def validateWriteRecipes(cls, recipes):
322  """Validate supplied recipes for this formatter.
323 
324  The recipes are supplemented with default values where appropriate.
325 
326  TODO: replace this custom validation code with Cerberus (DM-11846)
327 
328  Parameters
329  ----------
330  recipes : `dict`
331  Recipes to validate. Can be empty dict or `None`.
332 
333  Returns
334  -------
335  validated : `dict`
336  Validated recipes. Returns what was given if there are no
337  recipes listed.
338 
339  Raises
340  ------
341  RuntimeError
342  Raised if validation fails.
343  """
344  # Schemas define what should be there, and the default values (and by the default
345  # value, the expected type).
346  compressionSchema = {
347  "algorithm": "NONE",
348  "rows": 1,
349  "columns": 0,
350  "quantizeLevel": 0.0,
351  }
352  scalingSchema = {
353  "algorithm": "NONE",
354  "bitpix": 0,
355  "maskPlanes": ["NO_DATA"],
356  "seed": 0,
357  "quantizeLevel": 4.0,
358  "quantizePad": 5.0,
359  "fuzz": True,
360  "bscale": 1.0,
361  "bzero": 0.0,
362  }
363 
364  if not recipes:
365  # We can not insist on recipes being specified
366  return recipes
367 
368  def checkUnrecognized(entry, allowed, description):
369  """Check to see if the entry contains unrecognised keywords"""
370  unrecognized = set(entry) - set(allowed)
371  if unrecognized:
372  raise RuntimeError(
373  f"Unrecognized entries when parsing image compression recipe {description}: "
374  f"{unrecognized}")
375 
376  validated = {}
377  for name in recipes:
378  checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
379  validated[name] = {}
380  for plane in ("image", "mask", "variance"):
381  checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
382  f"{name}->{plane}")
383 
384  np = {}
385  validated[name][plane] = np
386  for settings, schema in (("compression", compressionSchema),
387  ("scaling", scalingSchema)):
388  np[settings] = {}
389  if settings not in recipes[name][plane]:
390  for key in schema:
391  np[settings][key] = schema[key]
392  continue
393  entry = recipes[name][plane][settings]
394  checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
395  for key in schema:
396  value = type(schema[key])(entry[key]) if key in entry else schema[key]
397  np[settings][key] = value
398  return validated
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.readMetadata
def readMetadata(self)
Definition: fitsExposureFormatter.py:92
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.metadata
def metadata(self)
Definition: fitsExposureFormatter.py:83
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter._metadata
_metadata
Definition: fitsExposureFormatter.py:79
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.getImageCompressionSettings
def getImageCompressionSettings(self, recipeName)
Definition: fitsExposureFormatter.py:284
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.write
def write(self, inMemoryDataset)
Definition: fitsExposureFormatter.py:254
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.readFull
def readFull(self, parameters=None)
Definition: fitsExposureFormatter.py:186
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.read
def read(self, component=None, parameters=None)
Definition: fitsExposureFormatter.py:213
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.readComponent
def readComponent(self, component, parameters=None)
Definition: fitsExposureFormatter.py:126
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.stripMetadata
def stripMetadata(self)
Definition: fitsExposureFormatter.py:105
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.extension
string extension
Definition: fitsExposureFormatter.py:78
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter
Definition: fitsExposureFormatter.py:30
lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter.validateWriteRecipes
def validateWriteRecipes(cls, recipes)
Definition: fitsExposureFormatter.py:321