Coverage for python/lsst/obs/base/formatters/fitsExposure.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("FitsExposureFormatter", "FitsImageFormatter", "FitsMaskFormatter",
23 "FitsMaskedImageFormatter")
25from abc import abstractmethod
26import warnings
28from lsst.daf.base import PropertySet
29from lsst.daf.butler import Formatter
30from lsst.daf.butler.core.utils import cached_getter
31from lsst.afw.image import ExposureFitsReader, ImageFitsReader, MaskFitsReader, MaskedImageFitsReader
32from lsst.afw.image import ExposureInfo, FilterLabel
33# Needed for ApCorrMap to resolve properly
34from lsst.afw.math import BoundedField # noqa: F401
37class FitsImageFormatterBase(Formatter):
38 """Base class formatter for image-like storage classes stored via FITS.
40 Notes
41 -----
42 This class makes no assumptions about how many HDUs are used to represent
43 the image on disk, and includes no support for writing. It's really just a
44 collection of miscellaneous boilerplate common to all FITS image
45 formatters.
47 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
48 (even if just to disable them by raising an exception).
49 """
51 extension = ".fits"
52 supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
54 unsupportedParameters = {}
55 """Support all parameters."""
57 @property
58 @cached_getter
59 def checked_parameters(self):
60 """The parameters passed by the butler user, after checking them
61 against the storage class and transforming `None` into an empty `dict`
62 (`dict`).
64 This is computed on first use and then cached. It should never be
65 accessed when writing. Subclasses that need additional checking should
66 delegate to `super` and then check the result before returning it.
67 """
68 parameters = self.fileDescriptor.parameters
69 if parameters is None:
70 parameters = {}
71 self.fileDescriptor.storageClass.validateParameters(parameters)
72 return parameters
74 def read(self, component=None):
75 # Docstring inherited.
76 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
77 if component is not None:
78 return self.readComponent(component)
79 else:
80 raise ValueError("Storage class inconsistency ({} vs {}) but no"
81 " component requested".format(self.fileDescriptor.readStorageClass.name,
82 self.fileDescriptor.storageClass.name))
83 return self.readFull()
85 @abstractmethod
86 def readComponent(self, component):
87 """Read a component dataset.
89 Parameters
90 ----------
91 component : `str`, optional
92 Component to read from the file.
94 Returns
95 -------
96 obj : component-dependent
97 In-memory component object.
99 Raises
100 ------
101 KeyError
102 Raised if the requested component cannot be handled.
103 """
104 raise NotImplementedError()
106 @abstractmethod
107 def readFull(self):
108 """Read the full dataset (while still accounting for parameters).
110 Returns
111 -------
112 obj : component-dependent
113 In-memory component object.
115 """
116 raise NotImplementedError()
119class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
120 """Base class formatter for image-like storage classes stored via FITS
121 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
123 Notes
124 -----
125 This class includes no support for writing.
127 Concrete subclasses must provide at least the `ReaderClass` attribute
128 and a `write` implementation (even just to disable writing by raising).
130 The provided implementation of `readComponent` handles only the 'bbox',
131 'dimensions', and 'xy0' components common to all image-like storage
132 classes. Subclasses with additional components should handle them first,
133 then delegate to ``super()`` for these (or, if necessary, delegate first
134 and catch `KeyError`).
136 The provided implementation of `readFull` handles only parameters that
137 can be forwarded directly to the reader class (usually ``bbox`` and
138 ``origin``). Concrete subclasses that need to handle additional parameters
139 should generally reimplement without delegating (the implementation is
140 trivial).
141 """
144class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
145 """Base class interface for image-like storage stored via FITS,
146 written using LSST code.
148 Notes
149 -----
150 Concrete subclasses must provide at least the `ReaderClass` attribute.
152 The provided implementation of `readComponent` handles only the 'bbox',
153 'dimensions', and 'xy0' components common to all image-like storage
154 classes. Subclasses with additional components should handle them first,
155 then delegate to ``super()`` for these (or, if necessary, delegate first
156 and catch `KeyError`).
158 The provided implementation of `readFull` handles only parameters that
159 can be forwarded directly to the reader class (usually ``bbox`` and
160 ``origin``). Concrete subclasses that need to handle additional parameters
161 should generally reimplement without delegating (the implementation is
162 trivial).
164 This Formatter supports write recipes, and assumes its in-memory type has
165 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
167 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
168 define ``image``, ``mask`` and ``variance`` entries, each of which may
169 contain ``compression`` and ``scaling`` entries. Defaults will be
170 provided for any missing elements under ``compression`` and
171 ``scaling``.
173 The allowed entries under ``compression`` are:
175 * ``algorithm`` (`str`): compression algorithm to use
176 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
177 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
178 * ``quantizeLevel`` (`float`): cfitsio quantization level
180 The allowed entries under ``scaling`` are:
182 * ``algorithm`` (`str`): scaling algorithm to use
183 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
184 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
185 * ``seed`` (`int`): seed for random number generator when fuzzing
186 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
187 statistics
188 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
189 ``STDEV_*`` scaling
190 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
191 ``STDEV_POSITIVE``/``NEGATIVE``)
192 * ``bscale`` (`float`): manually specified ``BSCALE``
193 (for ``MANUAL`` scaling)
194 * ``bzero`` (`float`): manually specified ``BSCALE``
195 (for ``MANUAL`` scaling)
197 A very simple example YAML recipe (for the ``Exposure`` specialization):
199 .. code-block:: yaml
201 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
202 default:
203 image: &default
204 compression:
205 algorithm: GZIP_SHUFFLE
206 mask: *default
207 variance: *default
209 """
210 supportedWriteParameters = frozenset({"recipe"})
211 ReaderClass: type # must be set by concrete subclasses
213 @property
214 @cached_getter
215 def reader(self):
216 """The reader object that backs this formatter's read operations.
218 This is computed on first use and then cached. It should never be
219 accessed when writing.
220 """
221 return self.ReaderClass(self.fileDescriptor.location.path)
223 def readComponent(self, component):
224 # Docstring inherited.
225 if component in ("bbox", "dimensions", "xy0"):
226 bbox = self.reader.readBBox()
227 if component == "dimensions":
228 return bbox.getDimensions()
229 elif component == "xy0":
230 return bbox.getMin()
231 else:
232 return bbox
233 else:
234 raise KeyError(f"Unknown component requested: {component}")
236 def readFull(self):
237 # Docstring inherited.
238 return self.reader.read(**self.checked_parameters)
240 def write(self, inMemoryDataset):
241 """Write a Python object to a file.
243 Parameters
244 ----------
245 inMemoryDataset : `object`
246 The Python object to store.
247 """
248 # Update the location with the formatter-preferred file extension
249 self.fileDescriptor.location.updateExtension(self.extension)
250 outputPath = self.fileDescriptor.location.path
252 # check to see if we have a recipe requested
253 recipeName = self.writeParameters.get("recipe")
254 recipe = self.getImageCompressionSettings(recipeName)
255 if recipe:
256 # Can not construct a PropertySet from a hierarchical
257 # dict but can update one.
258 ps = PropertySet()
259 ps.update(recipe)
260 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
261 else:
262 inMemoryDataset.writeFits(outputPath)
264 def getImageCompressionSettings(self, recipeName):
265 """Retrieve the relevant compression settings for this recipe.
267 Parameters
268 ----------
269 recipeName : `str`
270 Label associated with the collection of compression parameters
271 to select.
273 Returns
274 -------
275 settings : `dict`
276 The selected settings.
277 """
278 # if no recipe has been provided and there is no default
279 # return immediately
280 if not recipeName:
281 if "default" not in self.writeRecipes:
282 return {}
283 recipeName = "default"
285 if recipeName not in self.writeRecipes:
286 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
288 recipe = self.writeRecipes[recipeName]
290 # Set the seed based on dataId
291 seed = hash(tuple(self.dataId.items())) % 2**31
292 for plane in ("image", "mask", "variance"):
293 if plane in recipe and "scaling" in recipe[plane]:
294 scaling = recipe[plane]["scaling"]
295 if "seed" in scaling and scaling["seed"] == 0:
296 scaling["seed"] = seed
298 return recipe
300 @classmethod
301 def validateWriteRecipes(cls, recipes):
302 """Validate supplied recipes for this formatter.
304 The recipes are supplemented with default values where appropriate.
306 TODO: replace this custom validation code with Cerberus (DM-11846)
308 Parameters
309 ----------
310 recipes : `dict`
311 Recipes to validate. Can be empty dict or `None`.
313 Returns
314 -------
315 validated : `dict`
316 Validated recipes. Returns what was given if there are no
317 recipes listed.
319 Raises
320 ------
321 RuntimeError
322 Raised if validation fails.
323 """
324 # Schemas define what should be there, and the default values (and by
325 # the default value, the expected type).
326 compressionSchema = {
327 "algorithm": "NONE",
328 "rows": 1,
329 "columns": 0,
330 "quantizeLevel": 0.0,
331 }
332 scalingSchema = {
333 "algorithm": "NONE",
334 "bitpix": 0,
335 "maskPlanes": ["NO_DATA"],
336 "seed": 0,
337 "quantizeLevel": 4.0,
338 "quantizePad": 5.0,
339 "fuzz": True,
340 "bscale": 1.0,
341 "bzero": 0.0,
342 }
344 if not recipes:
345 # We can not insist on recipes being specified
346 return recipes
348 def checkUnrecognized(entry, allowed, description):
349 """Check to see if the entry contains unrecognised keywords"""
350 unrecognized = set(entry) - set(allowed)
351 if unrecognized:
352 raise RuntimeError(
353 f"Unrecognized entries when parsing image compression recipe {description}: "
354 f"{unrecognized}")
356 validated = {}
357 for name in recipes:
358 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
359 validated[name] = {}
360 for plane in ("image", "mask", "variance"):
361 checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
362 f"{name}->{plane}")
364 np = {}
365 validated[name][plane] = np
366 for settings, schema in (("compression", compressionSchema),
367 ("scaling", scalingSchema)):
368 np[settings] = {}
369 if settings not in recipes[name][plane]:
370 for key in schema:
371 np[settings][key] = schema[key]
372 continue
373 entry = recipes[name][plane][settings]
374 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
375 for key in schema:
376 value = type(schema[key])(entry[key]) if key in entry else schema[key]
377 np[settings][key] = value
378 return validated
381class FitsImageFormatter(StandardFitsImageFormatterBase):
382 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
383 from/to FITS.
384 """
386 ReaderClass = ImageFitsReader
389class FitsMaskFormatter(StandardFitsImageFormatterBase):
390 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
391 from/to FITS.
392 """
394 ReaderClass = MaskFitsReader
397class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
398 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
399 from/to FITS.
400 """
402 ReaderClass = MaskedImageFitsReader
404 def readComponent(self, component):
405 # Docstring inherited.
406 if component == "image":
407 return self.reader.readImage(**self.checked_parameters)
408 elif component == "mask":
409 return self.reader.readMask(**self.checked_parameters)
410 elif component == "variance":
411 return self.reader.readVariance(**self.checked_parameters)
412 else:
413 # Delegate to base for bbox, dimensions, xy0.
414 return super().readComponent(component)
417class FitsExposureFormatter(FitsMaskedImageFormatter):
418 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
419 from/to FITS.
421 Notes
422 -----
423 This class inherits from `FitsMaskedImageFormatter` even though
424 `lsst.afw.image.Exposure` doesn't inherit from
425 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
426 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
427 should be replaced with e.g. both calling a free function if that slight
428 type covariance violation ever becomes a practical problem.
429 """
431 ReaderClass = ExposureFitsReader
433 def readComponent(self, component):
434 # Docstring inherited.
435 # Generic components can be read via a string name; DM-27754 will make
436 # this mapping larger at the expense of the following one.
437 genericComponents = {
438 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
439 }
440 if (genericComponentName := genericComponents.get(component)) is not None:
441 return self.reader.readComponent(genericComponentName)
442 # Other components have hard-coded method names, but don't take
443 # parameters.
444 standardComponents = {
445 'metadata': 'readMetadata',
446 'wcs': 'readWcs',
447 'coaddInputs': 'readCoaddInputs',
448 'psf': 'readPsf',
449 'photoCalib': 'readPhotoCalib',
450 # TODO: deprecate in DM-27170, remove in DM-27177
451 'filter': 'readFilter',
452 # TODO: deprecate in DM-27177, remove in DM-27811
453 'filterLabel': 'readFilterLabel',
454 'validPolygon': 'readValidPolygon',
455 'apCorrMap': 'readApCorrMap',
456 'visitInfo': 'readVisitInfo',
457 'transmissionCurve': 'readTransmissionCurve',
458 'detector': 'readDetector',
459 'exposureInfo': 'readExposureInfo',
460 }
461 if (methodName := standardComponents.get(component)) is not None:
462 result = getattr(self.reader, methodName)()
463 if component == "filterLabel":
464 return self._fixFilterLabels(result)
465 return result
466 # Delegate to MaskedImage and ImageBase implementations for the rest.
467 return super().readComponent(component)
469 def readFull(self):
470 # Docstring inherited.
471 full = super().readFull()
472 full.getInfo().setFilterLabel(self._fixFilterLabels(full.getInfo().getFilterLabel()))
473 return full
475 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
476 """Compare the filter label read from the file with the one in the
477 data ID.
479 Parameters
480 ----------
481 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
482 Filter label read from the file, if there was one.
483 should_be_standardized : `bool`, optional
484 If `True`, expect ``file_filter_label`` to be consistent with the
485 data ID and warn only if it is not. If `False`, expect it to be
486 inconsistent and warn only if the data ID is incomplete and hence
487 the `FilterLabel` cannot be fixed. If `None` (default) guess
488 whether the file should be standardized by looking at the
489 serialization version number in file, which requires this method to
490 have been run after `readFull` or `readComponent`.
492 Returns
493 -------
494 filter_label : `lsst.afw.image.FilterLabel` or `None`
495 The preferred filter label; may be the given one or one built from
496 the data ID. `None` is returned if there should never be any
497 filters associated with this dataset type.
499 Notes
500 -----
501 Most test coverage for this method is in ci_hsc_gen3, where we have
502 much easier access to test data that exhibits the problems it attempts
503 to solve.
504 """
505 # Remember filter data ID keys that weren't in this particular data ID,
506 # so we can warn about them later.
507 missing = []
508 band = None
509 physical_filter = None
510 if "band" in self.dataId.graph.dimensions.names:
511 band = self.dataId.get("band")
512 # band isn't in the data ID; is that just because this data ID
513 # hasn't been filled in with everything the Registry knows, or
514 # because this dataset is never associated with a band?
515 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
516 missing.append("band")
517 if "physical_filter" in self.dataId.graph.dimensions.names:
518 physical_filter = self.dataId.get("physical_filter")
519 # Same check as above for band, but for physical_filter.
520 if (physical_filter is None and not self.dataId.hasFull()
521 and "physical_filter" in self.dataId.graph.implied.names):
522 missing.append("physical_filter")
523 if should_be_standardized is None:
524 version = self.reader.readSerializationVersion()
525 should_be_standardized = (version >= 2)
526 if missing:
527 # Data ID identifies a filter but the actual filter label values
528 # haven't been fetched from the database; we have no choice but
529 # to use the one in the file.
530 # Warn if that's more likely than not to be bad, because the file
531 # predates filter standardization.
532 if not should_be_standardized:
533 warnings.warn(f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
534 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
535 "Call Registry.expandDataId before Butler.get to avoid this.")
536 return file_filter_label
537 if band is None and physical_filter is None:
538 data_id_filter_label = None
539 else:
540 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
541 if data_id_filter_label != file_filter_label and should_be_standardized:
542 # File was written after FilterLabel and standardization, but its
543 # FilterLabel doesn't agree with the data ID: this indicates a bug
544 # in whatever code produced the Exposure (though it may be one that
545 # has been fixed since the file was written).
546 warnings.warn(f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
547 f"filter label mismatch (file is {file_filter_label}, data ID is "
548 f"{data_id_filter_label}). This is probably a bug in the code that produced it.")
549 return data_id_filter_label