Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 17%
194 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 22:31 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 22:31 -0800
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import warnings
31from abc import abstractmethod
32from typing import AbstractSet, ClassVar
34from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
35from lsst.afw.image import (
36 ExposureFitsReader,
37 ExposureInfo,
38 FilterLabel,
39 ImageFitsReader,
40 MaskedImageFitsReader,
41 MaskFitsReader,
42)
44# Needed for ApCorrMap to resolve properly
45from lsst.afw.math import BoundedField # noqa: F401
46from lsst.daf.base import PropertySet
47from lsst.daf.butler import Formatter
48from lsst.utils.classes import cached_getter
51class FitsImageFormatterBase(Formatter):
52 """Base class formatter for image-like storage classes stored via FITS.
54 Notes
55 -----
56 This class makes no assumptions about how many HDUs are used to represent
57 the image on disk, and includes no support for writing. It's really just a
58 collection of miscellaneous boilerplate common to all FITS image
59 formatters.
61 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
62 (even if just to disable them by raising an exception).
63 """
65 extension = ".fits"
66 supportedExtensions: ClassVar[AbstractSet[str]] = frozenset(
67 {".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"}
68 )
70 unsupportedParameters: ClassVar[AbstractSet[str]] = frozenset()
71 """Support all parameters."""
73 @property # type: ignore
74 @cached_getter
75 def checked_parameters(self):
76 """The parameters passed by the butler user, after checking them
77 against the storage class and transforming `None` into an empty `dict`
78 (`dict`).
80 This is computed on first use and then cached. It should never be
81 accessed when writing. Subclasses that need additional checking should
82 delegate to `super` and then check the result before returning it.
83 """
84 parameters = self.fileDescriptor.parameters
85 if parameters is None:
86 parameters = {}
87 self.fileDescriptor.storageClass.validateParameters(parameters)
88 return parameters
90 def read(self, component=None):
91 # Docstring inherited.
92 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
93 if component is not None:
94 return self.readComponent(component)
95 else:
96 raise ValueError(
97 "Storage class inconsistency ({} vs {}) but no"
98 " component requested".format(
99 self.fileDescriptor.readStorageClass.name, self.fileDescriptor.storageClass.name
100 )
101 )
102 return self.readFull()
104 @abstractmethod
105 def readComponent(self, component):
106 """Read a component dataset.
108 Parameters
109 ----------
110 component : `str`, optional
111 Component to read from the file.
113 Returns
114 -------
115 obj : component-dependent
116 In-memory component object.
118 Raises
119 ------
120 KeyError
121 Raised if the requested component cannot be handled.
122 """
123 raise NotImplementedError()
125 @abstractmethod
126 def readFull(self):
127 """Read the full dataset (while still accounting for parameters).
129 Returns
130 -------
131 obj : component-dependent
132 In-memory component object.
134 """
135 raise NotImplementedError()
138class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
139 """Base class formatter for image-like storage classes stored via FITS
140 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
142 Notes
143 -----
144 This class includes no support for writing.
146 Concrete subclasses must provide at least the `ReaderClass` attribute
147 and a `write` implementation (even just to disable writing by raising).
149 The provided implementation of `readComponent` handles only the 'bbox',
150 'dimensions', and 'xy0' components common to all image-like storage
151 classes. Subclasses with additional components should handle them first,
152 then delegate to ``super()`` for these (or, if necessary, delegate first
153 and catch `KeyError`).
155 The provided implementation of `readFull` handles only parameters that
156 can be forwarded directly to the reader class (usually ``bbox`` and
157 ``origin``). Concrete subclasses that need to handle additional parameters
158 should generally reimplement without delegating (the implementation is
159 trivial).
160 """
163class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
164 """Base class interface for image-like storage stored via FITS,
165 written using LSST code.
167 Notes
168 -----
169 Concrete subclasses must provide at least the `ReaderClass` attribute.
171 The provided implementation of `readComponent` handles only the 'bbox',
172 'dimensions', and 'xy0' components common to all image-like storage
173 classes. Subclasses with additional components should handle them first,
174 then delegate to ``super()`` for these (or, if necessary, delegate first
175 and catch `KeyError`).
177 The provided implementation of `readFull` handles only parameters that
178 can be forwarded directly to the reader class (usually ``bbox`` and
179 ``origin``). Concrete subclasses that need to handle additional parameters
180 should generally reimplement without delegating (the implementation is
181 trivial).
183 This Formatter supports write recipes, and assumes its in-memory type has
184 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
186 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
187 define ``image``, ``mask`` and ``variance`` entries, each of which may
188 contain ``compression`` and ``scaling`` entries. Defaults will be
189 provided for any missing elements under ``compression`` and
190 ``scaling``.
192 The allowed entries under ``compression`` are:
194 * ``algorithm`` (`str`): compression algorithm to use
195 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
196 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
197 * ``quantizeLevel`` (`float`): cfitsio quantization level
199 The allowed entries under ``scaling`` are:
201 * ``algorithm`` (`str`): scaling algorithm to use
202 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
203 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
204 * ``seed`` (`int`): seed for random number generator when fuzzing
205 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
206 statistics
207 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
208 ``STDEV_*`` scaling
209 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
210 ``STDEV_POSITIVE``/``NEGATIVE``)
211 * ``bscale`` (`float`): manually specified ``BSCALE``
212 (for ``MANUAL`` scaling)
213 * ``bzero`` (`float`): manually specified ``BSCALE``
214 (for ``MANUAL`` scaling)
216 A very simple example YAML recipe (for the ``Exposure`` specialization):
218 .. code-block:: yaml
220 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
221 default:
222 image: &default
223 compression:
224 algorithm: GZIP_SHUFFLE
225 mask: *default
226 variance: *default
228 """
230 supportedWriteParameters = frozenset({"recipe"})
231 ReaderClass: type # must be set by concrete subclasses
233 @property # type: ignore
234 @cached_getter
235 def reader(self):
236 """The reader object that backs this formatter's read operations.
238 This is computed on first use and then cached. It should never be
239 accessed when writing.
240 """
241 return self.ReaderClass(self.fileDescriptor.location.path)
243 def readComponent(self, component):
244 # Docstring inherited.
245 if component in ("bbox", "dimensions", "xy0"):
246 bbox = self.reader.readBBox()
247 if component == "dimensions":
248 return bbox.getDimensions()
249 elif component == "xy0":
250 return bbox.getMin()
251 else:
252 return bbox
253 else:
254 raise KeyError(f"Unknown component requested: {component}")
256 def readFull(self):
257 # Docstring inherited.
258 return self.reader.read(**self.checked_parameters)
260 def write(self, inMemoryDataset):
261 """Write a Python object to a file.
263 Parameters
264 ----------
265 inMemoryDataset : `object`
266 The Python object to store.
267 """
268 # Update the location with the formatter-preferred file extension
269 self.fileDescriptor.location.updateExtension(self.extension)
270 outputPath = self.fileDescriptor.location.path
272 # check to see if we have a recipe requested
273 recipeName = self.writeParameters.get("recipe")
274 recipe = self.getImageCompressionSettings(recipeName)
275 if recipe:
276 # Can not construct a PropertySet from a hierarchical
277 # dict but can update one.
278 ps = PropertySet()
279 ps.update(recipe)
280 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
281 else:
282 inMemoryDataset.writeFits(outputPath)
284 def getImageCompressionSettings(self, recipeName):
285 """Retrieve the relevant compression settings for this recipe.
287 Parameters
288 ----------
289 recipeName : `str`
290 Label associated with the collection of compression parameters
291 to select.
293 Returns
294 -------
295 settings : `dict`
296 The selected settings.
297 """
298 # if no recipe has been provided and there is no default
299 # return immediately
300 if not recipeName:
301 if "default" not in self.writeRecipes:
302 return {}
303 recipeName = "default"
305 if recipeName not in self.writeRecipes:
306 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
308 recipe = self.writeRecipes[recipeName]
310 # Set the seed based on dataId
311 seed = hash(tuple(self.dataId.items())) % 2**31
312 for plane in ("image", "mask", "variance"):
313 if plane in recipe and "scaling" in recipe[plane]:
314 scaling = recipe[plane]["scaling"]
315 if "seed" in scaling and scaling["seed"] == 0:
316 scaling["seed"] = seed
318 return recipe
320 @classmethod
321 def validateWriteRecipes(cls, recipes):
322 """Validate supplied recipes for this formatter.
324 The recipes are supplemented with default values where appropriate.
326 TODO: replace this custom validation code with Cerberus (DM-11846)
328 Parameters
329 ----------
330 recipes : `dict`
331 Recipes to validate. Can be empty dict or `None`.
333 Returns
334 -------
335 validated : `dict`
336 Validated recipes. Returns what was given if there are no
337 recipes listed.
339 Raises
340 ------
341 RuntimeError
342 Raised if validation fails.
343 """
344 # Schemas define what should be there, and the default values (and by
345 # the default value, the expected type).
346 compressionSchema = {
347 "algorithm": "NONE",
348 "rows": 1,
349 "columns": 0,
350 "quantizeLevel": 0.0,
351 }
352 scalingSchema = {
353 "algorithm": "NONE",
354 "bitpix": 0,
355 "maskPlanes": ["NO_DATA"],
356 "seed": 0,
357 "quantizeLevel": 4.0,
358 "quantizePad": 5.0,
359 "fuzz": True,
360 "bscale": 1.0,
361 "bzero": 0.0,
362 }
364 if not recipes:
365 # We can not insist on recipes being specified
366 return recipes
368 def checkUnrecognized(entry, allowed, description):
369 """Check to see if the entry contains unrecognised keywords"""
370 unrecognized = set(entry) - set(allowed)
371 if unrecognized:
372 raise RuntimeError(
373 f"Unrecognized entries when parsing image compression recipe {description}: "
374 f"{unrecognized}"
375 )
377 validated = {}
378 for name in recipes:
379 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
380 validated[name] = {}
381 for plane in ("image", "mask", "variance"):
382 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], f"{name}->{plane}")
384 np = {}
385 validated[name][plane] = np
386 for settings, schema in (("compression", compressionSchema), ("scaling", scalingSchema)):
387 np[settings] = {}
388 if settings not in recipes[name][plane]:
389 for key in schema:
390 np[settings][key] = schema[key]
391 continue
392 entry = recipes[name][plane][settings]
393 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
394 for key in schema:
395 value = type(schema[key])(entry[key]) if key in entry else schema[key]
396 np[settings][key] = value
397 return validated
400class FitsImageFormatter(StandardFitsImageFormatterBase):
401 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
402 from/to FITS.
403 """
405 ReaderClass = ImageFitsReader
408class FitsMaskFormatter(StandardFitsImageFormatterBase):
409 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
410 from/to FITS.
411 """
413 ReaderClass = MaskFitsReader
416class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
417 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
418 from/to FITS.
419 """
421 ReaderClass = MaskedImageFitsReader
423 def readComponent(self, component):
424 # Docstring inherited.
425 if component == "image":
426 return self.reader.readImage(**self.checked_parameters)
427 elif component == "mask":
428 return self.reader.readMask(**self.checked_parameters)
429 elif component == "variance":
430 return self.reader.readVariance(**self.checked_parameters)
431 else:
432 # Delegate to base for bbox, dimensions, xy0.
433 return super().readComponent(component)
436def standardizeAmplifierParameters(parameters, on_disk_detector):
437 """Preprocess the Exposure storage class's "amp" and "detector" parameters
439 This checks the given objects for consistency with the on-disk geometry and
440 converts amplifier IDs/names to Amplifier instances.
442 Parameters
443 ----------
444 parameters : `dict`
445 Dictionary of parameters passed to formatter. See the Exposure storage
446 class definition in daf_butler for allowed keys and values.
447 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
448 Detector that represents the on-disk image being loaded, or `None` if
449 this is unknown (and hence the user must provide one in
450 ``parameters`` if "amp" is in ``parameters``).
452 Returns
453 -------
454 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
455 An amplifier object that defines a subimage to load, or `None` if there
456 was no "amp" parameter.
457 detector : `lsst.afw.cameraGeom.Detector` or `None`
458 A detector object whose amplifiers are in the same s/orientation
459 state as the on-disk image. If there is no "amp" parameter,
460 ``on_disk_detector`` is simply passed through.
461 regions_differ : `bool`
462 `True` if the on-disk detector and the detector given in the parameters
463 had different bounding boxes for one or more regions. This can happen
464 if the true overscan region sizes can only be determined when the image
465 is actually read, but otherwise it should be considered user error.
466 """
467 if (amplifier := parameters.get("amp")) is None:
468 return None, on_disk_detector, False
469 if "bbox" in parameters or "origin" in parameters:
470 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
471 if isinstance(amplifier, (int, str)):
472 amp_key = amplifier
473 target_amplifier = None
474 else:
475 amp_key = amplifier.getName()
476 target_amplifier = amplifier
477 if (detector := parameters.get("detector")) is not None:
478 if on_disk_detector is not None:
479 # User passed a detector and we also found one on disk. Check them
480 # for consistency. Note that we are checking the amps we'd get
481 # from the two detectors against each other, not the amplifier we
482 # got directly from the user, as the latter is allowed to differ in
483 # assembly/orientation state.
484 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
485 if comparison & comparison.ASSEMBLY_DIFFERS:
486 raise ValueError(
487 "The given 'detector' has a different assembly state and/or orientation from "
488 f"the on-disk one for amp {amp_key}."
489 )
490 else:
491 if on_disk_detector is None:
492 raise ValueError(
493 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
494 "Please provide either a 'detector' parameter or an Amplifier instance in the "
495 "'amp' parameter."
496 )
497 comparison = AmplifierGeometryComparison.EQUAL
498 detector = on_disk_detector
499 if target_amplifier is None:
500 target_amplifier = detector[amp_key]
501 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
504class FitsExposureFormatter(FitsMaskedImageFormatter):
505 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
506 from/to FITS.
508 Notes
509 -----
510 This class inherits from `FitsMaskedImageFormatter` even though
511 `lsst.afw.image.Exposure` doesn't inherit from
512 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
513 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
514 should be replaced with e.g. both calling a free function if that slight
515 type covariance violation ever becomes a practical problem.
516 """
518 ReaderClass = ExposureFitsReader
520 def readComponent(self, component):
521 # Docstring inherited.
522 # Generic components can be read via a string name; DM-27754 will make
523 # this mapping larger at the expense of the following one.
524 genericComponents = {
525 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
526 }
527 if (genericComponentName := genericComponents.get(component)) is not None:
528 return self.reader.readComponent(genericComponentName)
529 # Other components have hard-coded method names, but don't take
530 # parameters.
531 standardComponents = {
532 "id": "readExposureId",
533 "metadata": "readMetadata",
534 "wcs": "readWcs",
535 "coaddInputs": "readCoaddInputs",
536 "psf": "readPsf",
537 "photoCalib": "readPhotoCalib",
538 "filter": "readFilter",
539 # TODO: deprecated; remove in DM-27811
540 "filterLabel": "readFilterLabel",
541 "validPolygon": "readValidPolygon",
542 "apCorrMap": "readApCorrMap",
543 "visitInfo": "readVisitInfo",
544 "transmissionCurve": "readTransmissionCurve",
545 "detector": "readDetector",
546 "exposureInfo": "readExposureInfo",
547 }
548 if (methodName := standardComponents.get(component)) is not None:
549 result = getattr(self.reader, methodName)()
550 # TODO: remove filterLabel in DM-27811
551 if component == "filterLabel":
552 warnings.warn(
553 "Exposure.filterLabel component is deprecated; use .filter instead. "
554 "Will be removed after v24.",
555 FutureWarning,
556 )
557 if component == "filter" or component == "filterLabel":
558 return self._fixFilterLabels(result)
559 return result
560 # Delegate to MaskedImage and ImageBase implementations for the rest.
561 return super().readComponent(component)
563 def readFull(self):
564 # Docstring inherited.
565 amplifier, detector, _ = standardizeAmplifierParameters(
566 self.checked_parameters,
567 self.reader.readDetector(),
568 )
569 if amplifier is not None:
570 amplifier_isolator = AmplifierIsolator(
571 amplifier,
572 self.reader.readBBox(),
573 detector,
574 )
575 result = amplifier_isolator.transform_subimage(
576 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
577 )
578 result.setDetector(amplifier_isolator.make_detector())
579 else:
580 result = self.reader.read(**self.checked_parameters)
581 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
582 return result
584 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
585 """Compare the filter label read from the file with the one in the
586 data ID.
588 Parameters
589 ----------
590 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
591 Filter label read from the file, if there was one.
592 should_be_standardized : `bool`, optional
593 If `True`, expect ``file_filter_label`` to be consistent with the
594 data ID and warn only if it is not. If `False`, expect it to be
595 inconsistent and warn only if the data ID is incomplete and hence
596 the `FilterLabel` cannot be fixed. If `None` (default) guess
597 whether the file should be standardized by looking at the
598 serialization version number in file, which requires this method to
599 have been run after `readFull` or `readComponent`.
601 Returns
602 -------
603 filter_label : `lsst.afw.image.FilterLabel` or `None`
604 The preferred filter label; may be the given one or one built from
605 the data ID. `None` is returned if there should never be any
606 filters associated with this dataset type.
608 Notes
609 -----
610 Most test coverage for this method is in ci_hsc_gen3, where we have
611 much easier access to test data that exhibits the problems it attempts
612 to solve.
613 """
614 # Remember filter data ID keys that weren't in this particular data ID,
615 # so we can warn about them later.
616 missing = []
617 band = None
618 physical_filter = None
619 if "band" in self.dataId.graph.dimensions.names:
620 band = self.dataId.get("band")
621 # band isn't in the data ID; is that just because this data ID
622 # hasn't been filled in with everything the Registry knows, or
623 # because this dataset is never associated with a band?
624 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
625 missing.append("band")
626 if "physical_filter" in self.dataId.graph.dimensions.names:
627 physical_filter = self.dataId.get("physical_filter")
628 # Same check as above for band, but for physical_filter.
629 if (
630 physical_filter is None
631 and not self.dataId.hasFull()
632 and "physical_filter" in self.dataId.graph.implied.names
633 ):
634 missing.append("physical_filter")
635 if should_be_standardized is None:
636 version = self.reader.readSerializationVersion()
637 should_be_standardized = version >= 2
638 if missing:
639 # Data ID identifies a filter but the actual filter label values
640 # haven't been fetched from the database; we have no choice but
641 # to use the one in the file.
642 # Warn if that's more likely than not to be bad, because the file
643 # predates filter standardization.
644 if not should_be_standardized:
645 warnings.warn(
646 f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
647 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
648 "Call Registry.expandDataId before Butler.get to avoid this."
649 )
650 return file_filter_label
651 if band is None and physical_filter is None:
652 data_id_filter_label = None
653 else:
654 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
655 if data_id_filter_label != file_filter_label and should_be_standardized:
656 # File was written after FilterLabel and standardization, but its
657 # FilterLabel doesn't agree with the data ID: this indicates a bug
658 # in whatever code produced the Exposure (though it may be one that
659 # has been fixed since the file was written).
660 warnings.warn(
661 f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
662 f"filter label mismatch (file is {file_filter_label}, data ID is "
663 f"{data_id_filter_label}). This is probably a bug in the code that produced it."
664 )
665 return data_id_filter_label