Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 21%
195 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-30 02:46 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-30 02:46 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import warnings
31from abc import abstractmethod
32from typing import AbstractSet, ClassVar
34from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
35from lsst.afw.image import (
36 ExposureFitsReader,
37 ExposureInfo,
38 FilterLabel,
39 ImageFitsReader,
40 MaskedImageFitsReader,
41 MaskFitsReader,
42)
44# Needed for ApCorrMap to resolve properly
45from lsst.afw.math import BoundedField # noqa: F401
46from lsst.daf.base import PropertySet
47from lsst.daf.butler import Formatter
48from lsst.utils.classes import cached_getter
51class FitsImageFormatterBase(Formatter):
52 """Base class formatter for image-like storage classes stored via FITS.
54 Notes
55 -----
56 This class makes no assumptions about how many HDUs are used to represent
57 the image on disk, and includes no support for writing. It's really just a
58 collection of miscellaneous boilerplate common to all FITS image
59 formatters.
61 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
62 (even if just to disable them by raising an exception).
63 """
65 extension = ".fits"
66 supportedExtensions: ClassVar[AbstractSet[str]] = frozenset(
67 {".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"}
68 )
70 unsupportedParameters: ClassVar[AbstractSet[str]] = frozenset()
71 """Support all parameters."""
73 @property # type: ignore
74 @cached_getter
75 def checked_parameters(self):
76 """The parameters passed by the butler user, after checking them
77 against the storage class and transforming `None` into an empty `dict`
78 (`dict`).
80 This is computed on first use and then cached. It should never be
81 accessed when writing. Subclasses that need additional checking should
82 delegate to `super` and then check the result before returning it.
83 """
84 parameters = self.fileDescriptor.parameters
85 if parameters is None:
86 parameters = {}
87 self.fileDescriptor.storageClass.validateParameters(parameters)
88 return parameters
90 def read(self, component=None):
91 # Docstring inherited.
92 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
93 if component is not None:
94 return self.readComponent(component)
95 else:
96 raise ValueError(
97 "Storage class inconsistency ({} vs {}) but no"
98 " component requested".format(
99 self.fileDescriptor.readStorageClass.name, self.fileDescriptor.storageClass.name
100 )
101 )
102 return self.readFull()
104 @abstractmethod
105 def readComponent(self, component):
106 """Read a component dataset.
108 Parameters
109 ----------
110 component : `str`, optional
111 Component to read from the file.
113 Returns
114 -------
115 obj : component-dependent
116 In-memory component object.
118 Raises
119 ------
120 KeyError
121 Raised if the requested component cannot be handled.
122 """
123 raise NotImplementedError()
125 @abstractmethod
126 def readFull(self):
127 """Read the full dataset (while still accounting for parameters).
129 Returns
130 -------
131 obj : component-dependent
132 In-memory component object.
134 """
135 raise NotImplementedError()
138class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
139 """Base class formatter for image-like storage classes stored via FITS
140 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
142 Notes
143 -----
144 This class includes no support for writing.
146 Concrete subclasses must provide at least the `ReaderClass` attribute
147 and a `write` implementation (even just to disable writing by raising).
149 The provided implementation of `readComponent` handles only the 'bbox',
150 'dimensions', and 'xy0' components common to all image-like storage
151 classes. Subclasses with additional components should handle them first,
152 then delegate to ``super()`` for these (or, if necessary, delegate first
153 and catch `KeyError`).
155 The provided implementation of `readFull` handles only parameters that
156 can be forwarded directly to the reader class (usually ``bbox`` and
157 ``origin``). Concrete subclasses that need to handle additional parameters
158 should generally reimplement without delegating (the implementation is
159 trivial).
160 """
163class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
164 """Base class interface for image-like storage stored via FITS,
165 written using LSST code.
167 Notes
168 -----
169 Concrete subclasses must provide at least the `ReaderClass` attribute.
171 The provided implementation of `readComponent` handles only the 'bbox',
172 'dimensions', and 'xy0' components common to all image-like storage
173 classes. Subclasses with additional components should handle them first,
174 then delegate to ``super()`` for these (or, if necessary, delegate first
175 and catch `KeyError`).
177 The provided implementation of `readFull` handles only parameters that
178 can be forwarded directly to the reader class (usually ``bbox`` and
179 ``origin``). Concrete subclasses that need to handle additional parameters
180 should generally reimplement without delegating (the implementation is
181 trivial).
183 This Formatter supports write recipes, and assumes its in-memory type has
184 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
186 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
187 define ``image``, ``mask`` and ``variance`` entries, each of which may
188 contain ``compression`` and ``scaling`` entries. Defaults will be
189 provided for any missing elements under ``compression`` and
190 ``scaling``.
192 The allowed entries under ``compression`` are:
194 * ``algorithm`` (`str`): compression algorithm to use
195 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
196 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
197 * ``quantizeLevel`` (`float`): cfitsio quantization level
199 The allowed entries under ``scaling`` are:
201 * ``algorithm`` (`str`): scaling algorithm to use
202 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
203 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
204 * ``seed`` (`int`): seed for random number generator when fuzzing
205 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
206 statistics
207 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
208 ``STDEV_*`` scaling
209 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
210 ``STDEV_POSITIVE``/``NEGATIVE``)
211 * ``bscale`` (`float`): manually specified ``BSCALE``
212 (for ``MANUAL`` scaling)
213 * ``bzero`` (`float`): manually specified ``BSCALE``
214 (for ``MANUAL`` scaling)
216 A very simple example YAML recipe (for the ``Exposure`` specialization):
218 .. code-block:: yaml
220 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
221 default:
222 image: &default
223 compression:
224 algorithm: GZIP_SHUFFLE
225 mask: *default
226 variance: *default
228 """
230 supportedWriteParameters = frozenset({"recipe"})
231 ReaderClass: type # must be set by concrete subclasses
233 @property # type: ignore
234 @cached_getter
235 def reader(self):
236 """The reader object that backs this formatter's read operations.
238 This is computed on first use and then cached. It should never be
239 accessed when writing.
240 """
241 return self.ReaderClass(self.fileDescriptor.location.path)
243 def readComponent(self, component):
244 # Docstring inherited.
245 if component in ("bbox", "dimensions", "xy0"):
246 bbox = self.reader.readBBox()
247 if component == "dimensions":
248 return bbox.getDimensions()
249 elif component == "xy0":
250 return bbox.getMin()
251 else:
252 return bbox
253 else:
254 raise KeyError(f"Unknown component requested: {component}")
256 def readFull(self):
257 # Docstring inherited.
258 return self.reader.read(**self.checked_parameters)
260 def write(self, inMemoryDataset):
261 """Write a Python object to a file.
263 Parameters
264 ----------
265 inMemoryDataset : `object`
266 The Python object to store.
267 """
268 # Update the location with the formatter-preferred file extension
269 self.fileDescriptor.location.updateExtension(self.extension)
270 outputPath = self.fileDescriptor.location.path
272 # check to see if we have a recipe requested
273 recipeName = self.writeParameters.get("recipe")
274 recipe = self.getImageCompressionSettings(recipeName)
275 if recipe:
276 # Can not construct a PropertySet from a hierarchical
277 # dict but can update one.
278 ps = PropertySet()
279 ps.update(recipe)
280 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
281 else:
282 inMemoryDataset.writeFits(outputPath)
284 def getImageCompressionSettings(self, recipeName):
285 """Retrieve the relevant compression settings for this recipe.
287 Parameters
288 ----------
289 recipeName : `str`
290 Label associated with the collection of compression parameters
291 to select.
293 Returns
294 -------
295 settings : `dict`
296 The selected settings.
297 """
298 # if no recipe has been provided and there is no default
299 # return immediately
300 if not recipeName:
301 if "default" not in self.writeRecipes:
302 return {}
303 recipeName = "default"
305 if recipeName not in self.writeRecipes:
306 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
308 recipe = self.writeRecipes[recipeName]
310 # Set the seed based on dataId
311 seed = hash(tuple(self.dataId.items())) % 2**31
312 for plane in ("image", "mask", "variance"):
313 if plane in recipe and "scaling" in recipe[plane]:
314 scaling = recipe[plane]["scaling"]
315 if "seed" in scaling and scaling["seed"] == 0:
316 scaling["seed"] = seed
318 return recipe
320 @classmethod
321 def validateWriteRecipes(cls, recipes):
322 """Validate supplied recipes for this formatter.
324 The recipes are supplemented with default values where appropriate.
326 TODO: replace this custom validation code with Cerberus (DM-11846)
328 Parameters
329 ----------
330 recipes : `dict`
331 Recipes to validate. Can be empty dict or `None`.
333 Returns
334 -------
335 validated : `dict`
336 Validated recipes. Returns what was given if there are no
337 recipes listed.
339 Raises
340 ------
341 RuntimeError
342 Raised if validation fails.
343 """
344 # Schemas define what should be there, and the default values (and by
345 # the default value, the expected type).
346 compressionSchema = {
347 "algorithm": "NONE",
348 "rows": 1,
349 "columns": 0,
350 "quantizeLevel": 0.0,
351 }
352 scalingSchema = {
353 "algorithm": "NONE",
354 "bitpix": 0,
355 "maskPlanes": ["NO_DATA"],
356 "seed": 0,
357 "quantizeLevel": 4.0,
358 "quantizePad": 5.0,
359 "fuzz": True,
360 "bscale": 1.0,
361 "bzero": 0.0,
362 }
364 if not recipes:
365 # We can not insist on recipes being specified
366 return recipes
368 def checkUnrecognized(entry, allowed, description):
369 """Check to see if the entry contains unrecognised keywords"""
370 unrecognized = set(entry) - set(allowed)
371 if unrecognized:
372 raise RuntimeError(
373 f"Unrecognized entries when parsing image compression recipe {description}: "
374 f"{unrecognized}"
375 )
377 validated = {}
378 for name in recipes:
379 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
380 validated[name] = {}
381 for plane in ("image", "mask", "variance"):
382 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], f"{name}->{plane}")
384 np = {}
385 validated[name][plane] = np
386 for settings, schema in (("compression", compressionSchema), ("scaling", scalingSchema)):
387 np[settings] = {}
388 if settings not in recipes[name][plane]:
389 for key in schema:
390 np[settings][key] = schema[key]
391 continue
392 entry = recipes[name][plane][settings]
393 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
394 for key in schema:
395 value = type(schema[key])(entry[key]) if key in entry else schema[key]
396 np[settings][key] = value
397 return validated
400class FitsImageFormatter(StandardFitsImageFormatterBase):
401 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
402 from/to FITS.
403 """
405 ReaderClass = ImageFitsReader
408class FitsMaskFormatter(StandardFitsImageFormatterBase):
409 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
410 from/to FITS.
411 """
413 ReaderClass = MaskFitsReader
416class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
417 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
418 from/to FITS.
419 """
421 ReaderClass = MaskedImageFitsReader
423 def readComponent(self, component):
424 # Docstring inherited.
425 if component == "image":
426 return self.reader.readImage(**self.checked_parameters)
427 elif component == "mask":
428 return self.reader.readMask(**self.checked_parameters)
429 elif component == "variance":
430 return self.reader.readVariance(**self.checked_parameters)
431 else:
432 # Delegate to base for bbox, dimensions, xy0.
433 return super().readComponent(component)
436def standardizeAmplifierParameters(parameters, on_disk_detector):
437 """Preprocess the Exposure storage class's "amp" and "detector" parameters
439 This checks the given objects for consistency with the on-disk geometry and
440 converts amplifier IDs/names to Amplifier instances.
442 Parameters
443 ----------
444 parameters : `dict`
445 Dictionary of parameters passed to formatter. See the Exposure storage
446 class definition in daf_butler for allowed keys and values.
447 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
448 Detector that represents the on-disk image being loaded, or `None` if
449 this is unknown (and hence the user must provide one in
450 ``parameters`` if "amp" is in ``parameters``).
452 Returns
453 -------
454 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
455 An amplifier object that defines a subimage to load, or `None` if there
456 was no "amp" parameter.
457 detector : `lsst.afw.cameraGeom.Detector` or `None`
458 A detector object whose amplifiers are in the same s/orientation
459 state as the on-disk image. If there is no "amp" parameter,
460 ``on_disk_detector`` is simply passed through.
461 regions_differ : `bool`
462 `True` if the on-disk detector and the detector given in the parameters
463 had different bounding boxes for one or more regions. This can happen
464 if the true overscan region sizes can only be determined when the image
465 is actually read, but otherwise it should be considered user error.
466 """
467 if (amplifier := parameters.get("amp")) is None:
468 return None, on_disk_detector, False
469 if "bbox" in parameters or "origin" in parameters:
470 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
471 if isinstance(amplifier, (int, str)):
472 amp_key = amplifier
473 target_amplifier = None
474 else:
475 amp_key = amplifier.getName()
476 target_amplifier = amplifier
477 if (detector := parameters.get("detector")) is not None:
478 if on_disk_detector is not None:
479 # User passed a detector and we also found one on disk. Check them
480 # for consistency. Note that we are checking the amps we'd get
481 # from the two detectors against each other, not the amplifier we
482 # got directly from the user, as the latter is allowed to differ in
483 # assembly/orientation state.
484 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
485 if comparison & comparison.ASSEMBLY_DIFFERS:
486 raise ValueError(
487 "The given 'detector' has a different assembly state and/or orientation from "
488 f"the on-disk one for amp {amp_key}."
489 )
490 else:
491 if on_disk_detector is None:
492 raise ValueError(
493 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
494 "Please provide either a 'detector' parameter or an Amplifier instance in the "
495 "'amp' parameter."
496 )
497 comparison = AmplifierGeometryComparison.EQUAL
498 detector = on_disk_detector
499 if target_amplifier is None:
500 target_amplifier = detector[amp_key]
501 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
504class FitsExposureFormatter(FitsMaskedImageFormatter):
505 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
506 from/to FITS.
508 Notes
509 -----
510 This class inherits from `FitsMaskedImageFormatter` even though
511 `lsst.afw.image.Exposure` doesn't inherit from
512 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
513 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
514 should be replaced with e.g. both calling a free function if that slight
515 type covariance violation ever becomes a practical problem.
516 """
518 ReaderClass = ExposureFitsReader
520 def readComponent(self, component):
521 # Docstring inherited.
522 # Generic components can be read via a string name; DM-27754 will make
523 # this mapping larger at the expense of the following one.
524 genericComponents = {
525 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
526 }
527 if (genericComponentName := genericComponents.get(component)) is not None:
528 return self.reader.readComponent(genericComponentName)
529 # Other components have hard-coded method names, but don't take
530 # parameters.
531 standardComponents = {
532 "id": "readExposureId",
533 "metadata": "readMetadata",
534 "wcs": "readWcs",
535 "coaddInputs": "readCoaddInputs",
536 "psf": "readPsf",
537 "photoCalib": "readPhotoCalib",
538 "filter": "readFilter",
539 "validPolygon": "readValidPolygon",
540 "apCorrMap": "readApCorrMap",
541 "visitInfo": "readVisitInfo",
542 "transmissionCurve": "readTransmissionCurve",
543 "detector": "readDetector",
544 "exposureInfo": "readExposureInfo",
545 }
546 if component == "filterLabel":
547 warnings.warn(
548 "Exposure.filterLabel component is deprecated; use .filter instead. "
549 "Will be removed after v24.",
550 FutureWarning,
551 stacklevel=2, # Report from caller.
552 )
553 component = "filter"
554 if (methodName := standardComponents.get(component)) is not None:
555 result = getattr(self.reader, methodName)()
556 if component == "filter":
557 return self._fixFilterLabels(result)
558 return result
559 # Delegate to MaskedImage and ImageBase implementations for the rest.
560 return super().readComponent(component)
562 def readFull(self):
563 # Docstring inherited.
564 amplifier, detector, _ = standardizeAmplifierParameters(
565 self.checked_parameters,
566 self.reader.readDetector(),
567 )
568 if amplifier is not None:
569 amplifier_isolator = AmplifierIsolator(
570 amplifier,
571 self.reader.readBBox(),
572 detector,
573 )
574 result = amplifier_isolator.transform_subimage(
575 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
576 )
577 result.setDetector(amplifier_isolator.make_detector())
578 else:
579 result = self.reader.read(**self.checked_parameters)
580 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
581 return result
583 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
584 """Compare the filter label read from the file with the one in the
585 data ID.
587 Parameters
588 ----------
589 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
590 Filter label read from the file, if there was one.
591 should_be_standardized : `bool`, optional
592 If `True`, expect ``file_filter_label`` to be consistent with the
593 data ID and warn only if it is not. If `False`, expect it to be
594 inconsistent and warn only if the data ID is incomplete and hence
595 the `FilterLabel` cannot be fixed. If `None` (default) guess
596 whether the file should be standardized by looking at the
597 serialization version number in file, which requires this method to
598 have been run after `readFull` or `readComponent`.
600 Returns
601 -------
602 filter_label : `lsst.afw.image.FilterLabel` or `None`
603 The preferred filter label; may be the given one or one built from
604 the data ID. `None` is returned if there should never be any
605 filters associated with this dataset type.
607 Notes
608 -----
609 Most test coverage for this method is in ci_hsc_gen3, where we have
610 much easier access to test data that exhibits the problems it attempts
611 to solve.
612 """
613 # Remember filter data ID keys that weren't in this particular data ID,
614 # so we can warn about them later.
615 missing = []
616 band = None
617 physical_filter = None
618 if "band" in self.dataId.graph.dimensions.names:
619 band = self.dataId.get("band")
620 # band isn't in the data ID; is that just because this data ID
621 # hasn't been filled in with everything the Registry knows, or
622 # because this dataset is never associated with a band?
623 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
624 missing.append("band")
625 if "physical_filter" in self.dataId.graph.dimensions.names:
626 physical_filter = self.dataId.get("physical_filter")
627 # Same check as above for band, but for physical_filter.
628 if (
629 physical_filter is None
630 and not self.dataId.hasFull()
631 and "physical_filter" in self.dataId.graph.implied.names
632 ):
633 missing.append("physical_filter")
634 if should_be_standardized is None:
635 version = self.reader.readSerializationVersion()
636 should_be_standardized = version >= 2
637 if missing:
638 # Data ID identifies a filter but the actual filter label values
639 # haven't been fetched from the database; we have no choice but
640 # to use the one in the file.
641 # Warn if that's more likely than not to be bad, because the file
642 # predates filter standardization.
643 if not should_be_standardized:
644 warnings.warn(
645 f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
646 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
647 "Call Registry.expandDataId before Butler.get to avoid this."
648 )
649 return file_filter_label
650 if band is None and physical_filter is None:
651 data_id_filter_label = None
652 else:
653 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
654 if data_id_filter_label != file_filter_label and should_be_standardized:
655 # File was written after FilterLabel and standardization, but its
656 # FilterLabel doesn't agree with the data ID: this indicates a bug
657 # in whatever code produced the Exposure (though it may be one that
658 # has been fixed since the file was written).
659 warnings.warn(
660 f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
661 f"filter label mismatch (file is {file_filter_label}, data ID is "
662 f"{data_id_filter_label}). This is probably a bug in the code that produced it."
663 )
664 return data_id_filter_label