Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 21%
192 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-05 02:41 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-05 02:41 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import warnings
31from abc import abstractmethod
32from typing import AbstractSet, ClassVar
34from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
35from lsst.afw.image import (
36 ExposureFitsReader,
37 ExposureInfo,
38 FilterLabel,
39 ImageFitsReader,
40 MaskedImageFitsReader,
41 MaskFitsReader,
42)
44# Needed for ApCorrMap to resolve properly
45from lsst.afw.math import BoundedField # noqa: F401
46from lsst.daf.base import PropertySet
47from lsst.daf.butler import Formatter
48from lsst.utils.classes import cached_getter
51class FitsImageFormatterBase(Formatter):
52 """Base class formatter for image-like storage classes stored via FITS.
54 Notes
55 -----
56 This class makes no assumptions about how many HDUs are used to represent
57 the image on disk, and includes no support for writing. It's really just a
58 collection of miscellaneous boilerplate common to all FITS image
59 formatters.
61 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
62 (even if just to disable them by raising an exception).
63 """
65 extension = ".fits"
66 supportedExtensions: ClassVar[AbstractSet[str]] = frozenset(
67 {".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"}
68 )
70 unsupportedParameters: ClassVar[AbstractSet[str]] = frozenset()
71 """Support all parameters."""
73 @property # type: ignore
74 @cached_getter
75 def checked_parameters(self):
76 """The parameters passed by the butler user, after checking them
77 against the storage class and transforming `None` into an empty `dict`
78 (`dict`).
80 This is computed on first use and then cached. It should never be
81 accessed when writing. Subclasses that need additional checking should
82 delegate to `super` and then check the result before returning it.
83 """
84 parameters = self.fileDescriptor.parameters
85 if parameters is None:
86 parameters = {}
87 self.fileDescriptor.storageClass.validateParameters(parameters)
88 return parameters
90 def read(self, component=None):
91 # Docstring inherited.
92 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
93 if component is not None:
94 return self.readComponent(component)
95 else:
96 raise ValueError(
97 "Storage class inconsistency ({} vs {}) but no"
98 " component requested".format(
99 self.fileDescriptor.readStorageClass.name, self.fileDescriptor.storageClass.name
100 )
101 )
102 return self.readFull()
104 @abstractmethod
105 def readComponent(self, component):
106 """Read a component dataset.
108 Parameters
109 ----------
110 component : `str`, optional
111 Component to read from the file.
113 Returns
114 -------
115 obj : component-dependent
116 In-memory component object.
118 Raises
119 ------
120 KeyError
121 Raised if the requested component cannot be handled.
122 """
123 raise NotImplementedError()
125 @abstractmethod
126 def readFull(self):
127 """Read the full dataset (while still accounting for parameters).
129 Returns
130 -------
131 obj : component-dependent
132 In-memory component object.
134 """
135 raise NotImplementedError()
138class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
139 """Base class formatter for image-like storage classes stored via FITS
140 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
142 Notes
143 -----
144 This class includes no support for writing.
146 Concrete subclasses must provide at least the `ReaderClass` attribute
147 and a `write` implementation (even just to disable writing by raising).
149 The provided implementation of `readComponent` handles only the 'bbox',
150 'dimensions', and 'xy0' components common to all image-like storage
151 classes. Subclasses with additional components should handle them first,
152 then delegate to ``super()`` for these (or, if necessary, delegate first
153 and catch `KeyError`).
155 The provided implementation of `readFull` handles only parameters that
156 can be forwarded directly to the reader class (usually ``bbox`` and
157 ``origin``). Concrete subclasses that need to handle additional parameters
158 should generally reimplement without delegating (the implementation is
159 trivial).
160 """
163class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
164 """Base class interface for image-like storage stored via FITS,
165 written using LSST code.
167 Notes
168 -----
169 Concrete subclasses must provide at least the `ReaderClass` attribute.
171 The provided implementation of `readComponent` handles only the 'bbox',
172 'dimensions', and 'xy0' components common to all image-like storage
173 classes. Subclasses with additional components should handle them first,
174 then delegate to ``super()`` for these (or, if necessary, delegate first
175 and catch `KeyError`).
177 The provided implementation of `readFull` handles only parameters that
178 can be forwarded directly to the reader class (usually ``bbox`` and
179 ``origin``). Concrete subclasses that need to handle additional parameters
180 should generally reimplement without delegating (the implementation is
181 trivial).
183 This Formatter supports write recipes, and assumes its in-memory type has
184 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
186 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
187 define ``image``, ``mask`` and ``variance`` entries, each of which may
188 contain ``compression`` and ``scaling`` entries. Defaults will be
189 provided for any missing elements under ``compression`` and
190 ``scaling``.
192 The allowed entries under ``compression`` are:
194 * ``algorithm`` (`str`): compression algorithm to use
195 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
196 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
197 * ``quantizeLevel`` (`float`): cfitsio quantization level
199 The allowed entries under ``scaling`` are:
201 * ``algorithm`` (`str`): scaling algorithm to use
202 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
203 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
204 * ``seed`` (`int`): seed for random number generator when fuzzing
205 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
206 statistics
207 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
208 ``STDEV_*`` scaling
209 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
210 ``STDEV_POSITIVE``/``NEGATIVE``)
211 * ``bscale`` (`float`): manually specified ``BSCALE``
212 (for ``MANUAL`` scaling)
213 * ``bzero`` (`float`): manually specified ``BSCALE``
214 (for ``MANUAL`` scaling)
216 A very simple example YAML recipe (for the ``Exposure`` specialization):
218 .. code-block:: yaml
220 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
221 default:
222 image: &default
223 compression:
224 algorithm: GZIP_SHUFFLE
225 mask: *default
226 variance: *default
228 """
230 supportedWriteParameters = frozenset({"recipe"})
231 ReaderClass: type # must be set by concrete subclasses
233 @property # type: ignore
234 @cached_getter
235 def reader(self):
236 """The reader object that backs this formatter's read operations.
238 This is computed on first use and then cached. It should never be
239 accessed when writing.
240 """
241 return self.ReaderClass(self.fileDescriptor.location.path)
243 def readComponent(self, component):
244 # Docstring inherited.
245 if component in ("bbox", "dimensions", "xy0"):
246 bbox = self.reader.readBBox()
247 if component == "dimensions":
248 return bbox.getDimensions()
249 elif component == "xy0":
250 return bbox.getMin()
251 else:
252 return bbox
253 else:
254 raise KeyError(f"Unknown component requested: {component}")
256 def readFull(self):
257 # Docstring inherited.
258 return self.reader.read(**self.checked_parameters)
260 def write(self, inMemoryDataset):
261 """Write a Python object to a file.
263 Parameters
264 ----------
265 inMemoryDataset : `object`
266 The Python object to store.
267 """
268 # Update the location with the formatter-preferred file extension
269 self.fileDescriptor.location.updateExtension(self.extension)
270 outputPath = self.fileDescriptor.location.path
272 # check to see if we have a recipe requested
273 recipeName = self.writeParameters.get("recipe")
274 recipe = self.getImageCompressionSettings(recipeName)
275 if recipe:
276 # Can not construct a PropertySet from a hierarchical
277 # dict but can update one.
278 ps = PropertySet()
279 ps.update(recipe)
280 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
281 else:
282 inMemoryDataset.writeFits(outputPath)
284 def getImageCompressionSettings(self, recipeName):
285 """Retrieve the relevant compression settings for this recipe.
287 Parameters
288 ----------
289 recipeName : `str`
290 Label associated with the collection of compression parameters
291 to select.
293 Returns
294 -------
295 settings : `dict`
296 The selected settings.
297 """
298 # if no recipe has been provided and there is no default
299 # return immediately
300 if not recipeName:
301 if "default" not in self.writeRecipes:
302 return {}
303 recipeName = "default"
305 if recipeName not in self.writeRecipes:
306 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
308 recipe = self.writeRecipes[recipeName]
310 # Set the seed based on dataId
311 seed = hash(tuple(self.dataId.items())) % 2**31
312 for plane in ("image", "mask", "variance"):
313 if plane in recipe and "scaling" in recipe[plane]:
314 scaling = recipe[plane]["scaling"]
315 if "seed" in scaling and scaling["seed"] == 0:
316 scaling["seed"] = seed
318 return recipe
320 @classmethod
321 def validateWriteRecipes(cls, recipes):
322 """Validate supplied recipes for this formatter.
324 The recipes are supplemented with default values where appropriate.
326 TODO: replace this custom validation code with Cerberus (DM-11846)
328 Parameters
329 ----------
330 recipes : `dict`
331 Recipes to validate. Can be empty dict or `None`.
333 Returns
334 -------
335 validated : `dict`
336 Validated recipes. Returns what was given if there are no
337 recipes listed.
339 Raises
340 ------
341 RuntimeError
342 Raised if validation fails.
343 """
344 # Schemas define what should be there, and the default values (and by
345 # the default value, the expected type).
346 compressionSchema = {
347 "algorithm": "NONE",
348 "rows": 1,
349 "columns": 0,
350 "quantizeLevel": 0.0,
351 }
352 scalingSchema = {
353 "algorithm": "NONE",
354 "bitpix": 0,
355 "maskPlanes": ["NO_DATA"],
356 "seed": 0,
357 "quantizeLevel": 4.0,
358 "quantizePad": 5.0,
359 "fuzz": True,
360 "bscale": 1.0,
361 "bzero": 0.0,
362 }
364 if not recipes:
365 # We can not insist on recipes being specified
366 return recipes
368 def checkUnrecognized(entry, allowed, description):
369 """Check to see if the entry contains unrecognised keywords"""
370 unrecognized = set(entry) - set(allowed)
371 if unrecognized:
372 raise RuntimeError(
373 f"Unrecognized entries when parsing image compression recipe {description}: "
374 f"{unrecognized}"
375 )
377 validated = {}
378 for name in recipes:
379 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
380 validated[name] = {}
381 for plane in ("image", "mask", "variance"):
382 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], f"{name}->{plane}")
384 np = {}
385 validated[name][plane] = np
386 for settings, schema in (("compression", compressionSchema), ("scaling", scalingSchema)):
387 np[settings] = {}
388 if settings not in recipes[name][plane]:
389 for key in schema:
390 np[settings][key] = schema[key]
391 continue
392 entry = recipes[name][plane][settings]
393 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
394 for key in schema:
395 value = type(schema[key])(entry[key]) if key in entry else schema[key]
396 np[settings][key] = value
397 return validated
400class FitsImageFormatter(StandardFitsImageFormatterBase):
401 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
402 from/to FITS.
403 """
405 ReaderClass = ImageFitsReader
408class FitsMaskFormatter(StandardFitsImageFormatterBase):
409 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
410 from/to FITS.
411 """
413 ReaderClass = MaskFitsReader
416class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
417 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
418 from/to FITS.
419 """
421 ReaderClass = MaskedImageFitsReader
423 def readComponent(self, component):
424 # Docstring inherited.
425 if component == "image":
426 return self.reader.readImage(**self.checked_parameters)
427 elif component == "mask":
428 return self.reader.readMask(**self.checked_parameters)
429 elif component == "variance":
430 return self.reader.readVariance(**self.checked_parameters)
431 else:
432 # Delegate to base for bbox, dimensions, xy0.
433 return super().readComponent(component)
436def standardizeAmplifierParameters(parameters, on_disk_detector):
437 """Preprocess the Exposure storage class's "amp" and "detector" parameters
439 This checks the given objects for consistency with the on-disk geometry and
440 converts amplifier IDs/names to Amplifier instances.
442 Parameters
443 ----------
444 parameters : `dict`
445 Dictionary of parameters passed to formatter. See the Exposure storage
446 class definition in daf_butler for allowed keys and values.
447 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
448 Detector that represents the on-disk image being loaded, or `None` if
449 this is unknown (and hence the user must provide one in
450 ``parameters`` if "amp" is in ``parameters``).
452 Returns
453 -------
454 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
455 An amplifier object that defines a subimage to load, or `None` if there
456 was no "amp" parameter.
457 detector : `lsst.afw.cameraGeom.Detector` or `None`
458 A detector object whose amplifiers are in the same s/orientation
459 state as the on-disk image. If there is no "amp" parameter,
460 ``on_disk_detector`` is simply passed through.
461 regions_differ : `bool`
462 `True` if the on-disk detector and the detector given in the parameters
463 had different bounding boxes for one or more regions. This can happen
464 if the true overscan region sizes can only be determined when the image
465 is actually read, but otherwise it should be considered user error.
466 """
467 if (amplifier := parameters.get("amp")) is None:
468 return None, on_disk_detector, False
469 if "bbox" in parameters or "origin" in parameters:
470 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
471 if isinstance(amplifier, (int, str)):
472 amp_key = amplifier
473 target_amplifier = None
474 else:
475 amp_key = amplifier.getName()
476 target_amplifier = amplifier
477 if (detector := parameters.get("detector")) is not None:
478 if on_disk_detector is not None:
479 # User passed a detector and we also found one on disk. Check them
480 # for consistency. Note that we are checking the amps we'd get
481 # from the two detectors against each other, not the amplifier we
482 # got directly from the user, as the latter is allowed to differ in
483 # assembly/orientation state.
484 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
485 if comparison & comparison.ASSEMBLY_DIFFERS:
486 raise ValueError(
487 "The given 'detector' has a different assembly state and/or orientation from "
488 f"the on-disk one for amp {amp_key}."
489 )
490 else:
491 if on_disk_detector is None:
492 raise ValueError(
493 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
494 "Please provide either a 'detector' parameter or an Amplifier instance in the "
495 "'amp' parameter."
496 )
497 comparison = AmplifierGeometryComparison.EQUAL
498 detector = on_disk_detector
499 if target_amplifier is None:
500 target_amplifier = detector[amp_key]
501 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
504class FitsExposureFormatter(FitsMaskedImageFormatter):
505 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
506 from/to FITS.
508 Notes
509 -----
510 This class inherits from `FitsMaskedImageFormatter` even though
511 `lsst.afw.image.Exposure` doesn't inherit from
512 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
513 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
514 should be replaced with e.g. both calling a free function if that slight
515 type covariance violation ever becomes a practical problem.
516 """
518 ReaderClass = ExposureFitsReader
520 def readComponent(self, component):
521 # Docstring inherited.
522 # Generic components can be read via a string name; DM-27754 will make
523 # this mapping larger at the expense of the following one.
524 genericComponents = {
525 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
526 }
527 if (genericComponentName := genericComponents.get(component)) is not None:
528 return self.reader.readComponent(genericComponentName)
529 # Other components have hard-coded method names, but don't take
530 # parameters.
531 standardComponents = {
532 "id": "readExposureId",
533 "metadata": "readMetadata",
534 "wcs": "readWcs",
535 "coaddInputs": "readCoaddInputs",
536 "psf": "readPsf",
537 "photoCalib": "readPhotoCalib",
538 # TODO: deprecate in DM-27170, remove in DM-27177
539 "filter": "readFilter",
540 # TODO: deprecate in DM-27177, remove in DM-27811
541 "filterLabel": "readFilterLabel",
542 "validPolygon": "readValidPolygon",
543 "apCorrMap": "readApCorrMap",
544 "visitInfo": "readVisitInfo",
545 "transmissionCurve": "readTransmissionCurve",
546 "detector": "readDetector",
547 "exposureInfo": "readExposureInfo",
548 }
549 if (methodName := standardComponents.get(component)) is not None:
550 result = getattr(self.reader, methodName)()
551 if component == "filterLabel":
552 return self._fixFilterLabels(result)
553 return result
554 # Delegate to MaskedImage and ImageBase implementations for the rest.
555 return super().readComponent(component)
557 def readFull(self):
558 # Docstring inherited.
559 amplifier, detector, _ = standardizeAmplifierParameters(
560 self.checked_parameters,
561 self.reader.readDetector(),
562 )
563 if amplifier is not None:
564 amplifier_isolator = AmplifierIsolator(
565 amplifier,
566 self.reader.readBBox(),
567 detector,
568 )
569 result = amplifier_isolator.transform_subimage(
570 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
571 )
572 result.setDetector(amplifier_isolator.make_detector())
573 else:
574 result = self.reader.read(**self.checked_parameters)
575 result.getInfo().setFilterLabel(self._fixFilterLabels(result.getInfo().getFilterLabel()))
576 return result
578 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
579 """Compare the filter label read from the file with the one in the
580 data ID.
582 Parameters
583 ----------
584 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
585 Filter label read from the file, if there was one.
586 should_be_standardized : `bool`, optional
587 If `True`, expect ``file_filter_label`` to be consistent with the
588 data ID and warn only if it is not. If `False`, expect it to be
589 inconsistent and warn only if the data ID is incomplete and hence
590 the `FilterLabel` cannot be fixed. If `None` (default) guess
591 whether the file should be standardized by looking at the
592 serialization version number in file, which requires this method to
593 have been run after `readFull` or `readComponent`.
595 Returns
596 -------
597 filter_label : `lsst.afw.image.FilterLabel` or `None`
598 The preferred filter label; may be the given one or one built from
599 the data ID. `None` is returned if there should never be any
600 filters associated with this dataset type.
602 Notes
603 -----
604 Most test coverage for this method is in ci_hsc_gen3, where we have
605 much easier access to test data that exhibits the problems it attempts
606 to solve.
607 """
608 # Remember filter data ID keys that weren't in this particular data ID,
609 # so we can warn about them later.
610 missing = []
611 band = None
612 physical_filter = None
613 if "band" in self.dataId.graph.dimensions.names:
614 band = self.dataId.get("band")
615 # band isn't in the data ID; is that just because this data ID
616 # hasn't been filled in with everything the Registry knows, or
617 # because this dataset is never associated with a band?
618 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
619 missing.append("band")
620 if "physical_filter" in self.dataId.graph.dimensions.names:
621 physical_filter = self.dataId.get("physical_filter")
622 # Same check as above for band, but for physical_filter.
623 if (
624 physical_filter is None
625 and not self.dataId.hasFull()
626 and "physical_filter" in self.dataId.graph.implied.names
627 ):
628 missing.append("physical_filter")
629 if should_be_standardized is None:
630 version = self.reader.readSerializationVersion()
631 should_be_standardized = version >= 2
632 if missing:
633 # Data ID identifies a filter but the actual filter label values
634 # haven't been fetched from the database; we have no choice but
635 # to use the one in the file.
636 # Warn if that's more likely than not to be bad, because the file
637 # predates filter standardization.
638 if not should_be_standardized:
639 warnings.warn(
640 f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
641 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
642 "Call Registry.expandDataId before Butler.get to avoid this."
643 )
644 return file_filter_label
645 if band is None and physical_filter is None:
646 data_id_filter_label = None
647 else:
648 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
649 if data_id_filter_label != file_filter_label and should_be_standardized:
650 # File was written after FilterLabel and standardization, but its
651 # FilterLabel doesn't agree with the data ID: this indicates a bug
652 # in whatever code produced the Exposure (though it may be one that
653 # has been fixed since the file was written).
654 warnings.warn(
655 f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
656 f"filter label mismatch (file is {file_filter_label}, data ID is "
657 f"{data_id_filter_label}). This is probably a bug in the code that produced it."
658 )
659 return data_id_filter_label