Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 21%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30from abc import abstractmethod
31import warnings
33from lsst.daf.base import PropertySet
34from lsst.daf.butler import Formatter
35from lsst.daf.butler.core.utils import cached_getter
36from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
37from lsst.afw.image import ExposureFitsReader, ImageFitsReader, MaskFitsReader, MaskedImageFitsReader
38from lsst.afw.image import ExposureInfo, FilterLabel
39# Needed for ApCorrMap to resolve properly
40from lsst.afw.math import BoundedField # noqa: F401
43class FitsImageFormatterBase(Formatter):
44 """Base class formatter for image-like storage classes stored via FITS.
46 Notes
47 -----
48 This class makes no assumptions about how many HDUs are used to represent
49 the image on disk, and includes no support for writing. It's really just a
50 collection of miscellaneous boilerplate common to all FITS image
51 formatters.
53 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
54 (even if just to disable them by raising an exception).
55 """
57 extension = ".fits"
58 supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
60 unsupportedParameters = {}
61 """Support all parameters."""
63 @property
64 @cached_getter
65 def checked_parameters(self):
66 """The parameters passed by the butler user, after checking them
67 against the storage class and transforming `None` into an empty `dict`
68 (`dict`).
70 This is computed on first use and then cached. It should never be
71 accessed when writing. Subclasses that need additional checking should
72 delegate to `super` and then check the result before returning it.
73 """
74 parameters = self.fileDescriptor.parameters
75 if parameters is None:
76 parameters = {}
77 self.fileDescriptor.storageClass.validateParameters(parameters)
78 return parameters
80 def read(self, component=None):
81 # Docstring inherited.
82 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
83 if component is not None:
84 return self.readComponent(component)
85 else:
86 raise ValueError("Storage class inconsistency ({} vs {}) but no"
87 " component requested".format(self.fileDescriptor.readStorageClass.name,
88 self.fileDescriptor.storageClass.name))
89 return self.readFull()
91 @abstractmethod
92 def readComponent(self, component):
93 """Read a component dataset.
95 Parameters
96 ----------
97 component : `str`, optional
98 Component to read from the file.
100 Returns
101 -------
102 obj : component-dependent
103 In-memory component object.
105 Raises
106 ------
107 KeyError
108 Raised if the requested component cannot be handled.
109 """
110 raise NotImplementedError()
112 @abstractmethod
113 def readFull(self):
114 """Read the full dataset (while still accounting for parameters).
116 Returns
117 -------
118 obj : component-dependent
119 In-memory component object.
121 """
122 raise NotImplementedError()
125class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
126 """Base class formatter for image-like storage classes stored via FITS
127 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
129 Notes
130 -----
131 This class includes no support for writing.
133 Concrete subclasses must provide at least the `ReaderClass` attribute
134 and a `write` implementation (even just to disable writing by raising).
136 The provided implementation of `readComponent` handles only the 'bbox',
137 'dimensions', and 'xy0' components common to all image-like storage
138 classes. Subclasses with additional components should handle them first,
139 then delegate to ``super()`` for these (or, if necessary, delegate first
140 and catch `KeyError`).
142 The provided implementation of `readFull` handles only parameters that
143 can be forwarded directly to the reader class (usually ``bbox`` and
144 ``origin``). Concrete subclasses that need to handle additional parameters
145 should generally reimplement without delegating (the implementation is
146 trivial).
147 """
150class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
151 """Base class interface for image-like storage stored via FITS,
152 written using LSST code.
154 Notes
155 -----
156 Concrete subclasses must provide at least the `ReaderClass` attribute.
158 The provided implementation of `readComponent` handles only the 'bbox',
159 'dimensions', and 'xy0' components common to all image-like storage
160 classes. Subclasses with additional components should handle them first,
161 then delegate to ``super()`` for these (or, if necessary, delegate first
162 and catch `KeyError`).
164 The provided implementation of `readFull` handles only parameters that
165 can be forwarded directly to the reader class (usually ``bbox`` and
166 ``origin``). Concrete subclasses that need to handle additional parameters
167 should generally reimplement without delegating (the implementation is
168 trivial).
170 This Formatter supports write recipes, and assumes its in-memory type has
171 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
173 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
174 define ``image``, ``mask`` and ``variance`` entries, each of which may
175 contain ``compression`` and ``scaling`` entries. Defaults will be
176 provided for any missing elements under ``compression`` and
177 ``scaling``.
179 The allowed entries under ``compression`` are:
181 * ``algorithm`` (`str`): compression algorithm to use
182 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
183 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
184 * ``quantizeLevel`` (`float`): cfitsio quantization level
186 The allowed entries under ``scaling`` are:
188 * ``algorithm`` (`str`): scaling algorithm to use
189 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
190 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
191 * ``seed`` (`int`): seed for random number generator when fuzzing
192 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
193 statistics
194 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
195 ``STDEV_*`` scaling
196 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
197 ``STDEV_POSITIVE``/``NEGATIVE``)
198 * ``bscale`` (`float`): manually specified ``BSCALE``
199 (for ``MANUAL`` scaling)
200 * ``bzero`` (`float`): manually specified ``BSCALE``
201 (for ``MANUAL`` scaling)
203 A very simple example YAML recipe (for the ``Exposure`` specialization):
205 .. code-block:: yaml
207 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
208 default:
209 image: &default
210 compression:
211 algorithm: GZIP_SHUFFLE
212 mask: *default
213 variance: *default
215 """
216 supportedWriteParameters = frozenset({"recipe"})
217 ReaderClass: type # must be set by concrete subclasses
219 @property
220 @cached_getter
221 def reader(self):
222 """The reader object that backs this formatter's read operations.
224 This is computed on first use and then cached. It should never be
225 accessed when writing.
226 """
227 return self.ReaderClass(self.fileDescriptor.location.path)
229 def readComponent(self, component):
230 # Docstring inherited.
231 if component in ("bbox", "dimensions", "xy0"):
232 bbox = self.reader.readBBox()
233 if component == "dimensions":
234 return bbox.getDimensions()
235 elif component == "xy0":
236 return bbox.getMin()
237 else:
238 return bbox
239 else:
240 raise KeyError(f"Unknown component requested: {component}")
242 def readFull(self):
243 # Docstring inherited.
244 return self.reader.read(**self.checked_parameters)
246 def write(self, inMemoryDataset):
247 """Write a Python object to a file.
249 Parameters
250 ----------
251 inMemoryDataset : `object`
252 The Python object to store.
253 """
254 # Update the location with the formatter-preferred file extension
255 self.fileDescriptor.location.updateExtension(self.extension)
256 outputPath = self.fileDescriptor.location.path
258 # check to see if we have a recipe requested
259 recipeName = self.writeParameters.get("recipe")
260 recipe = self.getImageCompressionSettings(recipeName)
261 if recipe:
262 # Can not construct a PropertySet from a hierarchical
263 # dict but can update one.
264 ps = PropertySet()
265 ps.update(recipe)
266 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
267 else:
268 inMemoryDataset.writeFits(outputPath)
270 def getImageCompressionSettings(self, recipeName):
271 """Retrieve the relevant compression settings for this recipe.
273 Parameters
274 ----------
275 recipeName : `str`
276 Label associated with the collection of compression parameters
277 to select.
279 Returns
280 -------
281 settings : `dict`
282 The selected settings.
283 """
284 # if no recipe has been provided and there is no default
285 # return immediately
286 if not recipeName:
287 if "default" not in self.writeRecipes:
288 return {}
289 recipeName = "default"
291 if recipeName not in self.writeRecipes:
292 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
294 recipe = self.writeRecipes[recipeName]
296 # Set the seed based on dataId
297 seed = hash(tuple(self.dataId.items())) % 2**31
298 for plane in ("image", "mask", "variance"):
299 if plane in recipe and "scaling" in recipe[plane]:
300 scaling = recipe[plane]["scaling"]
301 if "seed" in scaling and scaling["seed"] == 0:
302 scaling["seed"] = seed
304 return recipe
306 @classmethod
307 def validateWriteRecipes(cls, recipes):
308 """Validate supplied recipes for this formatter.
310 The recipes are supplemented with default values where appropriate.
312 TODO: replace this custom validation code with Cerberus (DM-11846)
314 Parameters
315 ----------
316 recipes : `dict`
317 Recipes to validate. Can be empty dict or `None`.
319 Returns
320 -------
321 validated : `dict`
322 Validated recipes. Returns what was given if there are no
323 recipes listed.
325 Raises
326 ------
327 RuntimeError
328 Raised if validation fails.
329 """
330 # Schemas define what should be there, and the default values (and by
331 # the default value, the expected type).
332 compressionSchema = {
333 "algorithm": "NONE",
334 "rows": 1,
335 "columns": 0,
336 "quantizeLevel": 0.0,
337 }
338 scalingSchema = {
339 "algorithm": "NONE",
340 "bitpix": 0,
341 "maskPlanes": ["NO_DATA"],
342 "seed": 0,
343 "quantizeLevel": 4.0,
344 "quantizePad": 5.0,
345 "fuzz": True,
346 "bscale": 1.0,
347 "bzero": 0.0,
348 }
350 if not recipes:
351 # We can not insist on recipes being specified
352 return recipes
354 def checkUnrecognized(entry, allowed, description):
355 """Check to see if the entry contains unrecognised keywords"""
356 unrecognized = set(entry) - set(allowed)
357 if unrecognized:
358 raise RuntimeError(
359 f"Unrecognized entries when parsing image compression recipe {description}: "
360 f"{unrecognized}")
362 validated = {}
363 for name in recipes:
364 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
365 validated[name] = {}
366 for plane in ("image", "mask", "variance"):
367 checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
368 f"{name}->{plane}")
370 np = {}
371 validated[name][plane] = np
372 for settings, schema in (("compression", compressionSchema),
373 ("scaling", scalingSchema)):
374 np[settings] = {}
375 if settings not in recipes[name][plane]:
376 for key in schema:
377 np[settings][key] = schema[key]
378 continue
379 entry = recipes[name][plane][settings]
380 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
381 for key in schema:
382 value = type(schema[key])(entry[key]) if key in entry else schema[key]
383 np[settings][key] = value
384 return validated
387class FitsImageFormatter(StandardFitsImageFormatterBase):
388 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
389 from/to FITS.
390 """
392 ReaderClass = ImageFitsReader
395class FitsMaskFormatter(StandardFitsImageFormatterBase):
396 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
397 from/to FITS.
398 """
400 ReaderClass = MaskFitsReader
403class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
404 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
405 from/to FITS.
406 """
408 ReaderClass = MaskedImageFitsReader
410 def readComponent(self, component):
411 # Docstring inherited.
412 if component == "image":
413 return self.reader.readImage(**self.checked_parameters)
414 elif component == "mask":
415 return self.reader.readMask(**self.checked_parameters)
416 elif component == "variance":
417 return self.reader.readVariance(**self.checked_parameters)
418 else:
419 # Delegate to base for bbox, dimensions, xy0.
420 return super().readComponent(component)
423def standardizeAmplifierParameters(parameters, on_disk_detector):
424 """Preprocess the Exposure storage class's "amp" and "detector" parameters
426 This checks the given objects for consistency with the on-disk geometry and
427 converts amplifier IDs/names to Amplifier instances.
429 Parameters
430 ----------
431 parameters : `dict`
432 Dictionary of parameters passed to formatter. See the Exposure storage
433 class definition in daf_butler for allowed keys and values.
434 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
435 Detector that represents the on-disk image being loaded, or `None` if
436 this is unknown (and hence the user must provide one in
437 ``parameters`` if "amp" is in ``parameters``).
439 Returns
440 -------
441 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
442 An amplifier object that defines a subimage to load, or `None` if there
443 was no "amp" parameter.
444 detector : `lsst.afw.cameraGeom.Detector` or `None`
445 A detector object whose amplifiers are in the same s/orientation
446 state as the on-disk image. If there is no "amp" parameter,
447 ``on_disk_detector`` is simply passed through.
448 regions_differ : `bool`
449 `True` if the on-disk detector and the detector given in the parameters
450 had different bounding boxes for one or more regions. This can happen
451 if the true overscan region sizes can only be determined when the image
452 is actually read, but otherwise it should be considered user error.
453 """
454 if (amplifier := parameters.get("amp")) is None:
455 return None, on_disk_detector, False
456 if "bbox" in parameters or "origin" in parameters:
457 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
458 if isinstance(amplifier, (int, str)):
459 amp_key = amplifier
460 target_amplifier = None
461 else:
462 amp_key = amplifier.getName()
463 target_amplifier = amplifier
464 if (detector := parameters.get("detector")) is not None:
465 if on_disk_detector is not None:
466 # User passed a detector and we also found one on disk. Check them
467 # for consistency. Note that we are checking the amps we'd get
468 # from the two detectors against each other, not the amplifier we
469 # got directly from the user, as the latter is allowed to differ in
470 # assembly/orientation state.
471 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
472 if comparison & comparison.ASSEMBLY_DIFFERS:
473 raise ValueError(
474 "The given 'detector' has a different assembly state and/or orientation from "
475 f"the on-disk one for amp {amp_key}."
476 )
477 else:
478 if on_disk_detector is None:
479 raise ValueError(
480 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
481 "Please provide either a 'detector' parameter or an Amplifier instance in the "
482 "'amp' parameter."
483 )
484 comparison = AmplifierGeometryComparison.EQUAL
485 detector = on_disk_detector
486 if target_amplifier is None:
487 target_amplifier = detector[amp_key]
488 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
491class FitsExposureFormatter(FitsMaskedImageFormatter):
492 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
493 from/to FITS.
495 Notes
496 -----
497 This class inherits from `FitsMaskedImageFormatter` even though
498 `lsst.afw.image.Exposure` doesn't inherit from
499 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
500 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
501 should be replaced with e.g. both calling a free function if that slight
502 type covariance violation ever becomes a practical problem.
503 """
505 ReaderClass = ExposureFitsReader
507 def readComponent(self, component):
508 # Docstring inherited.
509 # Generic components can be read via a string name; DM-27754 will make
510 # this mapping larger at the expense of the following one.
511 genericComponents = {
512 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
513 }
514 if (genericComponentName := genericComponents.get(component)) is not None:
515 return self.reader.readComponent(genericComponentName)
516 # Other components have hard-coded method names, but don't take
517 # parameters.
518 standardComponents = {
519 'metadata': 'readMetadata',
520 'wcs': 'readWcs',
521 'coaddInputs': 'readCoaddInputs',
522 'psf': 'readPsf',
523 'photoCalib': 'readPhotoCalib',
524 # TODO: deprecate in DM-27170, remove in DM-27177
525 'filter': 'readFilter',
526 # TODO: deprecate in DM-27177, remove in DM-27811
527 'filterLabel': 'readFilterLabel',
528 'validPolygon': 'readValidPolygon',
529 'apCorrMap': 'readApCorrMap',
530 'visitInfo': 'readVisitInfo',
531 'transmissionCurve': 'readTransmissionCurve',
532 'detector': 'readDetector',
533 'exposureInfo': 'readExposureInfo',
534 }
535 if (methodName := standardComponents.get(component)) is not None:
536 result = getattr(self.reader, methodName)()
537 if component == "filterLabel":
538 return self._fixFilterLabels(result)
539 return result
540 # Delegate to MaskedImage and ImageBase implementations for the rest.
541 return super().readComponent(component)
543 def readFull(self):
544 # Docstring inherited.
545 amplifier, detector, _ = standardizeAmplifierParameters(
546 self.checked_parameters,
547 self.reader.readDetector(),
548 )
549 if amplifier is not None:
550 amplifier_isolator = AmplifierIsolator(
551 amplifier,
552 self.reader.readBBox(),
553 detector,
554 )
555 result = amplifier_isolator.transform_subimage(
556 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
557 )
558 result.setDetector(amplifier_isolator.make_detector())
559 else:
560 result = self.reader.read(**self.checked_parameters)
561 result.getInfo().setFilterLabel(self._fixFilterLabels(result.getInfo().getFilterLabel()))
562 return result
564 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
565 """Compare the filter label read from the file with the one in the
566 data ID.
568 Parameters
569 ----------
570 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
571 Filter label read from the file, if there was one.
572 should_be_standardized : `bool`, optional
573 If `True`, expect ``file_filter_label`` to be consistent with the
574 data ID and warn only if it is not. If `False`, expect it to be
575 inconsistent and warn only if the data ID is incomplete and hence
576 the `FilterLabel` cannot be fixed. If `None` (default) guess
577 whether the file should be standardized by looking at the
578 serialization version number in file, which requires this method to
579 have been run after `readFull` or `readComponent`.
581 Returns
582 -------
583 filter_label : `lsst.afw.image.FilterLabel` or `None`
584 The preferred filter label; may be the given one or one built from
585 the data ID. `None` is returned if there should never be any
586 filters associated with this dataset type.
588 Notes
589 -----
590 Most test coverage for this method is in ci_hsc_gen3, where we have
591 much easier access to test data that exhibits the problems it attempts
592 to solve.
593 """
594 # Remember filter data ID keys that weren't in this particular data ID,
595 # so we can warn about them later.
596 missing = []
597 band = None
598 physical_filter = None
599 if "band" in self.dataId.graph.dimensions.names:
600 band = self.dataId.get("band")
601 # band isn't in the data ID; is that just because this data ID
602 # hasn't been filled in with everything the Registry knows, or
603 # because this dataset is never associated with a band?
604 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
605 missing.append("band")
606 if "physical_filter" in self.dataId.graph.dimensions.names:
607 physical_filter = self.dataId.get("physical_filter")
608 # Same check as above for band, but for physical_filter.
609 if (physical_filter is None and not self.dataId.hasFull()
610 and "physical_filter" in self.dataId.graph.implied.names):
611 missing.append("physical_filter")
612 if should_be_standardized is None:
613 version = self.reader.readSerializationVersion()
614 should_be_standardized = (version >= 2)
615 if missing:
616 # Data ID identifies a filter but the actual filter label values
617 # haven't been fetched from the database; we have no choice but
618 # to use the one in the file.
619 # Warn if that's more likely than not to be bad, because the file
620 # predates filter standardization.
621 if not should_be_standardized:
622 warnings.warn(f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
623 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
624 "Call Registry.expandDataId before Butler.get to avoid this.")
625 return file_filter_label
626 if band is None and physical_filter is None:
627 data_id_filter_label = None
628 else:
629 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
630 if data_id_filter_label != file_filter_label and should_be_standardized:
631 # File was written after FilterLabel and standardization, but its
632 # FilterLabel doesn't agree with the data ID: this indicates a bug
633 # in whatever code produced the Exposure (though it may be one that
634 # has been fixed since the file was written).
635 warnings.warn(f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
636 f"filter label mismatch (file is {file_filter_label}, data ID is "
637 f"{data_id_filter_label}). This is probably a bug in the code that produced it.")
638 return data_id_filter_label