Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 21%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import warnings
31from abc import abstractmethod
33from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
34from lsst.afw.image import (
35 ExposureFitsReader,
36 ExposureInfo,
37 FilterLabel,
38 ImageFitsReader,
39 MaskedImageFitsReader,
40 MaskFitsReader,
41)
43# Needed for ApCorrMap to resolve properly
44from lsst.afw.math import BoundedField # noqa: F401
45from lsst.daf.base import PropertySet
46from lsst.daf.butler import Formatter
47from lsst.utils.classes import cached_getter
50class FitsImageFormatterBase(Formatter):
51 """Base class formatter for image-like storage classes stored via FITS.
53 Notes
54 -----
55 This class makes no assumptions about how many HDUs are used to represent
56 the image on disk, and includes no support for writing. It's really just a
57 collection of miscellaneous boilerplate common to all FITS image
58 formatters.
60 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
61 (even if just to disable them by raising an exception).
62 """
64 extension = ".fits"
65 supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
67 unsupportedParameters = {}
68 """Support all parameters."""
70 @property
71 @cached_getter
72 def checked_parameters(self):
73 """The parameters passed by the butler user, after checking them
74 against the storage class and transforming `None` into an empty `dict`
75 (`dict`).
77 This is computed on first use and then cached. It should never be
78 accessed when writing. Subclasses that need additional checking should
79 delegate to `super` and then check the result before returning it.
80 """
81 parameters = self.fileDescriptor.parameters
82 if parameters is None:
83 parameters = {}
84 self.fileDescriptor.storageClass.validateParameters(parameters)
85 return parameters
87 def read(self, component=None):
88 # Docstring inherited.
89 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass:
90 if component is not None:
91 return self.readComponent(component)
92 else:
93 raise ValueError(
94 "Storage class inconsistency ({} vs {}) but no"
95 " component requested".format(
96 self.fileDescriptor.readStorageClass.name, self.fileDescriptor.storageClass.name
97 )
98 )
99 return self.readFull()
101 @abstractmethod
102 def readComponent(self, component):
103 """Read a component dataset.
105 Parameters
106 ----------
107 component : `str`, optional
108 Component to read from the file.
110 Returns
111 -------
112 obj : component-dependent
113 In-memory component object.
115 Raises
116 ------
117 KeyError
118 Raised if the requested component cannot be handled.
119 """
120 raise NotImplementedError()
122 @abstractmethod
123 def readFull(self):
124 """Read the full dataset (while still accounting for parameters).
126 Returns
127 -------
128 obj : component-dependent
129 In-memory component object.
131 """
132 raise NotImplementedError()
135class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
136 """Base class formatter for image-like storage classes stored via FITS
137 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
139 Notes
140 -----
141 This class includes no support for writing.
143 Concrete subclasses must provide at least the `ReaderClass` attribute
144 and a `write` implementation (even just to disable writing by raising).
146 The provided implementation of `readComponent` handles only the 'bbox',
147 'dimensions', and 'xy0' components common to all image-like storage
148 classes. Subclasses with additional components should handle them first,
149 then delegate to ``super()`` for these (or, if necessary, delegate first
150 and catch `KeyError`).
152 The provided implementation of `readFull` handles only parameters that
153 can be forwarded directly to the reader class (usually ``bbox`` and
154 ``origin``). Concrete subclasses that need to handle additional parameters
155 should generally reimplement without delegating (the implementation is
156 trivial).
157 """
160class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
161 """Base class interface for image-like storage stored via FITS,
162 written using LSST code.
164 Notes
165 -----
166 Concrete subclasses must provide at least the `ReaderClass` attribute.
168 The provided implementation of `readComponent` handles only the 'bbox',
169 'dimensions', and 'xy0' components common to all image-like storage
170 classes. Subclasses with additional components should handle them first,
171 then delegate to ``super()`` for these (or, if necessary, delegate first
172 and catch `KeyError`).
174 The provided implementation of `readFull` handles only parameters that
175 can be forwarded directly to the reader class (usually ``bbox`` and
176 ``origin``). Concrete subclasses that need to handle additional parameters
177 should generally reimplement without delegating (the implementation is
178 trivial).
180 This Formatter supports write recipes, and assumes its in-memory type has
181 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
183 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
184 define ``image``, ``mask`` and ``variance`` entries, each of which may
185 contain ``compression`` and ``scaling`` entries. Defaults will be
186 provided for any missing elements under ``compression`` and
187 ``scaling``.
189 The allowed entries under ``compression`` are:
191 * ``algorithm`` (`str`): compression algorithm to use
192 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
193 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
194 * ``quantizeLevel`` (`float`): cfitsio quantization level
196 The allowed entries under ``scaling`` are:
198 * ``algorithm`` (`str`): scaling algorithm to use
199 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
200 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
201 * ``seed`` (`int`): seed for random number generator when fuzzing
202 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
203 statistics
204 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
205 ``STDEV_*`` scaling
206 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
207 ``STDEV_POSITIVE``/``NEGATIVE``)
208 * ``bscale`` (`float`): manually specified ``BSCALE``
209 (for ``MANUAL`` scaling)
210 * ``bzero`` (`float`): manually specified ``BSCALE``
211 (for ``MANUAL`` scaling)
213 A very simple example YAML recipe (for the ``Exposure`` specialization):
215 .. code-block:: yaml
217 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
218 default:
219 image: &default
220 compression:
221 algorithm: GZIP_SHUFFLE
222 mask: *default
223 variance: *default
225 """
227 supportedWriteParameters = frozenset({"recipe"})
228 ReaderClass: type # must be set by concrete subclasses
230 @property
231 @cached_getter
232 def reader(self):
233 """The reader object that backs this formatter's read operations.
235 This is computed on first use and then cached. It should never be
236 accessed when writing.
237 """
238 return self.ReaderClass(self.fileDescriptor.location.path)
240 def readComponent(self, component):
241 # Docstring inherited.
242 if component in ("bbox", "dimensions", "xy0"):
243 bbox = self.reader.readBBox()
244 if component == "dimensions":
245 return bbox.getDimensions()
246 elif component == "xy0":
247 return bbox.getMin()
248 else:
249 return bbox
250 else:
251 raise KeyError(f"Unknown component requested: {component}")
253 def readFull(self):
254 # Docstring inherited.
255 return self.reader.read(**self.checked_parameters)
257 def write(self, inMemoryDataset):
258 """Write a Python object to a file.
260 Parameters
261 ----------
262 inMemoryDataset : `object`
263 The Python object to store.
264 """
265 # Update the location with the formatter-preferred file extension
266 self.fileDescriptor.location.updateExtension(self.extension)
267 outputPath = self.fileDescriptor.location.path
269 # check to see if we have a recipe requested
270 recipeName = self.writeParameters.get("recipe")
271 recipe = self.getImageCompressionSettings(recipeName)
272 if recipe:
273 # Can not construct a PropertySet from a hierarchical
274 # dict but can update one.
275 ps = PropertySet()
276 ps.update(recipe)
277 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
278 else:
279 inMemoryDataset.writeFits(outputPath)
281 def getImageCompressionSettings(self, recipeName):
282 """Retrieve the relevant compression settings for this recipe.
284 Parameters
285 ----------
286 recipeName : `str`
287 Label associated with the collection of compression parameters
288 to select.
290 Returns
291 -------
292 settings : `dict`
293 The selected settings.
294 """
295 # if no recipe has been provided and there is no default
296 # return immediately
297 if not recipeName:
298 if "default" not in self.writeRecipes:
299 return {}
300 recipeName = "default"
302 if recipeName not in self.writeRecipes:
303 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
305 recipe = self.writeRecipes[recipeName]
307 # Set the seed based on dataId
308 seed = hash(tuple(self.dataId.items())) % 2**31
309 for plane in ("image", "mask", "variance"):
310 if plane in recipe and "scaling" in recipe[plane]:
311 scaling = recipe[plane]["scaling"]
312 if "seed" in scaling and scaling["seed"] == 0:
313 scaling["seed"] = seed
315 return recipe
317 @classmethod
318 def validateWriteRecipes(cls, recipes):
319 """Validate supplied recipes for this formatter.
321 The recipes are supplemented with default values where appropriate.
323 TODO: replace this custom validation code with Cerberus (DM-11846)
325 Parameters
326 ----------
327 recipes : `dict`
328 Recipes to validate. Can be empty dict or `None`.
330 Returns
331 -------
332 validated : `dict`
333 Validated recipes. Returns what was given if there are no
334 recipes listed.
336 Raises
337 ------
338 RuntimeError
339 Raised if validation fails.
340 """
341 # Schemas define what should be there, and the default values (and by
342 # the default value, the expected type).
343 compressionSchema = {
344 "algorithm": "NONE",
345 "rows": 1,
346 "columns": 0,
347 "quantizeLevel": 0.0,
348 }
349 scalingSchema = {
350 "algorithm": "NONE",
351 "bitpix": 0,
352 "maskPlanes": ["NO_DATA"],
353 "seed": 0,
354 "quantizeLevel": 4.0,
355 "quantizePad": 5.0,
356 "fuzz": True,
357 "bscale": 1.0,
358 "bzero": 0.0,
359 }
361 if not recipes:
362 # We can not insist on recipes being specified
363 return recipes
365 def checkUnrecognized(entry, allowed, description):
366 """Check to see if the entry contains unrecognised keywords"""
367 unrecognized = set(entry) - set(allowed)
368 if unrecognized:
369 raise RuntimeError(
370 f"Unrecognized entries when parsing image compression recipe {description}: "
371 f"{unrecognized}"
372 )
374 validated = {}
375 for name in recipes:
376 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
377 validated[name] = {}
378 for plane in ("image", "mask", "variance"):
379 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], f"{name}->{plane}")
381 np = {}
382 validated[name][plane] = np
383 for settings, schema in (("compression", compressionSchema), ("scaling", scalingSchema)):
384 np[settings] = {}
385 if settings not in recipes[name][plane]:
386 for key in schema:
387 np[settings][key] = schema[key]
388 continue
389 entry = recipes[name][plane][settings]
390 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
391 for key in schema:
392 value = type(schema[key])(entry[key]) if key in entry else schema[key]
393 np[settings][key] = value
394 return validated
397class FitsImageFormatter(StandardFitsImageFormatterBase):
398 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
399 from/to FITS.
400 """
402 ReaderClass = ImageFitsReader
405class FitsMaskFormatter(StandardFitsImageFormatterBase):
406 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
407 from/to FITS.
408 """
410 ReaderClass = MaskFitsReader
413class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
414 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
415 from/to FITS.
416 """
418 ReaderClass = MaskedImageFitsReader
420 def readComponent(self, component):
421 # Docstring inherited.
422 if component == "image":
423 return self.reader.readImage(**self.checked_parameters)
424 elif component == "mask":
425 return self.reader.readMask(**self.checked_parameters)
426 elif component == "variance":
427 return self.reader.readVariance(**self.checked_parameters)
428 else:
429 # Delegate to base for bbox, dimensions, xy0.
430 return super().readComponent(component)
433def standardizeAmplifierParameters(parameters, on_disk_detector):
434 """Preprocess the Exposure storage class's "amp" and "detector" parameters
436 This checks the given objects for consistency with the on-disk geometry and
437 converts amplifier IDs/names to Amplifier instances.
439 Parameters
440 ----------
441 parameters : `dict`
442 Dictionary of parameters passed to formatter. See the Exposure storage
443 class definition in daf_butler for allowed keys and values.
444 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
445 Detector that represents the on-disk image being loaded, or `None` if
446 this is unknown (and hence the user must provide one in
447 ``parameters`` if "amp" is in ``parameters``).
449 Returns
450 -------
451 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
452 An amplifier object that defines a subimage to load, or `None` if there
453 was no "amp" parameter.
454 detector : `lsst.afw.cameraGeom.Detector` or `None`
455 A detector object whose amplifiers are in the same s/orientation
456 state as the on-disk image. If there is no "amp" parameter,
457 ``on_disk_detector`` is simply passed through.
458 regions_differ : `bool`
459 `True` if the on-disk detector and the detector given in the parameters
460 had different bounding boxes for one or more regions. This can happen
461 if the true overscan region sizes can only be determined when the image
462 is actually read, but otherwise it should be considered user error.
463 """
464 if (amplifier := parameters.get("amp")) is None:
465 return None, on_disk_detector, False
466 if "bbox" in parameters or "origin" in parameters:
467 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
468 if isinstance(amplifier, (int, str)):
469 amp_key = amplifier
470 target_amplifier = None
471 else:
472 amp_key = amplifier.getName()
473 target_amplifier = amplifier
474 if (detector := parameters.get("detector")) is not None:
475 if on_disk_detector is not None:
476 # User passed a detector and we also found one on disk. Check them
477 # for consistency. Note that we are checking the amps we'd get
478 # from the two detectors against each other, not the amplifier we
479 # got directly from the user, as the latter is allowed to differ in
480 # assembly/orientation state.
481 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
482 if comparison & comparison.ASSEMBLY_DIFFERS:
483 raise ValueError(
484 "The given 'detector' has a different assembly state and/or orientation from "
485 f"the on-disk one for amp {amp_key}."
486 )
487 else:
488 if on_disk_detector is None:
489 raise ValueError(
490 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
491 "Please provide either a 'detector' parameter or an Amplifier instance in the "
492 "'amp' parameter."
493 )
494 comparison = AmplifierGeometryComparison.EQUAL
495 detector = on_disk_detector
496 if target_amplifier is None:
497 target_amplifier = detector[amp_key]
498 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
501class FitsExposureFormatter(FitsMaskedImageFormatter):
502 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
503 from/to FITS.
505 Notes
506 -----
507 This class inherits from `FitsMaskedImageFormatter` even though
508 `lsst.afw.image.Exposure` doesn't inherit from
509 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
510 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
511 should be replaced with e.g. both calling a free function if that slight
512 type covariance violation ever becomes a practical problem.
513 """
515 ReaderClass = ExposureFitsReader
517 def readComponent(self, component):
518 # Docstring inherited.
519 # Generic components can be read via a string name; DM-27754 will make
520 # this mapping larger at the expense of the following one.
521 genericComponents = {
522 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
523 }
524 if (genericComponentName := genericComponents.get(component)) is not None:
525 return self.reader.readComponent(genericComponentName)
526 # Other components have hard-coded method names, but don't take
527 # parameters.
528 standardComponents = {
529 "id": "readExposureId",
530 "metadata": "readMetadata",
531 "wcs": "readWcs",
532 "coaddInputs": "readCoaddInputs",
533 "psf": "readPsf",
534 "photoCalib": "readPhotoCalib",
535 # TODO: deprecate in DM-27170, remove in DM-27177
536 "filter": "readFilter",
537 # TODO: deprecate in DM-27177, remove in DM-27811
538 "filterLabel": "readFilterLabel",
539 "validPolygon": "readValidPolygon",
540 "apCorrMap": "readApCorrMap",
541 "visitInfo": "readVisitInfo",
542 "transmissionCurve": "readTransmissionCurve",
543 "detector": "readDetector",
544 "exposureInfo": "readExposureInfo",
545 }
546 if (methodName := standardComponents.get(component)) is not None:
547 result = getattr(self.reader, methodName)()
548 if component == "filterLabel":
549 return self._fixFilterLabels(result)
550 return result
551 # Delegate to MaskedImage and ImageBase implementations for the rest.
552 return super().readComponent(component)
554 def readFull(self):
555 # Docstring inherited.
556 amplifier, detector, _ = standardizeAmplifierParameters(
557 self.checked_parameters,
558 self.reader.readDetector(),
559 )
560 if amplifier is not None:
561 amplifier_isolator = AmplifierIsolator(
562 amplifier,
563 self.reader.readBBox(),
564 detector,
565 )
566 result = amplifier_isolator.transform_subimage(
567 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
568 )
569 result.setDetector(amplifier_isolator.make_detector())
570 else:
571 result = self.reader.read(**self.checked_parameters)
572 result.getInfo().setFilterLabel(self._fixFilterLabels(result.getInfo().getFilterLabel()))
573 return result
575 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
576 """Compare the filter label read from the file with the one in the
577 data ID.
579 Parameters
580 ----------
581 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
582 Filter label read from the file, if there was one.
583 should_be_standardized : `bool`, optional
584 If `True`, expect ``file_filter_label`` to be consistent with the
585 data ID and warn only if it is not. If `False`, expect it to be
586 inconsistent and warn only if the data ID is incomplete and hence
587 the `FilterLabel` cannot be fixed. If `None` (default) guess
588 whether the file should be standardized by looking at the
589 serialization version number in file, which requires this method to
590 have been run after `readFull` or `readComponent`.
592 Returns
593 -------
594 filter_label : `lsst.afw.image.FilterLabel` or `None`
595 The preferred filter label; may be the given one or one built from
596 the data ID. `None` is returned if there should never be any
597 filters associated with this dataset type.
599 Notes
600 -----
601 Most test coverage for this method is in ci_hsc_gen3, where we have
602 much easier access to test data that exhibits the problems it attempts
603 to solve.
604 """
605 # Remember filter data ID keys that weren't in this particular data ID,
606 # so we can warn about them later.
607 missing = []
608 band = None
609 physical_filter = None
610 if "band" in self.dataId.graph.dimensions.names:
611 band = self.dataId.get("band")
612 # band isn't in the data ID; is that just because this data ID
613 # hasn't been filled in with everything the Registry knows, or
614 # because this dataset is never associated with a band?
615 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names:
616 missing.append("band")
617 if "physical_filter" in self.dataId.graph.dimensions.names:
618 physical_filter = self.dataId.get("physical_filter")
619 # Same check as above for band, but for physical_filter.
620 if (
621 physical_filter is None
622 and not self.dataId.hasFull()
623 and "physical_filter" in self.dataId.graph.implied.names
624 ):
625 missing.append("physical_filter")
626 if should_be_standardized is None:
627 version = self.reader.readSerializationVersion()
628 should_be_standardized = version >= 2
629 if missing:
630 # Data ID identifies a filter but the actual filter label values
631 # haven't been fetched from the database; we have no choice but
632 # to use the one in the file.
633 # Warn if that's more likely than not to be bad, because the file
634 # predates filter standardization.
635 if not should_be_standardized:
636 warnings.warn(
637 f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; "
638 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
639 "Call Registry.expandDataId before Butler.get to avoid this."
640 )
641 return file_filter_label
642 if band is None and physical_filter is None:
643 data_id_filter_label = None
644 else:
645 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
646 if data_id_filter_label != file_filter_label and should_be_standardized:
647 # File was written after FilterLabel and standardization, but its
648 # FilterLabel doesn't agree with the data ID: this indicates a bug
649 # in whatever code produced the Exposure (though it may be one that
650 # has been fixed since the file was written).
651 warnings.warn(
652 f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: "
653 f"filter label mismatch (file is {file_filter_label}, data ID is "
654 f"{data_id_filter_label}). This is probably a bug in the code that produced it."
655 )
656 return data_id_filter_label