Coverage for python / lsst / obs / base / formatters / fitsExposure.py: 17%
335 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-15 06:36 -0300
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import hashlib
31import json
32import logging
33import threading
34import uuid
35import warnings
36from abc import abstractmethod
37from collections.abc import Set
38from io import BytesIO
39from typing import Any, ClassVar, NamedTuple
41import astropy.io.fits
42import numpy as np
44import lsst.geom
45from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
46from lsst.afw.fits import CompressionOptions, MemFileManager
47from lsst.afw.geom.wcsUtils import getImageXY0FromMetadata
48from lsst.afw.image import (
49 ExposureFitsReader,
50 ExposureInfo,
51 FilterLabel,
52 ImageFitsReader,
53 MaskedImageFitsReader,
54 MaskFitsReader,
55)
57# Needed for ApCorrMap to resolve properly
58from lsst.afw.math import BoundedField # noqa: F401
59from lsst.daf.base import PropertyList
60from lsst.daf.butler import DatasetProvenance, FormatterV2
61from lsst.resources import ResourcePath
62from lsst.utils.classes import cached_getter
63from lsst.utils.introspection import find_outside_stacklevel
65from ..utils import add_provenance_to_fits_header
67_LOG = logging.getLogger(__name__)
69_ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ = False
70"""If True, the astropy code will always be used to read component and cutouts
71even if the file is local, the cutout is too large, or the dataset type is
72wrong. This should mostly be used for testing.
73"""
76class FitsImageFormatterBase(FormatterV2):
77 """Base class formatter for image-like storage classes stored via FITS.
79 Notes
80 -----
81 This class makes no assumptions about how many HDUs are used to represent
82 the image on disk, and includes no support for writing. It's really just a
83 collection of miscellaneous boilerplate common to all FITS image
84 formatters.
86 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
87 (even if just to disable them by raising an exception).
88 """
90 can_read_from_local_file = True
91 default_extension = ".fits"
92 supported_extensions: ClassVar[Set[str]] = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
94 unsupported_parameters: ClassVar[Set[str]] = frozenset()
95 """Support all parameters."""
97 _reader = None
98 _reader_path: str | None = None
100 ReaderClass: type # must be set by concrete subclasses
102 @property
103 def reader(self):
104 """The reader object that backs this formatter's read operations.
106 This is computed on first use and then cached. It should never be
107 accessed when writing. Currently assumes a local file.
108 """
109 if self._reader is None:
110 self._reader = self.ReaderClass(self._reader_path)
111 return self._reader
113 @property
114 @cached_getter
115 def checked_parameters(self):
116 """The parameters passed by the butler user, after checking them
117 against the storage class and transforming `None` into an empty `dict`
118 (`dict`).
120 This is computed on first use and then cached. It should never be
121 accessed when writing. Subclasses that need additional checking should
122 delegate to `super` and then check the result before returning it.
123 """
124 parameters = self.file_descriptor.parameters
125 if parameters is None:
126 parameters = {}
127 self.file_descriptor.storageClass.validateParameters(parameters)
128 return parameters
130 @property
131 def storageClass_dtype(self) -> np.dtype | None:
132 """The numpy data type associated with the storage class."""
133 dtype: np.dtype | None = None
134 try:
135 # lsst.afw.image.Exposure is generic base class and does not have
136 # the dtype attribute.
137 dtype = np.dtype(self.file_descriptor.storageClass.pytype.dtype) # type: ignore[attr-defined]
138 except AttributeError:
139 pass
140 return dtype
142 def read_from_local_file(self, path: str, component: str | None = None, expected_size: int = -1) -> Any:
143 # Docstring inherited.
144 # The methods doing the reading all currently assume local file
145 # and assume that the file descriptor refers to a local file.
146 # With FormatterV2 that file descriptor does not refer to a local
147 # file.
148 self._reader_path = path
149 self._reader = None # Ensure the reader class is reset.
150 try:
151 if component is not None:
152 in_memory_dataset = self.readComponent(component)
153 else:
154 in_memory_dataset = self.readFull()
155 finally:
156 self._reader = None # Release the file handle.
157 return in_memory_dataset
159 @abstractmethod
160 def readComponent(self, component):
161 """Read a component dataset.
163 Parameters
164 ----------
165 component : `str`, optional
166 Component to read from the file.
168 Returns
169 -------
170 obj : component-dependent
171 In-memory component object.
173 Raises
174 ------
175 KeyError
176 Raised if the requested component cannot be handled.
177 """
178 raise NotImplementedError()
180 @abstractmethod
181 def readFull(self):
182 """Read the full dataset (while still accounting for parameters).
184 Returns
185 -------
186 obj : component-dependent
187 In-memory component object.
189 """
190 raise NotImplementedError()
193class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
194 """Base class formatter for image-like storage classes stored via FITS
195 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
197 Notes
198 -----
199 This class includes no support for writing.
201 Concrete subclasses must provide at least the `ReaderClass` attribute
202 and a `write` implementation (even just to disable writing by raising).
204 The provided implementation of `readComponent` handles only the 'bbox',
205 'dimensions', and 'xy0' components common to all image-like storage
206 classes. Subclasses with additional components should handle them first,
207 then delegate to ``super()`` for these (or, if necessary, delegate first
208 and catch `KeyError`).
210 The provided implementation of `readFull` handles only parameters that
211 can be forwarded directly to the reader class (usually ``bbox`` and
212 ``origin``). Concrete subclasses that need to handle additional parameters
213 should generally reimplement without delegating (the implementation is
214 trivial).
215 """
218class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
219 """Base class interface for image-like storage stored via FITS,
220 written using LSST code.
222 Notes
223 -----
224 Concrete subclasses must provide at least the `ReaderClass` attribute.
226 The provided implementation of `readComponent` handles only the 'bbox',
227 'dimensions', and 'xy0' components common to all image-like storage
228 classes. Subclasses with additional components should handle them first,
229 then delegate to ``super()`` for these (or, if necessary, delegate first
230 and catch `KeyError`).
232 The provided implementation of `readFull` handles only parameters that
233 can be forwarded directly to the reader class (usually ``bbox`` and
234 ``origin``). Concrete subclasses that need to handle additional parameters
235 should generally reimplement without delegating (the implementation is
236 trivial).
238 This Formatter supports write recipes, and assumes its in-memory type has
239 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
241 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
242 define ``image``, ``mask`` and ``variance`` entries, each of which may
243 contain entries supported by
244 `lsst.afw.fits.CompressionOptions.from_mapping` (``null`` disables
245 compression).
247 A very simple example YAML recipe (for the ``Exposure`` specialization):
249 .. code-block:: yaml
251 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
252 default:
253 image: &default
254 algorithm: GZIP_2
255 mask: *default
256 variance: *default
258 """
260 supported_write_parameters = frozenset({"recipe"})
262 def readComponent(self, component):
263 # Docstring inherited.
264 if component in ("bbox", "dimensions", "xy0"):
265 bbox = self.reader.readBBox()
266 if component == "dimensions":
267 return bbox.getDimensions()
268 elif component == "xy0":
269 return bbox.getMin()
270 else:
271 return bbox
272 else:
273 raise KeyError(f"Unknown component requested: {component}")
275 def readFull(self):
276 # Docstring inherited.
277 return self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
279 def write_local_file(self, in_memory_dataset: Any, uri: ResourcePath) -> None:
280 # check to see if we have a recipe requested
281 recipeName = self.write_parameters.get("recipe")
282 recipe = self.get_image_compression_settings(recipeName)
283 if recipe:
284 in_memory_dataset.writeFitsWithOptions(uri.ospath, options=recipe)
285 else:
286 in_memory_dataset.writeFits(uri.ospath)
288 def get_image_compression_settings(self, recipeName):
289 """Retrieve the relevant compression settings for this recipe.
291 Parameters
292 ----------
293 recipeName : `str`
294 Label associated with the collection of compression parameters
295 to select.
297 Returns
298 -------
299 settings : `dict`
300 The selected settings.
301 """
302 # if no recipe has been provided and there is no default
303 # return immediately
304 if not recipeName:
305 if "default" not in self.write_recipes:
306 return {}
307 recipeName = "default"
309 if recipeName not in self.write_recipes:
310 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
312 recipe = self.write_recipes[recipeName]
313 if recipe is None:
314 return
315 seed: int | None = None
316 for plane in ("image", "mask", "variance"):
317 if plane in recipe and (quantization := recipe[plane].get("quantization")) is not None:
318 if quantization.get("seed", 0) == 0:
319 if seed is None:
320 # Set the seed based on data ID. We can't just use
321 # 'hash', since like 'set' that's not deterministic.
322 # And we can't rely on a DimensionPacker because those
323 # are only defined for certain combinations of
324 # dimensions. Doing an MD5 of the JSON feels like
325 # overkill but I don't really see anything much
326 # simpler.
327 hash_bytes = hashlib.md5(
328 json.dumps(list(self.data_id.required_values)).encode(),
329 usedforsecurity=False,
330 ).digest()
331 # And it *really* feels like overkill when we squash
332 # that into the [1, 10000] range allowed by FITS.
333 seed = 1 + int.from_bytes(hash_bytes) % 9999
334 _LOG.debug(
335 "Setting compression quantization seed for %s %s %s to %s.",
336 self.data_id,
337 self.dataset_ref.datasetType.name,
338 plane,
339 seed,
340 )
341 quantization["seed"] = seed
342 else:
343 _LOG.warning(
344 "Compression quantization seed for %s %s %s was set explicitly to %s.",
345 self.dataset_ref.datasetType.name,
346 self.data_id,
347 plane,
348 quantization["seed"],
349 )
350 else:
351 _LOG.debug(
352 "No quantization found for %s %s %s.",
353 self.dataset_ref.datasetType.name,
354 self.data_id,
355 plane,
356 )
357 return recipe
359 @classmethod
360 def validate_write_recipes(cls, recipes):
361 """Validate supplied recipes for this formatter.
363 The recipes are supplemented with default values where appropriate.
365 Parameters
366 ----------
367 recipes : `dict`
368 Recipes to validate. Can be empty dict or `None`.
370 Returns
371 -------
372 validated : `dict`
373 Validated recipes. Returns what was given if there are no
374 recipes listed.
376 Raises
377 ------
378 RuntimeError
379 Raised if validation fails.
380 """
381 if not recipes:
382 # We can not insist on recipes being specified.
383 return recipes
385 validated = {}
386 for name, recipe in recipes.items():
387 if recipe is not None:
388 validated[name] = {}
389 for plane in ["image", "mask", "variance"]:
390 try:
391 options = CompressionOptions.from_mapping(recipe[plane])
392 except Exception as err:
393 err.add_note(f"Validating write recipe {name!r} ({plane!r} section).")
394 raise
395 validated[name][plane] = options.to_dict()
396 else:
397 validated[name] = None
398 return validated
401class FitsImageFormatter(StandardFitsImageFormatterBase):
402 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
403 from/to FITS.
404 """
406 ReaderClass = ImageFitsReader
409class FitsMaskFormatter(StandardFitsImageFormatterBase):
410 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
411 from/to FITS.
412 """
414 ReaderClass = MaskFitsReader
417class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
418 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
419 from/to FITS.
420 """
422 ReaderClass = MaskedImageFitsReader
424 def readComponent(self, component):
425 # Docstring inherited.
426 if component == "image":
427 return self.reader.readImage(**self.checked_parameters, dtype=self.storageClass_dtype)
428 elif component == "mask":
429 return self.reader.readMask(**self.checked_parameters)
430 elif component == "variance":
431 return self.reader.readVariance(**self.checked_parameters, dtype=self.storageClass_dtype)
432 else:
433 # Delegate to base for bbox, dimensions, xy0.
434 return super().readComponent(component)
437def standardizeAmplifierParameters(parameters, on_disk_detector):
438 """Preprocess the Exposure storage class's "amp" and "detector" parameters.
440 This checks the given objects for consistency with the on-disk geometry and
441 converts amplifier IDs/names to Amplifier instances.
443 Parameters
444 ----------
445 parameters : `dict`
446 Dictionary of parameters passed to formatter. See the Exposure storage
447 class definition in daf_butler for allowed keys and values.
448 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
449 Detector that represents the on-disk image being loaded, or `None` if
450 this is unknown (and hence the user must provide one in
451 ``parameters`` if "amp" is in ``parameters``).
453 Returns
454 -------
455 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
456 An amplifier object that defines a subimage to load, or `None` if there
457 was no "amp" parameter.
458 detector : `lsst.afw.cameraGeom.Detector` or `None`
459 A detector object whose amplifiers are in the same s/orientation
460 state as the on-disk image. If there is no "amp" parameter,
461 ``on_disk_detector`` is simply passed through.
462 regions_differ : `bool`
463 `True` if the on-disk detector and the detector given in the parameters
464 had different bounding boxes for one or more regions. This can happen
465 if the true overscan region sizes can only be determined when the image
466 is actually read, but otherwise it should be considered user error.
467 """
468 if (amplifier := parameters.get("amp")) is None:
469 return None, on_disk_detector, False
470 if "bbox" in parameters or "origin" in parameters:
471 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
472 if isinstance(amplifier, int | str):
473 amp_key = amplifier
474 target_amplifier = None
475 else:
476 amp_key = amplifier.getName()
477 target_amplifier = amplifier
478 if (detector := parameters.get("detector")) is not None:
479 if on_disk_detector is not None:
480 # User passed a detector and we also found one on disk. Check them
481 # for consistency. Note that we are checking the amps we'd get
482 # from the two detectors against each other, not the amplifier we
483 # got directly from the user, as the latter is allowed to differ in
484 # assembly/orientation state.
485 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
486 if comparison & comparison.ASSEMBLY_DIFFERS:
487 raise ValueError(
488 "The given 'detector' has a different assembly state and/or orientation from "
489 f"the on-disk one for amp {amp_key}."
490 )
491 else:
492 if on_disk_detector is None:
493 raise ValueError(
494 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
495 "Please provide either a 'detector' parameter or an Amplifier instance in the "
496 "'amp' parameter."
497 )
498 comparison = AmplifierGeometryComparison.EQUAL
499 detector = on_disk_detector
500 if target_amplifier is None:
501 target_amplifier = detector[amp_key]
502 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
505class _ComponentCache(NamedTuple):
506 id_: uuid.UUID | None = None
507 reader: ExposureFitsReader | None = None
508 bbox: lsst.geom.Box2I | None = None
509 mem: MemFileManager | None = None
512class FitsExposureFormatter(FitsMaskedImageFormatter):
513 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
514 from/to FITS.
516 Notes
517 -----
518 This class inherits from `FitsMaskedImageFormatter` even though
519 `lsst.afw.image.Exposure` doesn't inherit from
520 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
521 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
522 should be replaced with e.g. both calling a free function if that slight
523 type covariance violation ever becomes a practical problem.
524 """
526 can_read_from_uri = True
527 ReaderClass = ExposureFitsReader
528 # TODO: Remove MemFileManager from cache when DM-49640 is fixed.
529 _lock = threading.Lock()
530 _cached_fits: _ComponentCache = _ComponentCache()
532 def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
533 # For now only support small non-pixel components. In future
534 # could work with cutouts.
535 self._reader = None # Guarantee things are reset.
537 # Full read, always use local file read.
538 if not component and not self.checked_parameters:
539 return NotImplemented
541 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and uri.isLocal:
542 # For a local URI allow afw to read it directly.
543 return NotImplemented
544 pixel_components = ("mask", "image", "variance")
546 if component in pixel_components:
547 # For pixel access currently this can not be cached in memory
548 # and the performance gains are unclear. Assume local file
549 # read with file caching for now.
550 return NotImplemented
552 # With current file layouts the non-pixel extensions account for 1/3
553 # of the file size and it is more efficient to download the entire
554 # file.
555 if not (
556 _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ
557 or self._dataset_ref.dataId.mapping.keys().isdisjoint({"tract", "patch"})
558 ):
559 return NotImplemented
561 # Cutouts can be optimized. For now only use this optimization
562 # if bbox is the only parameter and the number of pixels in the
563 # bounding box is reasonable.
564 bbox = None
565 origin = lsst.afw.image.PARENT
566 if not component:
567 # Try to support PARENT and LOCAL origin but if there are any
568 # other parameters do not attempt a cutout.
569 if self.checked_parameters.keys() - {"bbox", "origin"}:
570 return NotImplemented
571 bbox = self.checked_parameters["bbox"]
572 origin = self.checked_parameters.get("origin", lsst.afw.image.PARENT)
573 # For larger cutouts use the full file.
574 max_cutout_size = 500 * 500
575 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and bbox.width * bbox.height > max_cutout_size:
576 return NotImplemented
578 # We only cache component reads since those are small.
579 if component:
580 with self._lock:
581 cache = type(self)._cached_fits
582 if self.dataset_ref.id == cache.id_:
583 if component in {"xy0", "dimensions", "bbox"} and cache.bbox is not None:
584 match component:
585 case "xy0":
586 return cache.bbox.getMin()
587 case "dimensions":
588 return cache.bbox.getDimensions()
589 case "bbox":
590 return cache.bbox
591 else:
592 self._reader = cache.reader
593 return self.readComponent(component)
595 try:
596 fs, fspath = uri.to_fsspec()
597 except Exception:
598 # fsspec cannot be initialized, fall back to downloading the file.
599 return NotImplemented
601 bbox_component = None
602 try:
603 hdul = []
604 with fs.open(fspath) as f, astropy.io.fits.open(f) as fits_obj:
605 # Read all non-pixel components and cache.
606 for hdu in fits_obj:
607 hdr = hdu.header
608 extname = hdr.get("EXTNAME")
609 # Older files have IMAGE in EXTNAME in PRIMARY so check
610 # for EXTEND=T.
611 extend = hdr.get("EXTEND")
612 if not extend and extname and extname.lower() in pixel_components:
613 # Calculate the dimensional components for later
614 # caching. Do not derive from cached FITS reader
615 # because they depend on the dimensionality of the
616 # pixel data and we do not want to cache the pixel
617 # data.
618 if bbox_component is None:
619 shape = hdu.shape
620 dimensions = lsst.geom.Extent2I(shape[1], shape[0])
622 # XY0 is defined in the A WCS.
623 pl = PropertyList()
624 pl.update(hdr)
625 xy0 = getImageXY0FromMetadata(pl, "A", strip=False)
627 # This is the PARENT bbox.
628 bbox_component = lsst.geom.Box2I(xy0, dimensions)
630 # Handle cutout request.
631 if bbox:
632 if origin == lsst.afw.image.PARENT:
633 full_bbox = bbox_component
634 else:
635 full_bbox = lsst.geom.Box2I(
636 lsst.geom.Point2I(0, 0), bbox_component.getDimensions
637 )
638 minX = bbox.getBeginX() - full_bbox.getBeginX()
639 maxX = bbox.getEndX() - full_bbox.getBeginX()
640 minY = bbox.getBeginY() - full_bbox.getBeginY()
641 maxY = bbox.getEndY() - full_bbox.getBeginY()
642 data = hdu.section[minY:maxY, minX:maxX]
644 # Must correct the header WCS to take into
645 # account the offset.
646 if (k := "CRPIX1") in hdr:
647 hdr[k] -= minX
648 if (k := "CRPIX2") in hdr:
649 hdr[k] -= minY
650 if (k := "LTV1") in hdr:
651 hdr[k] = -bbox.getBeginX()
652 if (k := "LTV2") in hdr:
653 hdr[k] = -bbox.getBeginY()
654 if (k := "CRVAL1A") in hdr:
655 hdr[k] = bbox.getBeginX()
656 if (k := "CRVAL2A") in hdr:
657 hdr[k] = bbox.getBeginY()
658 else:
659 data = np.zeros([1, 1], dtype=np.int32)
661 # Construct a new HDU and copy the header.
662 stripped_hdu = astropy.io.fits.ImageHDU(data=data, header=hdr)
663 hdul.append(stripped_hdu)
664 else:
665 hdul.append(hdu)
666 stripped_fits = astropy.io.fits.HDUList(hdus=hdul)
667 # Write the FITS file to in-memory FITS.
668 buffer = BytesIO()
669 stripped_fits.writeto(buffer)
670 except Exception as e:
671 # For some reason we can't open the remote file so fall back.
672 _LOG.debug(
673 "Attempted remote read of components but encountered an error. "
674 "Falling back to file download. Error: %s",
675 str(e),
676 )
677 return NotImplemented
679 # Pass the new FITS buffer to the reader class without going through
680 # a temporary file. We can assume this is relatively small for
681 # components.
682 fits_data = buffer.getvalue()
683 mem = MemFileManager(len(fits_data))
684 mem.setData(fits_data, len(fits_data))
685 self._reader = self.ReaderClass(mem)
687 if component:
688 with self._lock:
689 type(self)._cached_fits = _ComponentCache(
690 id_=self.dataset_ref.id,
691 reader=self._reader,
692 mem=mem,
693 bbox=bbox_component,
694 )
695 match component:
696 case "xy0":
697 if bbox_component is None: # For mypy.
698 return None
699 return bbox_component.getMin()
700 case "dimensions":
701 if bbox_component is None:
702 return None
703 return bbox_component.getDimensions()
704 case "bbox":
705 return bbox_component
706 case _:
707 return self.readComponent(component)
708 else:
709 # Must be a cutout. We have applied the bbox parameter so no
710 # parameters should be passed here.
711 cutout = self.reader.read(dtype=self.storageClass_dtype)
712 cutout.getInfo().setFilter(self._fixFilterLabels(cutout.getInfo().getFilter()))
713 return cutout
715 def add_provenance(
716 self, in_memory_dataset: Any, /, *, provenance: DatasetProvenance | None = None
717 ) -> Any:
718 # Add provenance via FITS headers.
719 add_provenance_to_fits_header(in_memory_dataset.metadata, self.dataset_ref, provenance)
720 return in_memory_dataset
722 def readComponent(self, component):
723 # Docstring inherited.
724 # Generic components can be read via a string name; DM-27754 will make
725 # this mapping larger at the expense of the following one.
726 genericComponents = {
727 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
728 }
729 if (genericComponentName := genericComponents.get(component)) is not None:
730 return self.reader.readComponent(genericComponentName)
731 # Other components have hard-coded method names, but don't take
732 # parameters.
733 standardComponents = {
734 "id": "readExposureId",
735 "metadata": "readMetadata",
736 "wcs": "readWcs",
737 "coaddInputs": "readCoaddInputs",
738 "psf": "readPsf",
739 "photoCalib": "readPhotoCalib",
740 "filter": "readFilter",
741 "validPolygon": "readValidPolygon",
742 "apCorrMap": "readApCorrMap",
743 "visitInfo": "readVisitInfo",
744 "transmissionCurve": "readTransmissionCurve",
745 "detector": "readDetector",
746 "exposureInfo": "readExposureInfo",
747 }
748 if (methodName := standardComponents.get(component)) is not None:
749 result = getattr(self.reader, methodName)()
750 if component == "filter":
751 return self._fixFilterLabels(result)
752 return result
753 # Delegate to MaskedImage and ImageBase implementations for the rest.
754 return super().readComponent(component)
756 def readFull(self):
757 # Docstring inherited.
758 amplifier, detector, _ = standardizeAmplifierParameters(
759 self.checked_parameters,
760 self.reader.readDetector(),
761 )
762 if amplifier is not None:
763 amplifier_isolator = AmplifierIsolator(
764 amplifier,
765 self.reader.readBBox(),
766 detector,
767 )
768 result = amplifier_isolator.transform_subimage(
769 self.reader.read(bbox=amplifier_isolator.subimage_bbox, dtype=self.storageClass_dtype)
770 )
771 result.setDetector(amplifier_isolator.make_detector())
772 else:
773 result = self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
774 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
775 return result
777 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
778 """Compare the filter label read from the file with the one in the
779 data ID.
781 Parameters
782 ----------
783 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
784 Filter label read from the file, if there was one.
785 should_be_standardized : `bool`, optional
786 If `True`, expect ``file_filter_label`` to be consistent with the
787 data ID and warn only if it is not. If `False`, expect it to be
788 inconsistent and warn only if the data ID is incomplete and hence
789 the `FilterLabel` cannot be fixed. If `None` (default) guess
790 whether the file should be standardized by looking at the
791 serialization version number in file, which requires this method to
792 have been run after `readFull` or `readComponent`.
794 Returns
795 -------
796 filter_label : `lsst.afw.image.FilterLabel` or `None`
797 The preferred filter label; may be the given one or one built from
798 the data ID. `None` is returned if there should never be any
799 filters associated with this dataset type.
801 Notes
802 -----
803 Most test coverage for this method is in ci_hsc_gen3, where we have
804 much easier access to test data that exhibits the problems it attempts
805 to solve.
806 """
807 # Remember filter data ID keys that weren't in this particular data ID,
808 # so we can warn about them later.
809 missing = []
810 band = None
811 physical_filter = None
812 if "band" in self.data_id.dimensions.names:
813 band = self.data_id.get("band")
814 # band isn't in the data ID; is that just because this data ID
815 # hasn't been filled in with everything the Registry knows, or
816 # because this dataset is never associated with a band?
817 if band is None and not self.data_id.hasFull() and "band" in self.data_id.dimensions.implied:
818 missing.append("band")
819 if "physical_filter" in self.data_id.dimensions.names:
820 physical_filter = self.data_id.get("physical_filter")
821 # Same check as above for band, but for physical_filter.
822 if (
823 physical_filter is None
824 and not self.data_id.hasFull()
825 and "physical_filter" in self.data_id.dimensions.implied
826 ):
827 missing.append("physical_filter")
828 if should_be_standardized is None:
829 version = self.reader.readSerializationVersion()
830 should_be_standardized = version >= 2
831 if missing:
832 # Data ID identifies a filter but the actual filter label values
833 # haven't been fetched from the database; we have no choice but
834 # to use the one in the file.
835 # Warn if that's more likely than not to be bad, because the file
836 # predates filter standardization.
837 if not should_be_standardized:
838 warnings.warn(
839 f"Data ID {self.data_id} is missing (implied) value(s) for {missing}; "
840 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
841 "Call Registry.expandDataId before Butler.get to avoid this.",
842 # Report the warning from outside of middleware or the
843 # relevant runQuantum method.
844 stacklevel=find_outside_stacklevel(
845 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
846 ),
847 )
848 return file_filter_label
849 if band is None and physical_filter is None:
850 data_id_filter_label = None
851 else:
852 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
853 if data_id_filter_label != file_filter_label and should_be_standardized:
854 # File was written after FilterLabel and standardization, but its
855 # FilterLabel doesn't agree with the data ID: this indicates a bug
856 # in whatever code produced the Exposure (though it may be one that
857 # has been fixed since the file was written).
858 warnings.warn(
859 f"Reading {self.file_descriptor.location} with data ID {self.data_id}: "
860 f"filter label mismatch (file is {file_filter_label}, data ID is "
861 f"{data_id_filter_label}). This is probably a bug in the code that produced it.",
862 stacklevel=find_outside_stacklevel(
863 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
864 ),
865 )
866 return data_id_filter_label