Coverage for python/lsst/obs/base/formatters/fitsExposure.py: 16%
333 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-10-10 01:33 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import logging
31import threading
32import uuid
33import warnings
34from abc import abstractmethod
35from collections.abc import Set
36from io import BytesIO
37from typing import Any, ClassVar, NamedTuple
39import astropy.io.fits
40import numpy as np
42import lsst.geom
43from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
44from lsst.afw.fits import MemFileManager
45from lsst.afw.geom.wcsUtils import getImageXY0FromMetadata
46from lsst.afw.image import (
47 ExposureFitsReader,
48 ExposureInfo,
49 FilterLabel,
50 ImageFitsReader,
51 MaskedImageFitsReader,
52 MaskFitsReader,
53)
55# Needed for ApCorrMap to resolve properly
56from lsst.afw.math import BoundedField # noqa: F401
57from lsst.daf.base import PropertyList, PropertySet
58from lsst.daf.butler import DatasetProvenance, FormatterV2
59from lsst.resources import ResourcePath
60from lsst.utils.classes import cached_getter
61from lsst.utils.introspection import find_outside_stacklevel
63from ..utils import add_provenance_to_fits_header
65_LOG = logging.getLogger()
66_ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ = False
67"""If True, the astropy code will always be used to read component and cutouts
68even if the file is local, the cutout is too large, or the dataset type is
69wrong. This should mostly be used for testing.
70"""
73class FitsImageFormatterBase(FormatterV2):
74 """Base class formatter for image-like storage classes stored via FITS.
76 Notes
77 -----
78 This class makes no assumptions about how many HDUs are used to represent
79 the image on disk, and includes no support for writing. It's really just a
80 collection of miscellaneous boilerplate common to all FITS image
81 formatters.
83 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
84 (even if just to disable them by raising an exception).
85 """
87 can_read_from_local_file = True
88 default_extension = ".fits"
89 supported_extensions: ClassVar[Set[str]] = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
91 unsupported_parameters: ClassVar[Set[str]] = frozenset()
92 """Support all parameters."""
94 _reader = None
95 _reader_path: str | None = None
97 ReaderClass: type # must be set by concrete subclasses
99 @property
100 def reader(self):
101 """The reader object that backs this formatter's read operations.
103 This is computed on first use and then cached. It should never be
104 accessed when writing. Currently assumes a local file.
105 """
106 if self._reader is None:
107 self._reader = self.ReaderClass(self._reader_path)
108 return self._reader
110 @property
111 @cached_getter
112 def checked_parameters(self):
113 """The parameters passed by the butler user, after checking them
114 against the storage class and transforming `None` into an empty `dict`
115 (`dict`).
117 This is computed on first use and then cached. It should never be
118 accessed when writing. Subclasses that need additional checking should
119 delegate to `super` and then check the result before returning it.
120 """
121 parameters = self.file_descriptor.parameters
122 if parameters is None:
123 parameters = {}
124 self.file_descriptor.storageClass.validateParameters(parameters)
125 return parameters
127 def read_from_local_file(self, path: str, component: str | None = None, expected_size: int = -1) -> Any:
128 # Docstring inherited.
129 # The methods doing the reading all currently assume local file
130 # and assume that the file descriptor refers to a local file.
131 # With FormatterV2 that file descriptor does not refer to a local
132 # file.
133 self._reader_path = path
134 self._reader = None # Ensure the reader class is reset.
135 try:
136 if component is not None:
137 in_memory_dataset = self.readComponent(component)
138 else:
139 in_memory_dataset = self.readFull()
140 finally:
141 self._reader = None # Release the file handle.
142 return in_memory_dataset
144 @abstractmethod
145 def readComponent(self, component):
146 """Read a component dataset.
148 Parameters
149 ----------
150 component : `str`, optional
151 Component to read from the file.
153 Returns
154 -------
155 obj : component-dependent
156 In-memory component object.
158 Raises
159 ------
160 KeyError
161 Raised if the requested component cannot be handled.
162 """
163 raise NotImplementedError()
165 @abstractmethod
166 def readFull(self):
167 """Read the full dataset (while still accounting for parameters).
169 Returns
170 -------
171 obj : component-dependent
172 In-memory component object.
174 """
175 raise NotImplementedError()
178class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
179 """Base class formatter for image-like storage classes stored via FITS
180 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
182 Notes
183 -----
184 This class includes no support for writing.
186 Concrete subclasses must provide at least the `ReaderClass` attribute
187 and a `write` implementation (even just to disable writing by raising).
189 The provided implementation of `readComponent` handles only the 'bbox',
190 'dimensions', and 'xy0' components common to all image-like storage
191 classes. Subclasses with additional components should handle them first,
192 then delegate to ``super()`` for these (or, if necessary, delegate first
193 and catch `KeyError`).
195 The provided implementation of `readFull` handles only parameters that
196 can be forwarded directly to the reader class (usually ``bbox`` and
197 ``origin``). Concrete subclasses that need to handle additional parameters
198 should generally reimplement without delegating (the implementation is
199 trivial).
200 """
203class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
204 """Base class interface for image-like storage stored via FITS,
205 written using LSST code.
207 Notes
208 -----
209 Concrete subclasses must provide at least the `ReaderClass` attribute.
211 The provided implementation of `readComponent` handles only the 'bbox',
212 'dimensions', and 'xy0' components common to all image-like storage
213 classes. Subclasses with additional components should handle them first,
214 then delegate to ``super()`` for these (or, if necessary, delegate first
215 and catch `KeyError`).
217 The provided implementation of `readFull` handles only parameters that
218 can be forwarded directly to the reader class (usually ``bbox`` and
219 ``origin``). Concrete subclasses that need to handle additional parameters
220 should generally reimplement without delegating (the implementation is
221 trivial).
223 This Formatter supports write recipes, and assumes its in-memory type has
224 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
226 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
227 define ``image``, ``mask`` and ``variance`` entries, each of which may
228 contain ``compression`` and ``scaling`` entries. Defaults will be
229 provided for any missing elements under ``compression`` and
230 ``scaling``.
232 The allowed entries under ``compression`` are:
234 * ``algorithm`` (`str`): compression algorithm to use
235 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
236 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
237 * ``quantizeLevel`` (`float`): cfitsio quantization level
239 The allowed entries under ``scaling`` are:
241 * ``algorithm`` (`str`): scaling algorithm to use
242 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
243 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
244 * ``seed`` (`int`): seed for random number generator when fuzzing
245 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
246 statistics
247 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
248 ``STDEV_*`` scaling
249 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
250 ``STDEV_POSITIVE``/``NEGATIVE``)
251 * ``bscale`` (`float`): manually specified ``BSCALE``
252 (for ``MANUAL`` scaling)
253 * ``bzero`` (`float`): manually specified ``BSCALE``
254 (for ``MANUAL`` scaling)
256 A very simple example YAML recipe (for the ``Exposure`` specialization):
258 .. code-block:: yaml
260 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
261 default:
262 image: &default
263 compression:
264 algorithm: GZIP_SHUFFLE
265 mask: *default
266 variance: *default
268 """
270 supported_write_parameters = frozenset({"recipe"})
272 def readComponent(self, component):
273 # Docstring inherited.
274 if component in ("bbox", "dimensions", "xy0"):
275 bbox = self.reader.readBBox()
276 if component == "dimensions":
277 return bbox.getDimensions()
278 elif component == "xy0":
279 return bbox.getMin()
280 else:
281 return bbox
282 else:
283 raise KeyError(f"Unknown component requested: {component}")
285 def readFull(self):
286 # Docstring inherited.
287 return self.reader.read(**self.checked_parameters)
289 def write_local_file(self, in_memory_dataset: Any, uri: ResourcePath) -> None:
290 # check to see if we have a recipe requested
291 recipeName = self.write_parameters.get("recipe")
292 recipe = self.get_image_compression_settings(recipeName)
293 if recipe:
294 # Can not construct a PropertySet from a hierarchical
295 # dict but can update one.
296 ps = PropertySet()
297 ps.update(recipe)
298 in_memory_dataset.writeFitsWithOptions(uri.ospath, options=ps)
299 else:
300 in_memory_dataset.writeFits(uri.ospath)
302 def get_image_compression_settings(self, recipeName):
303 """Retrieve the relevant compression settings for this recipe.
305 Parameters
306 ----------
307 recipeName : `str`
308 Label associated with the collection of compression parameters
309 to select.
311 Returns
312 -------
313 settings : `dict`
314 The selected settings.
315 """
316 # if no recipe has been provided and there is no default
317 # return immediately
318 if not recipeName:
319 if "default" not in self.write_recipes:
320 return {}
321 recipeName = "default"
323 if recipeName not in self.write_recipes:
324 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
326 recipe = self.write_recipes[recipeName]
328 # Set the seed based on dataId
329 seed = hash(tuple(self.data_id.required.items())) % 2**31
330 for plane in ("image", "mask", "variance"):
331 if plane in recipe and "scaling" in recipe[plane]:
332 scaling = recipe[plane]["scaling"]
333 if "seed" in scaling and scaling["seed"] == 0:
334 scaling["seed"] = seed
336 return recipe
338 @classmethod
339 def validate_write_recipes(cls, recipes):
340 """Validate supplied recipes for this formatter.
342 The recipes are supplemented with default values where appropriate.
344 TODO: replace this custom validation code with Cerberus (DM-11846)
346 Parameters
347 ----------
348 recipes : `dict`
349 Recipes to validate. Can be empty dict or `None`.
351 Returns
352 -------
353 validated : `dict`
354 Validated recipes. Returns what was given if there are no
355 recipes listed.
357 Raises
358 ------
359 RuntimeError
360 Raised if validation fails.
361 """
362 # Schemas define what should be there, and the default values (and by
363 # the default value, the expected type).
364 compressionSchema = {
365 "algorithm": "NONE",
366 "rows": 1,
367 "columns": 0,
368 "quantizeLevel": 0.0,
369 }
370 scalingSchema = {
371 "algorithm": "NONE",
372 "bitpix": 0,
373 "maskPlanes": ["NO_DATA"],
374 "seed": 0,
375 "quantizeLevel": 4.0,
376 "quantizePad": 5.0,
377 "fuzz": True,
378 "bscale": 1.0,
379 "bzero": 0.0,
380 }
382 if not recipes:
383 # We can not insist on recipes being specified
384 return recipes
386 def checkUnrecognized(entry, allowed, description):
387 """Check to see if the entry contains unrecognised keywords."""
388 unrecognized = set(entry) - set(allowed)
389 if unrecognized:
390 raise RuntimeError(
391 f"Unrecognized entries when parsing image compression recipe {description}: "
392 f"{unrecognized}"
393 )
395 validated = {}
396 for name in recipes:
397 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
398 validated[name] = {}
399 for plane in ("image", "mask", "variance"):
400 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], f"{name}->{plane}")
402 np = {}
403 validated[name][plane] = np
404 for settings, schema in (("compression", compressionSchema), ("scaling", scalingSchema)):
405 np[settings] = {}
406 if settings not in recipes[name][plane]:
407 for key in schema:
408 np[settings][key] = schema[key]
409 continue
410 entry = recipes[name][plane][settings]
411 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
412 for key in schema:
413 value = type(schema[key])(entry[key]) if key in entry else schema[key]
414 np[settings][key] = value
415 return validated
418class FitsImageFormatter(StandardFitsImageFormatterBase):
419 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
420 from/to FITS.
421 """
423 ReaderClass = ImageFitsReader
426class FitsMaskFormatter(StandardFitsImageFormatterBase):
427 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
428 from/to FITS.
429 """
431 ReaderClass = MaskFitsReader
434class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
435 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
436 from/to FITS.
437 """
439 ReaderClass = MaskedImageFitsReader
441 def readComponent(self, component):
442 # Docstring inherited.
443 if component == "image":
444 return self.reader.readImage(**self.checked_parameters)
445 elif component == "mask":
446 return self.reader.readMask(**self.checked_parameters)
447 elif component == "variance":
448 return self.reader.readVariance(**self.checked_parameters)
449 else:
450 # Delegate to base for bbox, dimensions, xy0.
451 return super().readComponent(component)
454def standardizeAmplifierParameters(parameters, on_disk_detector):
455 """Preprocess the Exposure storage class's "amp" and "detector" parameters.
457 This checks the given objects for consistency with the on-disk geometry and
458 converts amplifier IDs/names to Amplifier instances.
460 Parameters
461 ----------
462 parameters : `dict`
463 Dictionary of parameters passed to formatter. See the Exposure storage
464 class definition in daf_butler for allowed keys and values.
465 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
466 Detector that represents the on-disk image being loaded, or `None` if
467 this is unknown (and hence the user must provide one in
468 ``parameters`` if "amp" is in ``parameters``).
470 Returns
471 -------
472 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
473 An amplifier object that defines a subimage to load, or `None` if there
474 was no "amp" parameter.
475 detector : `lsst.afw.cameraGeom.Detector` or `None`
476 A detector object whose amplifiers are in the same s/orientation
477 state as the on-disk image. If there is no "amp" parameter,
478 ``on_disk_detector`` is simply passed through.
479 regions_differ : `bool`
480 `True` if the on-disk detector and the detector given in the parameters
481 had different bounding boxes for one or more regions. This can happen
482 if the true overscan region sizes can only be determined when the image
483 is actually read, but otherwise it should be considered user error.
484 """
485 if (amplifier := parameters.get("amp")) is None:
486 return None, on_disk_detector, False
487 if "bbox" in parameters or "origin" in parameters:
488 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
489 if isinstance(amplifier, int | str):
490 amp_key = amplifier
491 target_amplifier = None
492 else:
493 amp_key = amplifier.getName()
494 target_amplifier = amplifier
495 if (detector := parameters.get("detector")) is not None:
496 if on_disk_detector is not None:
497 # User passed a detector and we also found one on disk. Check them
498 # for consistency. Note that we are checking the amps we'd get
499 # from the two detectors against each other, not the amplifier we
500 # got directly from the user, as the latter is allowed to differ in
501 # assembly/orientation state.
502 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
503 if comparison & comparison.ASSEMBLY_DIFFERS:
504 raise ValueError(
505 "The given 'detector' has a different assembly state and/or orientation from "
506 f"the on-disk one for amp {amp_key}."
507 )
508 else:
509 if on_disk_detector is None:
510 raise ValueError(
511 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
512 "Please provide either a 'detector' parameter or an Amplifier instance in the "
513 "'amp' parameter."
514 )
515 comparison = AmplifierGeometryComparison.EQUAL
516 detector = on_disk_detector
517 if target_amplifier is None:
518 target_amplifier = detector[amp_key]
519 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
522class _ComponentCache(NamedTuple):
523 id_: uuid.UUID | None = None
524 reader: ExposureFitsReader | None = None
525 bbox: lsst.geom.Box2I | None = None
526 mem: MemFileManager | None = None
529class FitsExposureFormatter(FitsMaskedImageFormatter):
530 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
531 from/to FITS.
533 Notes
534 -----
535 This class inherits from `FitsMaskedImageFormatter` even though
536 `lsst.afw.image.Exposure` doesn't inherit from
537 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
538 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
539 should be replaced with e.g. both calling a free function if that slight
540 type covariance violation ever becomes a practical problem.
541 """
543 can_read_from_uri = True
544 ReaderClass = ExposureFitsReader
545 # TODO: Remove MemFileManager from cache when DM-49640 is fixed.
546 _lock = threading.Lock()
547 _cached_fits: _ComponentCache = _ComponentCache()
549 def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
550 # For now only support small non-pixel components. In future
551 # could work with cutouts.
552 self._reader = None # Guarantee things are reset.
554 # Full read, always use local file read.
555 if not component and not self.checked_parameters:
556 return NotImplemented
558 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and uri.isLocal:
559 # For a local URI allow afw to read it directly.
560 return NotImplemented
561 pixel_components = ("mask", "image", "variance")
563 if component in pixel_components:
564 # For pixel access currently this can not be cached in memory
565 # and the performance gains are unclear. Assume local file
566 # read with file caching for now.
567 return NotImplemented
569 # With current file layouts the non-pixel extensions account for 1/3
570 # of the file size and it is more efficient to download the entire
571 # file.
572 if not (
573 _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ
574 or self._dataset_ref.dataId.mapping.keys().isdisjoint({"tract", "patch"})
575 ):
576 return NotImplemented
578 # Cutouts can be optimized. For now only use this optimization
579 # if bbox is the only parameter and the number of pixels in the
580 # bounding box is reasonable.
581 bbox = None
582 origin = lsst.afw.image.PARENT
583 if not component:
584 # Try to support PARENT and LOCAL origin but if there are any
585 # other parameters do not attempt a cutout.
586 if self.checked_parameters.keys() - {"bbox", "origin"}:
587 return NotImplemented
588 bbox = self.checked_parameters["bbox"]
589 origin = self.checked_parameters.get("origin", lsst.afw.image.PARENT)
590 # For larger cutouts use the full file.
591 max_cutout_size = 500 * 500
592 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and bbox.width * bbox.height > max_cutout_size:
593 return NotImplemented
595 # We only cache component reads since those are small.
596 if component:
597 with self._lock:
598 cache = type(self)._cached_fits
599 if self.dataset_ref.id == cache.id_:
600 if component in {"xy0", "dimensions", "bbox"} and cache.bbox is not None:
601 match component:
602 case "xy0":
603 return cache.bbox.getMin()
604 case "dimensions":
605 return cache.bbox.getDimensions()
606 case "bbox":
607 return cache.bbox
608 else:
609 self._reader = cache.reader
610 return self.readComponent(component)
612 try:
613 fs, fspath = uri.to_fsspec()
614 except Exception:
615 # fsspec cannot be initialized, fall back to downloading the file.
616 return NotImplemented
618 bbox_component = None
619 try:
620 hdul = []
621 with fs.open(fspath) as f, astropy.io.fits.open(f) as fits_obj:
622 # Read all non-pixel components and cache.
623 for hdu in fits_obj:
624 hdr = hdu.header
625 extname = hdr.get("EXTNAME")
626 # Older files have IMAGE in EXTNAME in PRIMARY so check
627 # for EXTEND=T.
628 extend = hdr.get("EXTEND")
629 if not extend and extname and extname.lower() in pixel_components:
630 # Calculate the dimensional components for later
631 # caching. Do not derive from cached FITS reader
632 # because they depend on the dimensionality of the
633 # pixel data and we do not want to cache the pixel
634 # data.
635 if bbox_component is None:
636 shape = hdu.shape
637 dimensions = lsst.geom.Extent2I(shape[1], shape[0])
639 # XY0 is defined in the A WCS.
640 pl = PropertyList()
641 pl.update(hdr)
642 xy0 = getImageXY0FromMetadata(pl, "A", strip=False)
644 # This is the PARENT bbox.
645 bbox_component = lsst.geom.Box2I(xy0, dimensions)
647 # Handle cutout request.
648 if bbox:
649 if origin == lsst.afw.image.PARENT:
650 full_bbox = bbox_component
651 else:
652 full_bbox = lsst.geom.Box2I(
653 lsst.geom.Point2I(0, 0), bbox_component.getDimensions
654 )
655 minX = bbox.getBeginX() - full_bbox.getBeginX()
656 maxX = bbox.getEndX() - full_bbox.getBeginX()
657 minY = bbox.getBeginY() - full_bbox.getBeginY()
658 maxY = bbox.getEndY() - full_bbox.getBeginY()
659 data = hdu.section[minY:maxY, minX:maxX]
661 # Must correct the header WCS to take into
662 # account the offset.
663 if (k := "CRPIX1") in hdr:
664 hdr[k] -= minX
665 if (k := "CRPIX2") in hdr:
666 hdr[k] -= minY
667 if (k := "LTV1") in hdr:
668 hdr[k] = -bbox.getBeginX()
669 if (k := "LTV2") in hdr:
670 hdr[k] = -bbox.getBeginY()
671 if (k := "CRVAL1A") in hdr:
672 hdr[k] = bbox.getBeginX()
673 if (k := "CRVAL2A") in hdr:
674 hdr[k] = bbox.getBeginY()
675 else:
676 data = np.zeros([1, 1], dtype=np.int32)
678 # Construct a new HDU and copy the header.
679 stripped_hdu = astropy.io.fits.ImageHDU(data=data, header=hdr)
680 hdul.append(stripped_hdu)
681 else:
682 hdul.append(hdu)
683 stripped_fits = astropy.io.fits.HDUList(hdus=hdul)
684 # Write the FITS file to in-memory FITS.
685 buffer = BytesIO()
686 stripped_fits.writeto(buffer)
687 except Exception as e:
688 # For some reason we can't open the remote file so fall back.
689 _LOG.debug(
690 "Attempted remote read of components but encountered an error. "
691 "Falling back to file download. Error: %s",
692 str(e),
693 )
694 return NotImplemented
696 # Pass the new FITS buffer to the reader class without going through
697 # a temporary file. We can assume this is relatively small for
698 # components.
699 fits_data = buffer.getvalue()
700 mem = MemFileManager(len(fits_data))
701 mem.setData(fits_data, len(fits_data))
702 self._reader = self.ReaderClass(mem)
704 if component:
705 with self._lock:
706 type(self)._cached_fits = _ComponentCache(
707 id_=self.dataset_ref.id,
708 reader=self._reader,
709 mem=mem,
710 bbox=bbox_component,
711 )
712 match component:
713 case "xy0":
714 if bbox_component is None: # For mypy.
715 return None
716 return bbox_component.getMin()
717 case "dimensions":
718 if bbox_component is None:
719 return None
720 return bbox_component.getDimensions()
721 case "bbox":
722 return bbox_component
723 case _:
724 return self.readComponent(component)
725 else:
726 # Must be a cutout. We have applied the bbox parameter so no
727 # parameters should be passed here.
728 cutout = self.reader.read()
729 cutout.getInfo().setFilter(self._fixFilterLabels(cutout.getInfo().getFilter()))
730 return cutout
732 def add_provenance(
733 self, in_memory_dataset: Any, /, *, provenance: DatasetProvenance | None = None
734 ) -> Any:
735 # Add provenance via FITS headers.
736 add_provenance_to_fits_header(in_memory_dataset.metadata, self.dataset_ref, provenance)
737 return in_memory_dataset
739 def readComponent(self, component):
740 # Docstring inherited.
741 # Generic components can be read via a string name; DM-27754 will make
742 # this mapping larger at the expense of the following one.
743 genericComponents = {
744 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
745 }
746 if (genericComponentName := genericComponents.get(component)) is not None:
747 return self.reader.readComponent(genericComponentName)
748 # Other components have hard-coded method names, but don't take
749 # parameters.
750 standardComponents = {
751 "id": "readExposureId",
752 "metadata": "readMetadata",
753 "wcs": "readWcs",
754 "coaddInputs": "readCoaddInputs",
755 "psf": "readPsf",
756 "photoCalib": "readPhotoCalib",
757 "filter": "readFilter",
758 "validPolygon": "readValidPolygon",
759 "apCorrMap": "readApCorrMap",
760 "visitInfo": "readVisitInfo",
761 "transmissionCurve": "readTransmissionCurve",
762 "detector": "readDetector",
763 "exposureInfo": "readExposureInfo",
764 }
765 if (methodName := standardComponents.get(component)) is not None:
766 result = getattr(self.reader, methodName)()
767 if component == "filter":
768 return self._fixFilterLabels(result)
769 return result
770 # Delegate to MaskedImage and ImageBase implementations for the rest.
771 return super().readComponent(component)
773 def readFull(self):
774 # Docstring inherited.
775 amplifier, detector, _ = standardizeAmplifierParameters(
776 self.checked_parameters,
777 self.reader.readDetector(),
778 )
779 if amplifier is not None:
780 amplifier_isolator = AmplifierIsolator(
781 amplifier,
782 self.reader.readBBox(),
783 detector,
784 )
785 result = amplifier_isolator.transform_subimage(
786 self.reader.read(bbox=amplifier_isolator.subimage_bbox)
787 )
788 result.setDetector(amplifier_isolator.make_detector())
789 else:
790 result = self.reader.read(**self.checked_parameters)
791 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
792 return result
794 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None):
795 """Compare the filter label read from the file with the one in the
796 data ID.
798 Parameters
799 ----------
800 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
801 Filter label read from the file, if there was one.
802 should_be_standardized : `bool`, optional
803 If `True`, expect ``file_filter_label`` to be consistent with the
804 data ID and warn only if it is not. If `False`, expect it to be
805 inconsistent and warn only if the data ID is incomplete and hence
806 the `FilterLabel` cannot be fixed. If `None` (default) guess
807 whether the file should be standardized by looking at the
808 serialization version number in file, which requires this method to
809 have been run after `readFull` or `readComponent`.
811 Returns
812 -------
813 filter_label : `lsst.afw.image.FilterLabel` or `None`
814 The preferred filter label; may be the given one or one built from
815 the data ID. `None` is returned if there should never be any
816 filters associated with this dataset type.
818 Notes
819 -----
820 Most test coverage for this method is in ci_hsc_gen3, where we have
821 much easier access to test data that exhibits the problems it attempts
822 to solve.
823 """
824 # Remember filter data ID keys that weren't in this particular data ID,
825 # so we can warn about them later.
826 missing = []
827 band = None
828 physical_filter = None
829 if "band" in self.data_id.dimensions.names:
830 band = self.data_id.get("band")
831 # band isn't in the data ID; is that just because this data ID
832 # hasn't been filled in with everything the Registry knows, or
833 # because this dataset is never associated with a band?
834 if band is None and not self.data_id.hasFull() and "band" in self.data_id.dimensions.implied:
835 missing.append("band")
836 if "physical_filter" in self.data_id.dimensions.names:
837 physical_filter = self.data_id.get("physical_filter")
838 # Same check as above for band, but for physical_filter.
839 if (
840 physical_filter is None
841 and not self.data_id.hasFull()
842 and "physical_filter" in self.data_id.dimensions.implied
843 ):
844 missing.append("physical_filter")
845 if should_be_standardized is None:
846 version = self.reader.readSerializationVersion()
847 should_be_standardized = version >= 2
848 if missing:
849 # Data ID identifies a filter but the actual filter label values
850 # haven't been fetched from the database; we have no choice but
851 # to use the one in the file.
852 # Warn if that's more likely than not to be bad, because the file
853 # predates filter standardization.
854 if not should_be_standardized:
855 warnings.warn(
856 f"Data ID {self.data_id} is missing (implied) value(s) for {missing}; "
857 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
858 "Call Registry.expandDataId before Butler.get to avoid this.",
859 # Report the warning from outside of middleware or the
860 # relevant runQuantum method.
861 stacklevel=find_outside_stacklevel(
862 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
863 ),
864 )
865 return file_filter_label
866 if band is None and physical_filter is None:
867 data_id_filter_label = None
868 else:
869 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
870 if data_id_filter_label != file_filter_label and should_be_standardized:
871 # File was written after FilterLabel and standardization, but its
872 # FilterLabel doesn't agree with the data ID: this indicates a bug
873 # in whatever code produced the Exposure (though it may be one that
874 # has been fixed since the file was written).
875 warnings.warn(
876 f"Reading {self.file_descriptor.location} with data ID {self.data_id}: "
877 f"filter label mismatch (file is {file_filter_label}, data ID is "
878 f"{data_id_filter_label}). This is probably a bug in the code that produced it.",
879 stacklevel=find_outside_stacklevel(
880 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
881 ),
882 )
883 return data_id_filter_label