Coverage for python / lsst / obs / base / formatters / fitsExposure.py: 17%
338 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-31 01:09 -0800
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsMaskFormatter",
26 "FitsMaskedImageFormatter",
27 "standardizeAmplifierParameters",
28)
30import hashlib
31import json
32import logging
33import threading
34import uuid
35import warnings
36from abc import abstractmethod
37from collections.abc import Mapping, Set
38from io import BytesIO
39from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol
41import astropy.io.fits
42import numpy as np
44import lsst.geom
45from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
46from lsst.afw.fits import CompressionOptions, MemFileManager
47from lsst.afw.geom.wcsUtils import getImageXY0FromMetadata
48from lsst.afw.image import (
49 ExposureFitsReader,
50 ExposureInfo,
51 FilterLabel,
52 ImageFitsReader,
53 MaskedImageFitsReader,
54 MaskFitsReader,
55)
57# Needed for ApCorrMap to resolve properly
58from lsst.afw.math import BoundedField # noqa: F401
59from lsst.daf.base import PropertyList
60from lsst.daf.butler import DatasetProvenance, FormatterV2
61from lsst.resources import ResourcePath
62from lsst.utils.classes import cached_getter
63from lsst.utils.introspection import find_outside_stacklevel
65from ..utils import add_provenance_to_fits_header
67if TYPE_CHECKING:
68 import lsst.afw.cameraGeom
71_LOG = logging.getLogger(__name__)
73_ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ = False
74"""If True, the astropy code will always be used to read component and cutouts
75even if the file is local, the cutout is too large, or the dataset type is
76wrong. This should mostly be used for testing.
77"""
80class _ReaderClassLike(Protocol):
81 def __init__(self, path: str) -> None: ...
82 def readBBox(self) -> lsst.geom.Box2I: ...
83 def read(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
84 def readImage(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
85 def readMask(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
86 def readVariance(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
87 def readDetector(self) -> lsst.afw.cameraGeom.Detector: ...
88 def readComponent(self, component: str) -> Any: ...
89 def readMetadata(self) -> PropertyList: ...
90 def readSerializationVersion(self) -> int: ...
93class FitsImageFormatterBase(FormatterV2):
94 """Base class formatter for image-like storage classes stored via FITS.
96 Notes
97 -----
98 This class makes no assumptions about how many HDUs are used to represent
99 the image on disk, and includes no support for writing. It's really just a
100 collection of miscellaneous boilerplate common to all FITS image
101 formatters.
103 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
104 (even if just to disable them by raising an exception).
105 """
107 can_read_from_local_file = True
108 default_extension = ".fits"
109 supported_extensions: ClassVar[Set[str]] = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
111 unsupported_parameters: ClassVar[Set[str]] = frozenset()
112 """Support all parameters."""
114 _reader = None
115 _reader_path: str | None = None
117 ReaderClass: type[_ReaderClassLike] # must be set by concrete subclasses
119 @property
120 def reader(self) -> _ReaderClassLike:
121 """The reader object that backs this formatter's read operations.
123 This is computed on first use and then cached. It should never be
124 accessed when writing. Currently assumes a local file.
125 """
126 if self._reader is None:
127 if self._reader_path is None:
128 raise RuntimeError("Internal error in formatter; failing to set path.")
129 self._reader = self.ReaderClass(self._reader_path)
130 return self._reader
132 @property
133 @cached_getter
134 def checked_parameters(self) -> dict[str, Any]:
135 """The parameters passed by the butler user, after checking them
136 against the storage class and transforming `None` into an empty `dict`
137 (`dict`).
139 This is computed on first use and then cached. It should never be
140 accessed when writing. Subclasses that need additional checking should
141 delegate to `super` and then check the result before returning it.
142 """
143 parameters = self.file_descriptor.parameters
144 if parameters is None:
145 parameters = {}
146 self.file_descriptor.storageClass.validateParameters(parameters)
147 return parameters
149 @property
150 def storageClass_dtype(self) -> np.dtype | None:
151 """The numpy data type associated with the storage class."""
152 dtype: np.dtype | None = None
153 try:
154 # lsst.afw.image.Exposure is generic base class and does not have
155 # the dtype attribute.
156 dtype = np.dtype(self.file_descriptor.storageClass.pytype.dtype) # type: ignore[attr-defined]
157 except AttributeError:
158 pass
159 return dtype
161 def read_from_local_file(self, path: str, component: str | None = None, expected_size: int = -1) -> Any:
162 # Docstring inherited.
163 # The methods doing the reading all currently assume local file
164 # and assume that the file descriptor refers to a local file.
165 # With FormatterV2 that file descriptor does not refer to a local
166 # file.
167 self._reader_path = path
168 self._reader = None # Ensure the reader class is reset.
169 try:
170 if component is not None:
171 in_memory_dataset = self.readComponent(component)
172 else:
173 in_memory_dataset = self.readFull()
174 finally:
175 self._reader = None # Release the file handle.
176 return in_memory_dataset
178 @abstractmethod
179 def readComponent(self, component: str) -> Any:
180 """Read a component dataset.
182 Parameters
183 ----------
184 component : `str`, optional
185 Component to read from the file.
187 Returns
188 -------
189 obj : component-dependent
190 In-memory component object.
192 Raises
193 ------
194 KeyError
195 Raised if the requested component cannot be handled.
196 """
197 raise NotImplementedError()
199 @abstractmethod
200 def readFull(self) -> Any:
201 """Read the full dataset (while still accounting for parameters).
203 Returns
204 -------
205 obj : component-dependent
206 In-memory component object.
208 """
209 raise NotImplementedError()
212class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
213 """Base class formatter for image-like storage classes stored via FITS
214 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
216 Notes
217 -----
218 This class includes no support for writing.
220 Concrete subclasses must provide at least the `ReaderClass` attribute
221 and a `write` implementation (even just to disable writing by raising).
223 The provided implementation of `readComponent` handles only the 'bbox',
224 'dimensions', and 'xy0' components common to all image-like storage
225 classes. Subclasses with additional components should handle them first,
226 then delegate to ``super()`` for these (or, if necessary, delegate first
227 and catch `KeyError`).
229 The provided implementation of `readFull` handles only parameters that
230 can be forwarded directly to the reader class (usually ``bbox`` and
231 ``origin``). Concrete subclasses that need to handle additional parameters
232 should generally reimplement without delegating (the implementation is
233 trivial).
234 """
237class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
238 """Base class interface for image-like storage stored via FITS,
239 written using LSST code.
241 Notes
242 -----
243 Concrete subclasses must provide at least the `ReaderClass` attribute.
245 The provided implementation of `readComponent` handles only the 'bbox',
246 'dimensions', and 'xy0' components common to all image-like storage
247 classes. Subclasses with additional components should handle them first,
248 then delegate to ``super()`` for these (or, if necessary, delegate first
249 and catch `KeyError`).
251 The provided implementation of `readFull` handles only parameters that
252 can be forwarded directly to the reader class (usually ``bbox`` and
253 ``origin``). Concrete subclasses that need to handle additional parameters
254 should generally reimplement without delegating (the implementation is
255 trivial).
257 This Formatter supports write recipes, and assumes its in-memory type has
258 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
260 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
261 define ``image``, ``mask`` and ``variance`` entries, each of which may
262 contain entries supported by
263 `lsst.afw.fits.CompressionOptions.from_mapping` (``null`` disables
264 compression).
266 A very simple example YAML recipe (for the ``Exposure`` specialization):
268 .. code-block:: yaml
270 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
271 default:
272 image: &default
273 algorithm: GZIP_2
274 mask: *default
275 variance: *default
277 """
279 supported_write_parameters = frozenset({"recipe"})
281 def readComponent(self, component: str) -> Any:
282 # Docstring inherited.
283 if component in ("bbox", "dimensions", "xy0"):
284 bbox = self.reader.readBBox()
285 if component == "dimensions":
286 return bbox.getDimensions()
287 elif component == "xy0":
288 return bbox.getMin()
289 else:
290 return bbox
291 else:
292 raise KeyError(f"Unknown component requested: {component}")
294 def readFull(self) -> Any:
295 # Docstring inherited.
296 return self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
298 def write_local_file(self, in_memory_dataset: Any, uri: ResourcePath) -> None:
299 # check to see if we have a recipe requested
300 recipeName = self.write_parameters.get("recipe")
301 recipe = self.get_image_compression_settings(recipeName)
302 if recipe:
303 in_memory_dataset.writeFitsWithOptions(uri.ospath, options=recipe)
304 else:
305 in_memory_dataset.writeFits(uri.ospath)
307 def get_image_compression_settings(self, recipeName: str | None) -> dict:
308 """Retrieve the relevant compression settings for this recipe.
310 Parameters
311 ----------
312 recipeName : `str` or `None`
313 Label associated with the collection of compression parameters
314 to select.
316 Returns
317 -------
318 settings : `dict`
319 The selected settings.
320 """
321 # if no recipe has been provided and there is no default
322 # return immediately
323 if not recipeName:
324 if "default" not in self.write_recipes:
325 return {}
326 recipeName = "default"
328 if recipeName not in self.write_recipes:
329 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
331 recipe = self.write_recipes[recipeName]
332 if recipe is None:
333 return {}
334 seed: int | None = None
335 for plane in ("image", "mask", "variance"):
336 if plane in recipe and (quantization := recipe[plane].get("quantization")) is not None:
337 if quantization.get("seed", 0) == 0:
338 if seed is None:
339 # Set the seed based on data ID. We can't just use
340 # 'hash', since like 'set' that's not deterministic.
341 # And we can't rely on a DimensionPacker because those
342 # are only defined for certain combinations of
343 # dimensions. Doing an MD5 of the JSON feels like
344 # overkill but I don't really see anything much
345 # simpler.
346 hash_bytes = hashlib.md5(
347 json.dumps(list(self.data_id.required_values)).encode(),
348 usedforsecurity=False,
349 ).digest()
350 # And it *really* feels like overkill when we squash
351 # that into the [1, 10000] range allowed by FITS.
352 seed = 1 + int.from_bytes(hash_bytes) % 9999
353 _LOG.debug(
354 "Setting compression quantization seed for %s %s %s to %s.",
355 self.data_id,
356 self.dataset_ref.datasetType.name,
357 plane,
358 seed,
359 )
360 quantization["seed"] = seed
361 else:
362 _LOG.warning(
363 "Compression quantization seed for %s %s %s was set explicitly to %s.",
364 self.dataset_ref.datasetType.name,
365 self.data_id,
366 plane,
367 quantization["seed"],
368 )
369 else:
370 _LOG.debug(
371 "No quantization found for %s %s %s.",
372 self.dataset_ref.datasetType.name,
373 self.data_id,
374 plane,
375 )
376 return recipe
378 @classmethod
379 def validate_write_recipes(cls, recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None:
380 """Validate supplied recipes for this formatter.
382 The recipes are supplemented with default values where appropriate.
384 Parameters
385 ----------
386 recipes : `dict` or `None`
387 Recipes to validate. Can be empty dict or `None`.
389 Returns
390 -------
391 validated : `dict`
392 Validated recipes. Returns what was given if there are no
393 recipes listed.
395 Raises
396 ------
397 RuntimeError
398 Raised if validation fails.
399 """
400 if not recipes:
401 # We can not insist on recipes being specified.
402 return recipes
404 validated: dict[str, Any] = {}
405 for name, recipe in recipes.items():
406 if recipe is not None:
407 validated[name] = {}
408 for plane in ["image", "mask", "variance"]:
409 try:
410 options = CompressionOptions.from_mapping(recipe[plane])
411 except Exception as err:
412 err.add_note(f"Validating write recipe {name!r} ({plane!r} section).")
413 raise
414 validated[name][plane] = options.to_dict()
415 else:
416 validated[name] = None
417 return validated
420class FitsImageFormatter(StandardFitsImageFormatterBase):
421 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
422 from/to FITS.
423 """
425 ReaderClass = ImageFitsReader
428class FitsMaskFormatter(StandardFitsImageFormatterBase):
429 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
430 from/to FITS.
431 """
433 ReaderClass = MaskFitsReader
436class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
437 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
438 from/to FITS.
439 """
441 ReaderClass = MaskedImageFitsReader
443 def readComponent(self, component: str) -> Any:
444 # Docstring inherited.
445 if component == "image":
446 return self.reader.readImage(**self.checked_parameters, dtype=self.storageClass_dtype)
447 elif component == "mask":
448 return self.reader.readMask(**self.checked_parameters)
449 elif component == "variance":
450 return self.reader.readVariance(**self.checked_parameters, dtype=self.storageClass_dtype)
451 else:
452 # Delegate to base for bbox, dimensions, xy0.
453 return super().readComponent(component)
456def standardizeAmplifierParameters(
457 parameters: dict[str, Any], on_disk_detector: lsst.afw.cameraGeom.Detector | None
458) -> tuple[lsst.afw.cameraGeom.Amplifier, lsst.afw.cameraGeom.Detector, bool]:
459 """Preprocess the Exposure storage class's "amp" and "detector" parameters.
461 This checks the given objects for consistency with the on-disk geometry and
462 converts amplifier IDs/names to Amplifier instances.
464 Parameters
465 ----------
466 parameters : `dict`
467 Dictionary of parameters passed to formatter. See the Exposure storage
468 class definition in daf_butler for allowed keys and values.
469 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
470 Detector that represents the on-disk image being loaded, or `None` if
471 this is unknown (and hence the user must provide one in
472 ``parameters`` if "amp" is in ``parameters``).
474 Returns
475 -------
476 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
477 An amplifier object that defines a subimage to load, or `None` if there
478 was no "amp" parameter.
479 detector : `lsst.afw.cameraGeom.Detector` or `None`
480 A detector object whose amplifiers are in the same s/orientation
481 state as the on-disk image. If there is no "amp" parameter,
482 ``on_disk_detector`` is simply passed through.
483 regions_differ : `bool`
484 `True` if the on-disk detector and the detector given in the parameters
485 had different bounding boxes for one or more regions. This can happen
486 if the true overscan region sizes can only be determined when the image
487 is actually read, but otherwise it should be considered user error.
488 """
489 if (amplifier := parameters.get("amp")) is None:
490 return None, on_disk_detector, False
491 if "bbox" in parameters or "origin" in parameters:
492 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
493 if isinstance(amplifier, int | str):
494 amp_key = amplifier
495 target_amplifier = None
496 else:
497 amp_key = amplifier.getName()
498 target_amplifier = amplifier
499 if (detector := parameters.get("detector")) is not None:
500 if on_disk_detector is not None:
501 # User passed a detector and we also found one on disk. Check them
502 # for consistency. Note that we are checking the amps we'd get
503 # from the two detectors against each other, not the amplifier we
504 # got directly from the user, as the latter is allowed to differ in
505 # assembly/orientation state.
506 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
507 if comparison & comparison.ASSEMBLY_DIFFERS:
508 raise ValueError(
509 "The given 'detector' has a different assembly state and/or orientation from "
510 f"the on-disk one for amp {amp_key}."
511 )
512 else:
513 if on_disk_detector is None:
514 raise ValueError(
515 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
516 "Please provide either a 'detector' parameter or an Amplifier instance in the "
517 "'amp' parameter."
518 )
519 comparison = AmplifierGeometryComparison.EQUAL
520 detector = on_disk_detector
521 if target_amplifier is None:
522 target_amplifier = detector[amp_key]
523 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
526class _ComponentCache(NamedTuple):
527 id_: uuid.UUID | None = None
528 reader: ExposureFitsReader | None = None
529 bbox: lsst.geom.Box2I | None = None
530 mem: MemFileManager | None = None
533class FitsExposureFormatter(FitsMaskedImageFormatter):
534 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
535 from/to FITS.
537 Notes
538 -----
539 This class inherits from `FitsMaskedImageFormatter` even though
540 `lsst.afw.image.Exposure` doesn't inherit from
541 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
542 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
543 should be replaced with e.g. both calling a free function if that slight
544 type covariance violation ever becomes a practical problem.
545 """
547 can_read_from_uri = True
548 ReaderClass = ExposureFitsReader
549 # TODO: Remove MemFileManager from cache when DM-49640 is fixed.
550 _lock = threading.Lock()
551 _cached_fits: _ComponentCache = _ComponentCache()
553 def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
554 # For now only support small non-pixel components. In future
555 # could work with cutouts.
556 self._reader = None # Guarantee things are reset.
558 # Full read, always use local file read.
559 if not component and not self.checked_parameters:
560 return NotImplemented
562 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and uri.isLocal:
563 # For a local URI allow afw to read it directly.
564 return NotImplemented
565 pixel_components = ("mask", "image", "variance")
567 if component in pixel_components:
568 # For pixel access currently this can not be cached in memory
569 # and the performance gains are unclear. Assume local file
570 # read with file caching for now.
571 return NotImplemented
573 # With current file layouts the non-pixel extensions account for 1/3
574 # of the file size and it is more efficient to download the entire
575 # file.
576 if not (
577 _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ
578 or self._dataset_ref.dataId.mapping.keys().isdisjoint({"tract", "patch"})
579 ):
580 return NotImplemented
582 # Cutouts can be optimized. For now only use this optimization
583 # if bbox is the only parameter and the number of pixels in the
584 # bounding box is reasonable.
585 bbox = None
586 origin = lsst.afw.image.PARENT
587 if not component:
588 # Try to support PARENT and LOCAL origin but if there are any
589 # other parameters do not attempt a cutout.
590 if self.checked_parameters.keys() - {"bbox", "origin"}:
591 return NotImplemented
592 bbox = self.checked_parameters["bbox"]
593 origin = self.checked_parameters.get("origin", lsst.afw.image.PARENT)
594 # For larger cutouts use the full file.
595 max_cutout_size = 500 * 500
596 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and bbox.width * bbox.height > max_cutout_size:
597 return NotImplemented
599 # We only cache component reads since those are small.
600 if component:
601 with self._lock:
602 cache = type(self)._cached_fits
603 if self.dataset_ref.id == cache.id_:
604 if component in {"xy0", "dimensions", "bbox"} and cache.bbox is not None:
605 match component:
606 case "xy0":
607 return cache.bbox.getMin()
608 case "dimensions":
609 return cache.bbox.getDimensions()
610 case "bbox":
611 return cache.bbox
612 else:
613 self._reader = cache.reader
614 return self.readComponent(component)
616 try:
617 fs, fspath = uri.to_fsspec()
618 except Exception:
619 # fsspec cannot be initialized, fall back to downloading the file.
620 return NotImplemented
622 bbox_component = None
623 try:
624 hdul = []
625 with fs.open(fspath) as f, astropy.io.fits.open(f) as fits_obj:
626 # Read all non-pixel components and cache.
627 for hdu in fits_obj:
628 hdr = hdu.header
629 extname = hdr.get("EXTNAME")
630 # Older files have IMAGE in EXTNAME in PRIMARY so check
631 # for EXTEND=T.
632 extend = hdr.get("EXTEND")
633 if not extend and extname and extname.lower() in pixel_components:
634 # Calculate the dimensional components for later
635 # caching. Do not derive from cached FITS reader
636 # because they depend on the dimensionality of the
637 # pixel data and we do not want to cache the pixel
638 # data.
639 if bbox_component is None:
640 shape = hdu.shape
641 dimensions = lsst.geom.Extent2I(shape[1], shape[0])
643 # XY0 is defined in the A WCS.
644 pl = PropertyList()
645 pl.update(hdr)
646 xy0 = getImageXY0FromMetadata(pl, "A", strip=False)
648 # This is the PARENT bbox.
649 bbox_component = lsst.geom.Box2I(xy0, dimensions)
651 # Handle cutout request.
652 if bbox:
653 if origin == lsst.afw.image.PARENT:
654 full_bbox = bbox_component
655 else:
656 full_bbox = lsst.geom.Box2I(
657 lsst.geom.Point2I(0, 0), bbox_component.getDimensions
658 )
659 minX = bbox.getBeginX() - full_bbox.getBeginX()
660 maxX = bbox.getEndX() - full_bbox.getBeginX()
661 minY = bbox.getBeginY() - full_bbox.getBeginY()
662 maxY = bbox.getEndY() - full_bbox.getBeginY()
663 data = hdu.section[minY:maxY, minX:maxX]
665 # Must correct the header WCS to take into
666 # account the offset.
667 if (k := "CRPIX1") in hdr:
668 hdr[k] -= minX
669 if (k := "CRPIX2") in hdr:
670 hdr[k] -= minY
671 if (k := "LTV1") in hdr:
672 hdr[k] = -bbox.getBeginX()
673 if (k := "LTV2") in hdr:
674 hdr[k] = -bbox.getBeginY()
675 if (k := "CRVAL1A") in hdr:
676 hdr[k] = bbox.getBeginX()
677 if (k := "CRVAL2A") in hdr:
678 hdr[k] = bbox.getBeginY()
679 else:
680 data = np.zeros([1, 1], dtype=np.int32)
682 # Construct a new HDU and copy the header.
683 stripped_hdu = astropy.io.fits.ImageHDU(data=data, header=hdr)
684 hdul.append(stripped_hdu)
685 else:
686 hdul.append(hdu)
687 stripped_fits = astropy.io.fits.HDUList(hdus=hdul)
688 # Write the FITS file to in-memory FITS.
689 buffer = BytesIO()
690 stripped_fits.writeto(buffer)
691 except Exception as e:
692 # For some reason we can't open the remote file so fall back.
693 _LOG.debug(
694 "Attempted remote read of components but encountered an error. "
695 "Falling back to file download. Error: %s",
696 str(e),
697 )
698 return NotImplemented
700 # Pass the new FITS buffer to the reader class without going through
701 # a temporary file. We can assume this is relatively small for
702 # components.
703 fits_data = buffer.getvalue()
704 mem = MemFileManager(len(fits_data))
705 mem.setData(fits_data, len(fits_data))
706 self._reader = self.ReaderClass(mem)
708 if component:
709 with self._lock:
710 type(self)._cached_fits = _ComponentCache(
711 id_=self.dataset_ref.id,
712 reader=self._reader,
713 mem=mem,
714 bbox=bbox_component,
715 )
716 match component:
717 case "xy0":
718 if bbox_component is None: # For mypy.
719 return None
720 return bbox_component.getMin()
721 case "dimensions":
722 if bbox_component is None:
723 return None
724 return bbox_component.getDimensions()
725 case "bbox":
726 return bbox_component
727 case _:
728 return self.readComponent(component)
729 else:
730 # Must be a cutout. We have applied the bbox parameter so no
731 # parameters should be passed here.
732 cutout = self.reader.read(dtype=self.storageClass_dtype)
733 cutout.getInfo().setFilter(self._fixFilterLabels(cutout.getInfo().getFilter()))
734 return cutout
736 def add_provenance(
737 self, in_memory_dataset: Any, /, *, provenance: DatasetProvenance | None = None
738 ) -> Any:
739 # Add provenance via FITS headers.
740 add_provenance_to_fits_header(in_memory_dataset.metadata, self.dataset_ref, provenance)
741 return in_memory_dataset
743 def readComponent(self, component: str) -> Any:
744 # Docstring inherited.
745 # Generic components can be read via a string name; DM-27754 will make
746 # this mapping larger at the expense of the following one.
747 genericComponents = {
748 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
749 }
750 if (genericComponentName := genericComponents.get(component)) is not None:
751 return self.reader.readComponent(genericComponentName)
752 # Other components have hard-coded method names, but don't take
753 # parameters.
754 standardComponents = {
755 "id": "readExposureId",
756 "metadata": "readMetadata",
757 "wcs": "readWcs",
758 "coaddInputs": "readCoaddInputs",
759 "psf": "readPsf",
760 "photoCalib": "readPhotoCalib",
761 "filter": "readFilter",
762 "validPolygon": "readValidPolygon",
763 "apCorrMap": "readApCorrMap",
764 "visitInfo": "readVisitInfo",
765 "transmissionCurve": "readTransmissionCurve",
766 "detector": "readDetector",
767 "exposureInfo": "readExposureInfo",
768 }
769 if (methodName := standardComponents.get(component)) is not None:
770 result = getattr(self.reader, methodName)()
771 if component == "filter":
772 return self._fixFilterLabels(result)
773 return result
774 # Delegate to MaskedImage and ImageBase implementations for the rest.
775 return super().readComponent(component)
777 def readFull(self) -> Any:
778 # Docstring inherited.
779 amplifier, detector, _ = standardizeAmplifierParameters(
780 self.checked_parameters,
781 self.reader.readDetector(),
782 )
783 if amplifier is not None:
784 amplifier_isolator = AmplifierIsolator(
785 amplifier,
786 self.reader.readBBox(),
787 detector,
788 )
789 result = amplifier_isolator.transform_subimage(
790 self.reader.read(bbox=amplifier_isolator.subimage_bbox, dtype=self.storageClass_dtype)
791 )
792 result.setDetector(amplifier_isolator.make_detector())
793 else:
794 result = self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
795 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
796 return result
798 def _fixFilterLabels(
799 self, file_filter_label: lsst.afw.image.FilterLabel, should_be_standardized: bool | None = None
800 ) -> lsst.afw.image.FilterLabel:
801 """Compare the filter label read from the file with the one in the
802 data ID.
804 Parameters
805 ----------
806 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
807 Filter label read from the file, if there was one.
808 should_be_standardized : `bool`, optional
809 If `True`, expect ``file_filter_label`` to be consistent with the
810 data ID and warn only if it is not. If `False`, expect it to be
811 inconsistent and warn only if the data ID is incomplete and hence
812 the `FilterLabel` cannot be fixed. If `None` (default) guess
813 whether the file should be standardized by looking at the
814 serialization version number in file, which requires this method to
815 have been run after `readFull` or `readComponent`.
817 Returns
818 -------
819 filter_label : `lsst.afw.image.FilterLabel` or `None`
820 The preferred filter label; may be the given one or one built from
821 the data ID. `None` is returned if there should never be any
822 filters associated with this dataset type.
824 Notes
825 -----
826 Most test coverage for this method is in ci_hsc_gen3, where we have
827 much easier access to test data that exhibits the problems it attempts
828 to solve.
829 """
830 # Remember filter data ID keys that weren't in this particular data ID,
831 # so we can warn about them later.
832 missing = []
833 band = None
834 physical_filter = None
835 if "band" in self.data_id.dimensions.names:
836 band = self.data_id.get("band")
837 # band isn't in the data ID; is that just because this data ID
838 # hasn't been filled in with everything the Registry knows, or
839 # because this dataset is never associated with a band?
840 if band is None and not self.data_id.hasFull() and "band" in self.data_id.dimensions.implied:
841 missing.append("band")
842 if "physical_filter" in self.data_id.dimensions.names:
843 physical_filter = self.data_id.get("physical_filter")
844 # Same check as above for band, but for physical_filter.
845 if (
846 physical_filter is None
847 and not self.data_id.hasFull()
848 and "physical_filter" in self.data_id.dimensions.implied
849 ):
850 missing.append("physical_filter")
851 if should_be_standardized is None:
852 version = self.reader.readSerializationVersion()
853 should_be_standardized = version >= 2
854 if missing:
855 # Data ID identifies a filter but the actual filter label values
856 # haven't been fetched from the database; we have no choice but
857 # to use the one in the file.
858 # Warn if that's more likely than not to be bad, because the file
859 # predates filter standardization.
860 if not should_be_standardized:
861 warnings.warn(
862 f"Data ID {self.data_id} is missing (implied) value(s) for {missing}; "
863 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
864 "Call Registry.expandDataId before Butler.get to avoid this.",
865 # Report the warning from outside of middleware or the
866 # relevant runQuantum method.
867 stacklevel=find_outside_stacklevel(
868 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
869 ),
870 )
871 return file_filter_label
872 if band is None and physical_filter is None:
873 data_id_filter_label = None
874 else:
875 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
876 if data_id_filter_label != file_filter_label and should_be_standardized:
877 # File was written after FilterLabel and standardization, but its
878 # FilterLabel doesn't agree with the data ID: this indicates a bug
879 # in whatever code produced the Exposure (though it may be one that
880 # has been fixed since the file was written).
881 warnings.warn(
882 f"Reading {self.file_descriptor.location} with data ID {self.data_id}: "
883 f"filter label mismatch (file is {file_filter_label}, data ID is "
884 f"{data_id_filter_label}). This is probably a bug in the code that produced it.",
885 stacklevel=find_outside_stacklevel(
886 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
887 ),
888 )
889 return data_id_filter_label