Coverage for python / lsst / obs / base / formatters / fitsExposure.py: 17%
338 statements
« prev ^ index » next coverage.py v7.13.2, created at 2026-02-03 01:34 -0800
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = (
23 "FitsExposureFormatter",
24 "FitsImageFormatter",
25 "FitsImageFormatterBase",
26 "FitsMaskFormatter",
27 "FitsMaskedImageFormatter",
28 "standardizeAmplifierParameters",
29)
31import hashlib
32import json
33import logging
34import threading
35import uuid
36import warnings
37from abc import abstractmethod
38from collections.abc import Mapping, Set
39from io import BytesIO
40from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol
42import astropy.io.fits
43import numpy as np
45import lsst.geom
46from lsst.afw.cameraGeom import AmplifierGeometryComparison, AmplifierIsolator
47from lsst.afw.fits import CompressionOptions, MemFileManager
48from lsst.afw.geom.wcsUtils import getImageXY0FromMetadata
49from lsst.afw.image import (
50 ExposureFitsReader,
51 ExposureInfo,
52 FilterLabel,
53 ImageFitsReader,
54 MaskedImageFitsReader,
55 MaskFitsReader,
56)
58# Needed for ApCorrMap to resolve properly
59from lsst.afw.math import BoundedField # noqa: F401
60from lsst.daf.base import PropertyList
61from lsst.daf.butler import DatasetProvenance, FormatterV2
62from lsst.resources import ResourcePath
63from lsst.utils.classes import cached_getter
64from lsst.utils.introspection import find_outside_stacklevel
66from ..utils import add_provenance_to_fits_header
68if TYPE_CHECKING:
69 import lsst.afw.cameraGeom
72_LOG = logging.getLogger(__name__)
74_ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ = False
75"""If True, the astropy code will always be used to read component and cutouts
76even if the file is local, the cutout is too large, or the dataset type is
77wrong. This should mostly be used for testing.
78"""
81class _ReaderClassLike(Protocol):
82 def __init__(self, path: str) -> None: ...
83 def readBBox(self) -> lsst.geom.Box2I: ...
84 def read(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
85 def readImage(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
86 def readMask(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
87 def readVariance(self, bbox: lsst.geom.Box2I = lsst.geom.Box2I(), dtype: Any = None) -> Any: ...
88 def readDetector(self) -> lsst.afw.cameraGeom.Detector: ...
89 def readComponent(self, component: str) -> Any: ...
90 def readMetadata(self) -> PropertyList: ...
91 def readSerializationVersion(self) -> int: ...
94class FitsImageFormatterBase(FormatterV2):
95 """Base class formatter for image-like storage classes stored via FITS.
97 Notes
98 -----
99 This class makes no assumptions about how many HDUs are used to represent
100 the image on disk, and includes no support for writing. It's really just a
101 collection of miscellaneous boilerplate common to all FITS image
102 formatters.
104 Concrete subclasses must implement `readComponent`, `readFull`, and `write`
105 (even if just to disable them by raising an exception).
106 """
108 can_read_from_local_file = True
109 default_extension = ".fits"
110 supported_extensions: ClassVar[Set[str]] = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
112 unsupported_parameters: ClassVar[Set[str]] = frozenset()
113 """Support all parameters."""
115 _reader = None
116 _reader_path: str | None = None
118 ReaderClass: type[_ReaderClassLike] # must be set by concrete subclasses
120 @property
121 def reader(self) -> _ReaderClassLike:
122 """The reader object that backs this formatter's read operations.
124 This is computed on first use and then cached. It should never be
125 accessed when writing. Currently assumes a local file.
126 """
127 if self._reader is None:
128 if self._reader_path is None:
129 raise RuntimeError("Internal error in formatter; failing to set path.")
130 self._reader = self.ReaderClass(self._reader_path)
131 return self._reader
133 @property
134 @cached_getter
135 def checked_parameters(self) -> dict[str, Any]:
136 """The parameters passed by the butler user, after checking them
137 against the storage class and transforming `None` into an empty `dict`
138 (`dict`).
140 This is computed on first use and then cached. It should never be
141 accessed when writing. Subclasses that need additional checking should
142 delegate to `super` and then check the result before returning it.
143 """
144 parameters = self.file_descriptor.parameters
145 if parameters is None:
146 parameters = {}
147 self.file_descriptor.storageClass.validateParameters(parameters)
148 return parameters
150 @property
151 def storageClass_dtype(self) -> np.dtype | None:
152 """The numpy data type associated with the storage class."""
153 dtype: np.dtype | None = None
154 try:
155 # lsst.afw.image.Exposure is generic base class and does not have
156 # the dtype attribute.
157 dtype = np.dtype(self.file_descriptor.storageClass.pytype.dtype) # type: ignore[attr-defined]
158 except AttributeError:
159 pass
160 return dtype
162 def read_from_local_file(self, path: str, component: str | None = None, expected_size: int = -1) -> Any:
163 # Docstring inherited.
164 # The methods doing the reading all currently assume local file
165 # and assume that the file descriptor refers to a local file.
166 # With FormatterV2 that file descriptor does not refer to a local
167 # file.
168 self._reader_path = path
169 self._reader = None # Ensure the reader class is reset.
170 try:
171 if component is not None:
172 in_memory_dataset = self.readComponent(component)
173 else:
174 in_memory_dataset = self.readFull()
175 finally:
176 self._reader = None # Release the file handle.
177 return in_memory_dataset
179 @abstractmethod
180 def readComponent(self, component: str) -> Any:
181 """Read a component dataset.
183 Parameters
184 ----------
185 component : `str`, optional
186 Component to read from the file.
188 Returns
189 -------
190 obj : component-dependent
191 In-memory component object.
193 Raises
194 ------
195 KeyError
196 Raised if the requested component cannot be handled.
197 """
198 raise NotImplementedError()
200 @abstractmethod
201 def readFull(self) -> Any:
202 """Read the full dataset (while still accounting for parameters).
204 Returns
205 -------
206 obj : component-dependent
207 In-memory component object.
209 """
210 raise NotImplementedError()
213class ReaderFitsImageFormatterBase(FitsImageFormatterBase):
214 """Base class formatter for image-like storage classes stored via FITS
215 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`.
217 Notes
218 -----
219 This class includes no support for writing.
221 Concrete subclasses must provide at least the `ReaderClass` attribute
222 and a `write` implementation (even just to disable writing by raising).
224 The provided implementation of `readComponent` handles only the 'bbox',
225 'dimensions', and 'xy0' components common to all image-like storage
226 classes. Subclasses with additional components should handle them first,
227 then delegate to ``super()`` for these (or, if necessary, delegate first
228 and catch `KeyError`).
230 The provided implementation of `readFull` handles only parameters that
231 can be forwarded directly to the reader class (usually ``bbox`` and
232 ``origin``). Concrete subclasses that need to handle additional parameters
233 should generally reimplement without delegating (the implementation is
234 trivial).
235 """
238class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase):
239 """Base class interface for image-like storage stored via FITS,
240 written using LSST code.
242 Notes
243 -----
244 Concrete subclasses must provide at least the `ReaderClass` attribute.
246 The provided implementation of `readComponent` handles only the 'bbox',
247 'dimensions', and 'xy0' components common to all image-like storage
248 classes. Subclasses with additional components should handle them first,
249 then delegate to ``super()`` for these (or, if necessary, delegate first
250 and catch `KeyError`).
252 The provided implementation of `readFull` handles only parameters that
253 can be forwarded directly to the reader class (usually ``bbox`` and
254 ``origin``). Concrete subclasses that need to handle additional parameters
255 should generally reimplement without delegating (the implementation is
256 trivial).
258 This Formatter supports write recipes, and assumes its in-memory type has
259 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods.
261 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should
262 define ``image``, ``mask`` and ``variance`` entries, each of which may
263 contain entries supported by
264 `lsst.afw.fits.CompressionOptions.from_mapping` (``null`` disables
265 compression).
267 A very simple example YAML recipe (for the ``Exposure`` specialization):
269 .. code-block:: yaml
271 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
272 default:
273 image: &default
274 algorithm: GZIP_2
275 mask: *default
276 variance: *default
278 """
280 supported_write_parameters = frozenset({"recipe"})
282 def readComponent(self, component: str) -> Any:
283 # Docstring inherited.
284 if component in ("bbox", "dimensions", "xy0"):
285 bbox = self.reader.readBBox()
286 if component == "dimensions":
287 return bbox.getDimensions()
288 elif component == "xy0":
289 return bbox.getMin()
290 else:
291 return bbox
292 else:
293 raise KeyError(f"Unknown component requested: {component}")
295 def readFull(self) -> Any:
296 # Docstring inherited.
297 return self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
299 def write_local_file(self, in_memory_dataset: Any, uri: ResourcePath) -> None:
300 # check to see if we have a recipe requested
301 recipeName = self.write_parameters.get("recipe")
302 recipe = self.get_image_compression_settings(recipeName)
303 if recipe:
304 in_memory_dataset.writeFitsWithOptions(uri.ospath, options=recipe)
305 else:
306 in_memory_dataset.writeFits(uri.ospath)
308 def get_image_compression_settings(self, recipeName: str | None) -> dict:
309 """Retrieve the relevant compression settings for this recipe.
311 Parameters
312 ----------
313 recipeName : `str` or `None`
314 Label associated with the collection of compression parameters
315 to select.
317 Returns
318 -------
319 settings : `dict`
320 The selected settings.
321 """
322 # if no recipe has been provided and there is no default
323 # return immediately
324 if not recipeName:
325 if "default" not in self.write_recipes:
326 return {}
327 recipeName = "default"
329 if recipeName not in self.write_recipes:
330 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
332 recipe = self.write_recipes[recipeName]
333 if recipe is None:
334 return {}
335 seed: int | None = None
336 for plane in ("image", "mask", "variance"):
337 if plane in recipe and (quantization := recipe[plane].get("quantization")) is not None:
338 if quantization.get("seed", 0) == 0:
339 if seed is None:
340 # Set the seed based on data ID. We can't just use
341 # 'hash', since like 'set' that's not deterministic.
342 # And we can't rely on a DimensionPacker because those
343 # are only defined for certain combinations of
344 # dimensions. Doing an MD5 of the JSON feels like
345 # overkill but I don't really see anything much
346 # simpler.
347 hash_bytes = hashlib.md5(
348 json.dumps(list(self.data_id.required_values)).encode(),
349 usedforsecurity=False,
350 ).digest()
351 # And it *really* feels like overkill when we squash
352 # that into the [1, 10000] range allowed by FITS.
353 seed = 1 + int.from_bytes(hash_bytes) % 9999
354 _LOG.debug(
355 "Setting compression quantization seed for %s %s %s to %s.",
356 self.data_id,
357 self.dataset_ref.datasetType.name,
358 plane,
359 seed,
360 )
361 quantization["seed"] = seed
362 else:
363 _LOG.warning(
364 "Compression quantization seed for %s %s %s was set explicitly to %s.",
365 self.dataset_ref.datasetType.name,
366 self.data_id,
367 plane,
368 quantization["seed"],
369 )
370 else:
371 _LOG.debug(
372 "No quantization found for %s %s %s.",
373 self.dataset_ref.datasetType.name,
374 self.data_id,
375 plane,
376 )
377 return recipe
379 @classmethod
380 def validate_write_recipes(cls, recipes: Mapping[str, Any] | None) -> Mapping[str, Any] | None:
381 """Validate supplied recipes for this formatter.
383 The recipes are supplemented with default values where appropriate.
385 Parameters
386 ----------
387 recipes : `dict` or `None`
388 Recipes to validate. Can be empty dict or `None`.
390 Returns
391 -------
392 validated : `dict`
393 Validated recipes. Returns what was given if there are no
394 recipes listed.
396 Raises
397 ------
398 RuntimeError
399 Raised if validation fails.
400 """
401 if not recipes:
402 # We can not insist on recipes being specified.
403 return recipes
405 validated: dict[str, Any] = {}
406 for name, recipe in recipes.items():
407 if recipe is not None:
408 validated[name] = {}
409 for plane in ["image", "mask", "variance"]:
410 try:
411 options = CompressionOptions.from_mapping(recipe[plane])
412 except Exception as err:
413 err.add_note(f"Validating write recipe {name!r} ({plane!r} section).")
414 raise
415 validated[name][plane] = options.to_dict()
416 else:
417 validated[name] = None
418 return validated
421class FitsImageFormatter(StandardFitsImageFormatterBase):
422 """Concrete formatter for reading/writing `~lsst.afw.image.Image`
423 from/to FITS.
424 """
426 ReaderClass = ImageFitsReader
429class FitsMaskFormatter(StandardFitsImageFormatterBase):
430 """Concrete formatter for reading/writing `~lsst.afw.image.Mask`
431 from/to FITS.
432 """
434 ReaderClass = MaskFitsReader
437class FitsMaskedImageFormatter(StandardFitsImageFormatterBase):
438 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage`
439 from/to FITS.
440 """
442 ReaderClass = MaskedImageFitsReader
444 def readComponent(self, component: str) -> Any:
445 # Docstring inherited.
446 if component == "image":
447 return self.reader.readImage(**self.checked_parameters, dtype=self.storageClass_dtype)
448 elif component == "mask":
449 return self.reader.readMask(**self.checked_parameters)
450 elif component == "variance":
451 return self.reader.readVariance(**self.checked_parameters, dtype=self.storageClass_dtype)
452 else:
453 # Delegate to base for bbox, dimensions, xy0.
454 return super().readComponent(component)
457def standardizeAmplifierParameters(
458 parameters: dict[str, Any], on_disk_detector: lsst.afw.cameraGeom.Detector | None
459) -> tuple[lsst.afw.cameraGeom.Amplifier, lsst.afw.cameraGeom.Detector, bool]:
460 """Preprocess the Exposure storage class's "amp" and "detector" parameters.
462 This checks the given objects for consistency with the on-disk geometry and
463 converts amplifier IDs/names to Amplifier instances.
465 Parameters
466 ----------
467 parameters : `dict`
468 Dictionary of parameters passed to formatter. See the Exposure storage
469 class definition in daf_butler for allowed keys and values.
470 on_disk_detector : `lsst.afw.cameraGeom.Detector` or `None`
471 Detector that represents the on-disk image being loaded, or `None` if
472 this is unknown (and hence the user must provide one in
473 ``parameters`` if "amp" is in ``parameters``).
475 Returns
476 -------
477 amplifier : `lsst.afw.cameraGeom.Amplifier` or `None`
478 An amplifier object that defines a subimage to load, or `None` if there
479 was no "amp" parameter.
480 detector : `lsst.afw.cameraGeom.Detector` or `None`
481 A detector object whose amplifiers are in the same s/orientation
482 state as the on-disk image. If there is no "amp" parameter,
483 ``on_disk_detector`` is simply passed through.
484 regions_differ : `bool`
485 `True` if the on-disk detector and the detector given in the parameters
486 had different bounding boxes for one or more regions. This can happen
487 if the true overscan region sizes can only be determined when the image
488 is actually read, but otherwise it should be considered user error.
489 """
490 if (amplifier := parameters.get("amp")) is None:
491 return None, on_disk_detector, False
492 if "bbox" in parameters or "origin" in parameters:
493 raise ValueError("Cannot pass 'amp' with 'bbox' or 'origin'.")
494 if isinstance(amplifier, int | str):
495 amp_key = amplifier
496 target_amplifier = None
497 else:
498 amp_key = amplifier.getName()
499 target_amplifier = amplifier
500 if (detector := parameters.get("detector")) is not None:
501 if on_disk_detector is not None:
502 # User passed a detector and we also found one on disk. Check them
503 # for consistency. Note that we are checking the amps we'd get
504 # from the two detectors against each other, not the amplifier we
505 # got directly from the user, as the latter is allowed to differ in
506 # assembly/orientation state.
507 comparison = on_disk_detector[amp_key].compareGeometry(detector[amp_key])
508 if comparison & comparison.ASSEMBLY_DIFFERS:
509 raise ValueError(
510 "The given 'detector' has a different assembly state and/or orientation from "
511 f"the on-disk one for amp {amp_key}."
512 )
513 else:
514 if on_disk_detector is None:
515 raise ValueError(
516 f"No on-disk detector and no detector given; cannot load amplifier from key {amp_key}. "
517 "Please provide either a 'detector' parameter or an Amplifier instance in the "
518 "'amp' parameter."
519 )
520 comparison = AmplifierGeometryComparison.EQUAL
521 detector = on_disk_detector
522 if target_amplifier is None:
523 target_amplifier = detector[amp_key]
524 return target_amplifier, detector, comparison & comparison.REGIONS_DIFFER
527class _ComponentCache(NamedTuple):
528 id_: uuid.UUID | None = None
529 reader: ExposureFitsReader | None = None
530 bbox: lsst.geom.Box2I | None = None
531 mem: MemFileManager | None = None
534class FitsExposureFormatter(FitsMaskedImageFormatter):
535 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure`
536 from/to FITS.
538 Notes
539 -----
540 This class inherits from `FitsMaskedImageFormatter` even though
541 `lsst.afw.image.Exposure` doesn't inherit from
542 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to
543 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and
544 should be replaced with e.g. both calling a free function if that slight
545 type covariance violation ever becomes a practical problem.
546 """
548 can_read_from_uri = True
549 ReaderClass = ExposureFitsReader
550 # TODO: Remove MemFileManager from cache when DM-49640 is fixed.
551 _lock = threading.Lock()
552 _cached_fits: _ComponentCache = _ComponentCache()
554 def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
555 # For now only support small non-pixel components. In future
556 # could work with cutouts.
557 self._reader = None # Guarantee things are reset.
559 # Full read, always use local file read.
560 if not component and not self.checked_parameters:
561 return NotImplemented
563 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and uri.isLocal:
564 # For a local URI allow afw to read it directly.
565 return NotImplemented
566 pixel_components = ("mask", "image", "variance")
568 if component in pixel_components:
569 # For pixel access currently this can not be cached in memory
570 # and the performance gains are unclear. Assume local file
571 # read with file caching for now.
572 return NotImplemented
574 # With current file layouts the non-pixel extensions account for 1/3
575 # of the file size and it is more efficient to download the entire
576 # file.
577 if not (
578 _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ
579 or self._dataset_ref.dataId.mapping.keys().isdisjoint({"tract", "patch"})
580 ):
581 return NotImplemented
583 # Cutouts can be optimized. For now only use this optimization
584 # if bbox is the only parameter and the number of pixels in the
585 # bounding box is reasonable.
586 bbox = None
587 origin = lsst.afw.image.PARENT
588 if not component:
589 # Try to support PARENT and LOCAL origin but if there are any
590 # other parameters do not attempt a cutout.
591 if self.checked_parameters.keys() - {"bbox", "origin"}:
592 return NotImplemented
593 bbox = self.checked_parameters["bbox"]
594 origin = self.checked_parameters.get("origin", lsst.afw.image.PARENT)
595 # For larger cutouts use the full file.
596 max_cutout_size = 500 * 500
597 if not _ALWAYS_USE_ASTROPY_FOR_COMPONENT_READ and bbox.width * bbox.height > max_cutout_size:
598 return NotImplemented
600 # We only cache component reads since those are small.
601 if component:
602 with self._lock:
603 cache = type(self)._cached_fits
604 if self.dataset_ref.id == cache.id_:
605 if component in {"xy0", "dimensions", "bbox"} and cache.bbox is not None:
606 match component:
607 case "xy0":
608 return cache.bbox.getMin()
609 case "dimensions":
610 return cache.bbox.getDimensions()
611 case "bbox":
612 return cache.bbox
613 else:
614 self._reader = cache.reader
615 return self.readComponent(component)
617 try:
618 fs, fspath = uri.to_fsspec()
619 except Exception:
620 # fsspec cannot be initialized, fall back to downloading the file.
621 return NotImplemented
623 bbox_component = None
624 try:
625 hdul = []
626 with fs.open(fspath) as f, astropy.io.fits.open(f) as fits_obj:
627 # Read all non-pixel components and cache.
628 for hdu in fits_obj:
629 hdr = hdu.header
630 extname = hdr.get("EXTNAME")
631 # Older files have IMAGE in EXTNAME in PRIMARY so check
632 # for EXTEND=T.
633 extend = hdr.get("EXTEND")
634 if not extend and extname and extname.lower() in pixel_components:
635 # Calculate the dimensional components for later
636 # caching. Do not derive from cached FITS reader
637 # because they depend on the dimensionality of the
638 # pixel data and we do not want to cache the pixel
639 # data.
640 if bbox_component is None:
641 shape = hdu.shape
642 dimensions = lsst.geom.Extent2I(shape[1], shape[0])
644 # XY0 is defined in the A WCS.
645 pl = PropertyList()
646 pl.update(hdr)
647 xy0 = getImageXY0FromMetadata(pl, "A", strip=False)
649 # This is the PARENT bbox.
650 bbox_component = lsst.geom.Box2I(xy0, dimensions)
652 # Handle cutout request.
653 if bbox:
654 if origin == lsst.afw.image.PARENT:
655 full_bbox = bbox_component
656 else:
657 full_bbox = lsst.geom.Box2I(
658 lsst.geom.Point2I(0, 0), bbox_component.getDimensions
659 )
660 minX = bbox.getBeginX() - full_bbox.getBeginX()
661 maxX = bbox.getEndX() - full_bbox.getBeginX()
662 minY = bbox.getBeginY() - full_bbox.getBeginY()
663 maxY = bbox.getEndY() - full_bbox.getBeginY()
664 data = hdu.section[minY:maxY, minX:maxX]
666 # Must correct the header WCS to take into
667 # account the offset.
668 if (k := "CRPIX1") in hdr:
669 hdr[k] -= minX
670 if (k := "CRPIX2") in hdr:
671 hdr[k] -= minY
672 if (k := "LTV1") in hdr:
673 hdr[k] = -bbox.getBeginX()
674 if (k := "LTV2") in hdr:
675 hdr[k] = -bbox.getBeginY()
676 if (k := "CRVAL1A") in hdr:
677 hdr[k] = bbox.getBeginX()
678 if (k := "CRVAL2A") in hdr:
679 hdr[k] = bbox.getBeginY()
680 else:
681 data = np.zeros([1, 1], dtype=np.int32)
683 # Construct a new HDU and copy the header.
684 stripped_hdu = astropy.io.fits.ImageHDU(data=data, header=hdr)
685 hdul.append(stripped_hdu)
686 else:
687 hdul.append(hdu)
688 stripped_fits = astropy.io.fits.HDUList(hdus=hdul)
689 # Write the FITS file to in-memory FITS.
690 buffer = BytesIO()
691 stripped_fits.writeto(buffer)
692 except Exception as e:
693 # For some reason we can't open the remote file so fall back.
694 _LOG.debug(
695 "Attempted remote read of components but encountered an error. "
696 "Falling back to file download. Error: %s",
697 str(e),
698 )
699 return NotImplemented
701 # Pass the new FITS buffer to the reader class without going through
702 # a temporary file. We can assume this is relatively small for
703 # components.
704 fits_data = buffer.getvalue()
705 mem = MemFileManager(len(fits_data))
706 mem.setData(fits_data, len(fits_data))
707 self._reader = self.ReaderClass(mem)
709 if component:
710 with self._lock:
711 type(self)._cached_fits = _ComponentCache(
712 id_=self.dataset_ref.id,
713 reader=self._reader,
714 mem=mem,
715 bbox=bbox_component,
716 )
717 match component:
718 case "xy0":
719 if bbox_component is None: # For mypy.
720 return None
721 return bbox_component.getMin()
722 case "dimensions":
723 if bbox_component is None:
724 return None
725 return bbox_component.getDimensions()
726 case "bbox":
727 return bbox_component
728 case _:
729 return self.readComponent(component)
730 else:
731 # Must be a cutout. We have applied the bbox parameter so no
732 # parameters should be passed here.
733 cutout = self.reader.read(dtype=self.storageClass_dtype)
734 cutout.getInfo().setFilter(self._fixFilterLabels(cutout.getInfo().getFilter()))
735 return cutout
737 def add_provenance(
738 self, in_memory_dataset: Any, /, *, provenance: DatasetProvenance | None = None
739 ) -> Any:
740 # Add provenance via FITS headers.
741 add_provenance_to_fits_header(in_memory_dataset.metadata, self.dataset_ref, provenance)
742 return in_memory_dataset
744 def readComponent(self, component: str) -> Any:
745 # Docstring inherited.
746 # Generic components can be read via a string name; DM-27754 will make
747 # this mapping larger at the expense of the following one.
748 genericComponents = {
749 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS,
750 }
751 if (genericComponentName := genericComponents.get(component)) is not None:
752 return self.reader.readComponent(genericComponentName)
753 # Other components have hard-coded method names, but don't take
754 # parameters.
755 standardComponents = {
756 "id": "readExposureId",
757 "metadata": "readMetadata",
758 "wcs": "readWcs",
759 "coaddInputs": "readCoaddInputs",
760 "psf": "readPsf",
761 "photoCalib": "readPhotoCalib",
762 "filter": "readFilter",
763 "validPolygon": "readValidPolygon",
764 "apCorrMap": "readApCorrMap",
765 "visitInfo": "readVisitInfo",
766 "transmissionCurve": "readTransmissionCurve",
767 "detector": "readDetector",
768 "exposureInfo": "readExposureInfo",
769 }
770 if (methodName := standardComponents.get(component)) is not None:
771 result = getattr(self.reader, methodName)()
772 if component == "filter":
773 return self._fixFilterLabels(result)
774 return result
775 # Delegate to MaskedImage and ImageBase implementations for the rest.
776 return super().readComponent(component)
778 def readFull(self) -> Any:
779 # Docstring inherited.
780 amplifier, detector, _ = standardizeAmplifierParameters(
781 self.checked_parameters,
782 self.reader.readDetector(),
783 )
784 if amplifier is not None:
785 amplifier_isolator = AmplifierIsolator(
786 amplifier,
787 self.reader.readBBox(),
788 detector,
789 )
790 result = amplifier_isolator.transform_subimage(
791 self.reader.read(bbox=amplifier_isolator.subimage_bbox, dtype=self.storageClass_dtype)
792 )
793 result.setDetector(amplifier_isolator.make_detector())
794 else:
795 result = self.reader.read(**self.checked_parameters, dtype=self.storageClass_dtype)
796 result.getInfo().setFilter(self._fixFilterLabels(result.getInfo().getFilter()))
797 return result
799 def _fixFilterLabels(
800 self, file_filter_label: lsst.afw.image.FilterLabel, should_be_standardized: bool | None = None
801 ) -> lsst.afw.image.FilterLabel:
802 """Compare the filter label read from the file with the one in the
803 data ID.
805 Parameters
806 ----------
807 file_filter_label : `lsst.afw.image.FilterLabel` or `None`
808 Filter label read from the file, if there was one.
809 should_be_standardized : `bool`, optional
810 If `True`, expect ``file_filter_label`` to be consistent with the
811 data ID and warn only if it is not. If `False`, expect it to be
812 inconsistent and warn only if the data ID is incomplete and hence
813 the `FilterLabel` cannot be fixed. If `None` (default) guess
814 whether the file should be standardized by looking at the
815 serialization version number in file, which requires this method to
816 have been run after `readFull` or `readComponent`.
818 Returns
819 -------
820 filter_label : `lsst.afw.image.FilterLabel` or `None`
821 The preferred filter label; may be the given one or one built from
822 the data ID. `None` is returned if there should never be any
823 filters associated with this dataset type.
825 Notes
826 -----
827 Most test coverage for this method is in ci_hsc_gen3, where we have
828 much easier access to test data that exhibits the problems it attempts
829 to solve.
830 """
831 # Remember filter data ID keys that weren't in this particular data ID,
832 # so we can warn about them later.
833 missing = []
834 band = None
835 physical_filter = None
836 if "band" in self.data_id.dimensions.names:
837 band = self.data_id.get("band")
838 # band isn't in the data ID; is that just because this data ID
839 # hasn't been filled in with everything the Registry knows, or
840 # because this dataset is never associated with a band?
841 if band is None and not self.data_id.hasFull() and "band" in self.data_id.dimensions.implied:
842 missing.append("band")
843 if "physical_filter" in self.data_id.dimensions.names:
844 physical_filter = self.data_id.get("physical_filter")
845 # Same check as above for band, but for physical_filter.
846 if (
847 physical_filter is None
848 and not self.data_id.hasFull()
849 and "physical_filter" in self.data_id.dimensions.implied
850 ):
851 missing.append("physical_filter")
852 if should_be_standardized is None:
853 version = self.reader.readSerializationVersion()
854 should_be_standardized = version >= 2
855 if missing:
856 # Data ID identifies a filter but the actual filter label values
857 # haven't been fetched from the database; we have no choice but
858 # to use the one in the file.
859 # Warn if that's more likely than not to be bad, because the file
860 # predates filter standardization.
861 if not should_be_standardized:
862 warnings.warn(
863 f"Data ID {self.data_id} is missing (implied) value(s) for {missing}; "
864 "the correctness of this Exposure's FilterLabel cannot be guaranteed. "
865 "Call Registry.expandDataId before Butler.get to avoid this.",
866 # Report the warning from outside of middleware or the
867 # relevant runQuantum method.
868 stacklevel=find_outside_stacklevel(
869 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
870 ),
871 )
872 return file_filter_label
873 if band is None and physical_filter is None:
874 data_id_filter_label = None
875 else:
876 data_id_filter_label = FilterLabel(band=band, physical=physical_filter)
877 if data_id_filter_label != file_filter_label and should_be_standardized:
878 # File was written after FilterLabel and standardization, but its
879 # FilterLabel doesn't agree with the data ID: this indicates a bug
880 # in whatever code produced the Exposure (though it may be one that
881 # has been fixed since the file was written).
882 warnings.warn(
883 f"Reading {self.file_descriptor.location} with data ID {self.data_id}: "
884 f"filter label mismatch (file is {file_filter_label}, data ID is "
885 f"{data_id_filter_label}). This is probably a bug in the code that produced it.",
886 stacklevel=find_outside_stacklevel(
887 "lsst.obs.base", "lsst.pipe.base", "lsst.daf.butler", allow_methods={"runQuantum"}
888 ),
889 )
890 return data_id_filter_label