Coverage for python/lsst/obs/base/formatters/fitsExposure.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("FitsExposureFormatter", "FitsImageFormatter", "FitsMaskFormatter",
23 "FitsMaskedImageFormatter")
25from astro_metadata_translator import fix_header
26from lsst.daf.base import PropertySet
27from lsst.daf.butler import Formatter
28from lsst.afw.image import ExposureFitsReader, ImageFitsReader, MaskFitsReader, MaskedImageFitsReader
29# Needed for ApCorrMap to resolve properly
30from lsst.afw.math import BoundedField # noqa: F401
33class FitsExposureFormatter(Formatter):
34 """Interface for reading and writing Exposures to and from FITS files.
36 This Formatter supports write recipes.
38 Each ``FitsExposureFormatter`` recipe for FITS compression should
39 define ``image``, ``mask`` and ``variance`` entries, each of which may
40 contain ``compression`` and ``scaling`` entries. Defaults will be
41 provided for any missing elements under ``compression`` and
42 ``scaling``.
44 The allowed entries under ``compression`` are:
46 * ``algorithm`` (`str`): compression algorithm to use
47 * ``rows`` (`int`): number of rows per tile (0 = entire dimension)
48 * ``columns`` (`int`): number of columns per tile (0 = entire dimension)
49 * ``quantizeLevel`` (`float`): cfitsio quantization level
51 The allowed entries under ``scaling`` are:
53 * ``algorithm`` (`str`): scaling algorithm to use
54 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64)
55 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values?
56 * ``seed`` (`int`): seed for random number generator when fuzzing
57 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing
58 statistics
59 * ``quantizeLevel`` (`float`): divisor of the standard deviation for
60 ``STDEV_*`` scaling
61 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for
62 ``STDEV_POSITIVE``/``NEGATIVE``)
63 * ``bscale`` (`float`): manually specified ``BSCALE``
64 (for ``MANUAL`` scaling)
65 * ``bzero`` (`float`): manually specified ``BSCALE``
66 (for ``MANUAL`` scaling)
68 A very simple example YAML recipe:
70 .. code-block:: yaml
72 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter:
73 default:
74 image: &default
75 compression:
76 algorithm: GZIP_SHUFFLE
77 mask: *default
78 variance: *default
80 """
81 supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"})
82 extension = ".fits"
83 _metadata = None
84 supportedWriteParameters = frozenset({"recipe"})
85 _readerClass = ExposureFitsReader
87 unsupportedParameters = {}
88 """Support all parameters."""
90 @property
91 def metadata(self):
92 """The metadata read from this file. It will be stripped as
93 components are extracted from it
94 (`lsst.daf.base.PropertyList`).
95 """
96 if self._metadata is None:
97 self._metadata = self.readMetadata()
98 return self._metadata
100 def readMetadata(self):
101 """Read all header metadata directly into a PropertyList.
103 Returns
104 -------
105 metadata : `~lsst.daf.base.PropertyList`
106 Header metadata.
107 """
108 # Do not use ExposureFitsReader.readMetadata because that strips
109 # out lots of headers and there is no way to recover them
110 from lsst.afw.image import readMetadata
111 md = readMetadata(self.fileDescriptor.location.path)
112 fix_header(md)
113 return md
115 def stripMetadata(self):
116 """Remove metadata entries that are parsed into components.
118 This is only called when just the metadata is requested; stripping
119 entries there forces code that wants other components to ask for those
120 components directly rather than trying to extract them from the
121 metadata manually, which is fragile. This behavior is an intentional
122 change from Gen2.
124 Parameters
125 ----------
126 metadata : `~lsst.daf.base.PropertyList`
127 Header metadata, to be modified in-place.
128 """
129 # TODO: make sure this covers everything, by delegating to something
130 # that doesn't yet exist in afw.image.ExposureInfo.
131 from lsst.afw.image import bboxFromMetadata
132 from lsst.afw.geom import makeSkyWcs
134 # Protect against the metadata being missing
135 try:
136 bboxFromMetadata(self.metadata) # always strips
137 except LookupError:
138 pass
139 try:
140 makeSkyWcs(self.metadata, strip=True)
141 except Exception:
142 pass
144 def readComponent(self, component, parameters=None):
145 """Read a component held by the Exposure.
147 Parameters
148 ----------
149 component : `str`, optional
150 Component to read from the file.
151 parameters : `dict`, optional
152 If specified, a dictionary of slicing parameters that
153 overrides those in ``fileDescriptor``.
155 Returns
156 -------
157 obj : component-dependent
158 In-memory component object.
160 Raises
161 ------
162 KeyError
163 Raised if the requested component cannot be handled.
164 """
166 # Metadata is handled explicitly elsewhere
167 componentMap = {'wcs': ('readWcs', False),
168 'coaddInputs': ('readCoaddInputs', False),
169 'psf': ('readPsf', False),
170 'image': ('readImage', True),
171 'mask': ('readMask', True),
172 'variance': ('readVariance', True),
173 'photoCalib': ('readPhotoCalib', False),
174 'bbox': ('readBBox', True),
175 'dimensions': ('readBBox', True),
176 'xy0': ('readXY0', True),
177 'filter': ('readFilter', False),
178 'validPolygon': ('readValidPolygon', False),
179 'apCorrMap': ('readApCorrMap', False),
180 'visitInfo': ('readVisitInfo', False),
181 'transmissionCurve': ('readTransmissionCurve', False),
182 'detector': ('readDetector', False),
183 'extras': ('readExtraComponents', False),
184 'exposureInfo': ('readExposureInfo', False),
185 }
186 method, hasParams = componentMap.get(component, (None, False))
188 if method:
189 # This reader can read standalone Image/Mask files as well
190 # when dealing with components.
191 reader = self._readerClass(self.fileDescriptor.location.path)
192 caller = getattr(reader, method, None)
194 if caller:
195 if parameters is None:
196 parameters = self.fileDescriptor.parameters
197 if parameters is None:
198 parameters = {}
199 self.fileDescriptor.storageClass.validateParameters(parameters)
201 if hasParams and parameters:
202 thisComponent = caller(**parameters)
203 else:
204 thisComponent = caller()
205 if component == "dimensions" and thisComponent is not None:
206 thisComponent = thisComponent.getDimensions()
207 return thisComponent
208 else:
209 raise KeyError(f"Unknown component requested: {component}")
211 def readFull(self, parameters=None):
212 """Read the full Exposure object.
214 Parameters
215 ----------
216 parameters : `dict`, optional
217 If specified a dictionary of slicing parameters that overrides
218 those in ``fileDescriptor``.
220 Returns
221 -------
222 exposure : `~lsst.afw.image.Exposure`
223 Complete in-memory exposure.
224 """
225 fileDescriptor = self.fileDescriptor
226 if parameters is None:
227 parameters = fileDescriptor.parameters
228 if parameters is None:
229 parameters = {}
230 fileDescriptor.storageClass.validateParameters(parameters)
231 reader = self._readerClass(fileDescriptor.location.path)
232 return reader.read(**parameters)
234 def read(self, component=None):
235 """Read data from a file.
237 Parameters
238 ----------
239 component : `str`, optional
240 Component to read from the file. Only used if the `StorageClass`
241 for reading differed from the `StorageClass` used to write the
242 file.
244 Returns
245 -------
246 inMemoryDataset : `object`
247 The requested data as a Python object. The type of object
248 is controlled by the specific formatter.
250 Raises
251 ------
252 ValueError
253 Component requested but this file does not seem to be a concrete
254 composite.
255 KeyError
256 Raised when parameters passed with fileDescriptor are not
257 supported.
258 """
259 fileDescriptor = self.fileDescriptor
260 if fileDescriptor.readStorageClass != fileDescriptor.storageClass:
261 if component == "metadata":
262 self.stripMetadata()
263 return self.metadata
264 elif component is not None:
265 return self.readComponent(component)
266 else:
267 raise ValueError("Storage class inconsistency ({} vs {}) but no"
268 " component requested".format(fileDescriptor.readStorageClass.name,
269 fileDescriptor.storageClass.name))
270 return self.readFull()
272 def write(self, inMemoryDataset):
273 """Write a Python object to a file.
275 Parameters
276 ----------
277 inMemoryDataset : `object`
278 The Python object to store.
280 Returns
281 -------
282 path : `str`
283 The `URI` where the primary file is stored.
284 """
285 # Update the location with the formatter-preferred file extension
286 self.fileDescriptor.location.updateExtension(self.extension)
287 outputPath = self.fileDescriptor.location.path
289 # check to see if we have a recipe requested
290 recipeName = self.writeParameters.get("recipe")
291 recipe = self.getImageCompressionSettings(recipeName)
292 if recipe:
293 # Can not construct a PropertySet from a hierarchical
294 # dict but can update one.
295 ps = PropertySet()
296 ps.update(recipe)
297 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
298 else:
299 inMemoryDataset.writeFits(outputPath)
300 return self.fileDescriptor.location.pathInStore
302 def getImageCompressionSettings(self, recipeName):
303 """Retrieve the relevant compression settings for this recipe.
305 Parameters
306 ----------
307 recipeName : `str`
308 Label associated with the collection of compression parameters
309 to select.
311 Returns
312 -------
313 settings : `dict`
314 The selected settings.
315 """
316 # if no recipe has been provided and there is no default
317 # return immediately
318 if not recipeName:
319 if "default" not in self.writeRecipes:
320 return {}
321 recipeName = "default"
323 if recipeName not in self.writeRecipes:
324 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}")
326 recipe = self.writeRecipes[recipeName]
328 # Set the seed based on dataId
329 seed = hash(tuple(self.dataId.items())) % 2**31
330 for plane in ("image", "mask", "variance"):
331 if plane in recipe and "scaling" in recipe[plane]:
332 scaling = recipe[plane]["scaling"]
333 if "seed" in scaling and scaling["seed"] == 0:
334 scaling["seed"] = seed
336 return recipe
338 @classmethod
339 def validateWriteRecipes(cls, recipes):
340 """Validate supplied recipes for this formatter.
342 The recipes are supplemented with default values where appropriate.
344 TODO: replace this custom validation code with Cerberus (DM-11846)
346 Parameters
347 ----------
348 recipes : `dict`
349 Recipes to validate. Can be empty dict or `None`.
351 Returns
352 -------
353 validated : `dict`
354 Validated recipes. Returns what was given if there are no
355 recipes listed.
357 Raises
358 ------
359 RuntimeError
360 Raised if validation fails.
361 """
362 # Schemas define what should be there, and the default values (and by
363 # the default value, the expected type).
364 compressionSchema = {
365 "algorithm": "NONE",
366 "rows": 1,
367 "columns": 0,
368 "quantizeLevel": 0.0,
369 }
370 scalingSchema = {
371 "algorithm": "NONE",
372 "bitpix": 0,
373 "maskPlanes": ["NO_DATA"],
374 "seed": 0,
375 "quantizeLevel": 4.0,
376 "quantizePad": 5.0,
377 "fuzz": True,
378 "bscale": 1.0,
379 "bzero": 0.0,
380 }
382 if not recipes:
383 # We can not insist on recipes being specified
384 return recipes
386 def checkUnrecognized(entry, allowed, description):
387 """Check to see if the entry contains unrecognised keywords"""
388 unrecognized = set(entry) - set(allowed)
389 if unrecognized:
390 raise RuntimeError(
391 f"Unrecognized entries when parsing image compression recipe {description}: "
392 f"{unrecognized}")
394 validated = {}
395 for name in recipes:
396 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name)
397 validated[name] = {}
398 for plane in ("image", "mask", "variance"):
399 checkUnrecognized(recipes[name][plane], ["compression", "scaling"],
400 f"{name}->{plane}")
402 np = {}
403 validated[name][plane] = np
404 for settings, schema in (("compression", compressionSchema),
405 ("scaling", scalingSchema)):
406 np[settings] = {}
407 if settings not in recipes[name][plane]:
408 for key in schema:
409 np[settings][key] = schema[key]
410 continue
411 entry = recipes[name][plane][settings]
412 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}")
413 for key in schema:
414 value = type(schema[key])(entry[key]) if key in entry else schema[key]
415 np[settings][key] = value
416 return validated
419class FitsImageFormatter(FitsExposureFormatter):
420 """Specialisation for `~lsst.afw.image.Image` reading.
421 """
423 _readerClass = ImageFitsReader
426class FitsMaskFormatter(FitsExposureFormatter):
427 """Specialisation for `~lsst.afw.image.Mask` reading.
428 """
430 _readerClass = MaskFitsReader
433class FitsMaskedImageFormatter(FitsExposureFormatter):
434 """Specialisation for `~lsst.afw.image.MaskedImage` reading.
435 """
437 _readerClass = MaskedImageFitsReader