Coverage for python/lsst/ip/isr/calibType.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import datetime
23import logging
24import os.path
25import warnings
26import yaml
27import numpy as np
29from astropy.table import Table
30from astropy.io import fits
32from lsst.daf.base import PropertyList
35__all__ = ["IsrCalib", "IsrProvenance"]
38class IsrCalib(abc.ABC):
39 """Generic calibration type.
41 Subclasses must implement the toDict, fromDict, toTable, fromTable
42 methods that allow the calibration information to be converted
43 from dictionaries and afw tables. This will allow the calibration
44 to be persisted using the base class read/write methods.
46 The validate method is intended to provide a common way to check
47 that the calibration is valid (internally consistent) and
48 appropriate (usable with the intended data). The apply method is
49 intended to allow the calibration to be applied in a consistent
50 manner.
52 Parameters
53 ----------
54 camera : `lsst.afw.cameraGeom.Camera`, optional
55 Camera to extract metadata from.
56 detector : `lsst.afw.cameraGeom.Detector`, optional
57 Detector to extract metadata from.
58 log : `logging.Logger`, optional
59 Log for messages.
60 """
61 _OBSTYPE = 'generic'
62 _SCHEMA = 'NO SCHEMA'
63 _VERSION = 0
65 def __init__(self, camera=None, detector=None, log=None, **kwargs):
66 self._instrument = None
67 self._raftName = None
68 self._slotName = None
69 self._detectorName = None
70 self._detectorSerial = None
71 self._detectorId = None
72 self._filter = None
73 self._calibId = None
74 self._metadata = PropertyList()
75 self.setMetadata(PropertyList())
76 self.calibInfoFromDict(kwargs)
78 # Define the required attributes for this calibration.
79 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
80 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName',
81 '_detectorName', '_detectorSerial', '_detectorId',
82 '_filter', '_calibId', '_metadata'])
84 self.log = log if log else logging.getLogger(__name__.partition(".")[2])
86 if detector:
87 self.fromDetector(detector)
88 self.updateMetadata(camera=camera, detector=detector)
90 def __str__(self):
91 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
93 def __eq__(self, other):
94 """Calibration equivalence.
96 Running ``calib.log.setLevel(0)`` enables debug statements to
97 identify problematic fields.
98 """
99 if not isinstance(other, self.__class__):
100 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__)
101 return False
103 for attr in self._requiredAttributes:
104 attrSelf = getattr(self, attr)
105 attrOther = getattr(other, attr)
107 if isinstance(attrSelf, dict):
108 # Dictionary of arrays.
109 if attrSelf.keys() != attrOther.keys():
110 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
111 return False
112 for key in attrSelf:
113 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True):
114 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
115 return False
116 elif isinstance(attrSelf, np.ndarray):
117 # Bare array.
118 if not np.allclose(attrSelf, attrOther, equal_nan=True):
119 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther)
120 return False
121 elif type(attrSelf) != type(attrOther):
122 if set([attrSelf, attrOther]) == set([None, ""]):
123 # Fits converts None to "", but None is not "".
124 continue
125 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
126 attrSelf, attrOther)
127 return False
128 else:
129 if attrSelf != attrOther:
130 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther)
131 return False
133 return True
135 @property
136 def requiredAttributes(self):
137 return self._requiredAttributes
139 @requiredAttributes.setter
140 def requiredAttributes(self, value):
141 self._requiredAttributes = value
143 def getMetadata(self):
144 """Retrieve metadata associated with this calibration.
146 Returns
147 -------
148 meta : `lsst.daf.base.PropertyList`
149 Metadata. The returned `~lsst.daf.base.PropertyList` can be
150 modified by the caller and the changes will be written to
151 external files.
152 """
153 return self._metadata
155 def setMetadata(self, metadata):
156 """Store a copy of the supplied metadata with this calibration.
158 Parameters
159 ----------
160 metadata : `lsst.daf.base.PropertyList`
161 Metadata to associate with the calibration. Will be copied and
162 overwrite existing metadata.
163 """
164 if metadata is not None:
165 self._metadata.update(metadata)
167 # Ensure that we have the obs type required by calibration ingest
168 self._metadata["OBSTYPE"] = self._OBSTYPE
169 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
170 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
172 if isinstance(metadata, dict):
173 self.calibInfoFromDict(metadata)
174 elif isinstance(metadata, PropertyList):
175 self.calibInfoFromDict(metadata.toDict())
177 def updateMetadata(self, camera=None, detector=None, filterName=None,
178 setCalibId=False, setCalibInfo=False, setDate=False,
179 **kwargs):
180 """Update metadata keywords with new values.
182 Parameters
183 ----------
184 camera : `lsst.afw.cameraGeom.Camera`, optional
185 Reference camera to use to set _instrument field.
186 detector : `lsst.afw.cameraGeom.Detector`, optional
187 Reference detector to use to set _detector* fields.
188 filterName : `str`, optional
189 Filter name to assign to this calibration.
190 setCalibId : `bool`, optional
191 Construct the _calibId field from other fields.
192 setCalibInfo : `bool`, optional
193 Set calibration parameters from metadata.
194 setDate : `bool`, optional
195 Ensure the metadata CALIBDATE fields are set to the current datetime.
196 kwargs : `dict` or `collections.abc.Mapping`, optional
197 Set of key=value pairs to assign to the metadata.
198 """
199 mdOriginal = self.getMetadata()
200 mdSupplemental = dict()
202 for k, v in kwargs.items():
203 if isinstance(v, fits.card.Undefined):
204 kwargs[k] = None
206 if setCalibInfo:
207 self.calibInfoFromDict(kwargs)
209 if camera:
210 self._instrument = camera.getName()
212 if detector:
213 self._detectorName = detector.getName()
214 self._detectorSerial = detector.getSerial()
215 self._detectorId = detector.getId()
216 if "_" in self._detectorName:
217 (self._raftName, self._slotName) = self._detectorName.split("_")
219 if filterName:
220 # TOD0 DM-28093: I think this whole comment can go away, if we
221 # always use physicalLabel everywhere in ip_isr.
222 # If set via:
223 # exposure.getInfo().getFilter().getName()
224 # then this will hold the abstract filter.
225 self._filter = filterName
227 if setDate:
228 date = datetime.datetime.now()
229 mdSupplemental['CALIBDATE'] = date.isoformat()
230 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
231 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
233 if setCalibId:
234 values = []
235 values.append(f"instrument={self._instrument}") if self._instrument else None
236 values.append(f"raftName={self._raftName}") if self._raftName else None
237 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
238 values.append(f"detector={self._detectorId}") if self._detectorId else None
239 values.append(f"filter={self._filter}") if self._filter else None
241 calibDate = mdOriginal.get('CALIBDATE', mdSupplemental.get('CALIBDATE', None))
242 values.append(f"calibDate={calibDate}") if calibDate else None
244 self._calibId = " ".join(values)
246 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
247 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
248 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
249 self._metadata["DETECTOR"] = self._detectorId
250 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
251 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
252 self._metadata["FILTER"] = self._filter if self._filter else None
253 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
255 mdSupplemental.update(kwargs)
256 mdOriginal.update(mdSupplemental)
258 def calibInfoFromDict(self, dictionary):
259 """Handle common keywords.
261 This isn't an ideal solution, but until all calibrations
262 expect to find everything in the metadata, they still need to
263 search through dictionaries.
265 Parameters
266 ----------
267 dictionary : `dict` or `lsst.daf.base.PropertyList`
268 Source for the common keywords.
270 Raises
271 ------
272 RuntimeError :
273 Raised if the dictionary does not match the expected OBSTYPE.
275 """
277 def search(haystack, needles):
278 """Search dictionary 'haystack' for an entry in 'needles'
279 """
280 test = [haystack.get(x) for x in needles]
281 test = set([x for x in test if x is not None])
282 if len(test) == 0:
283 if 'metadata' in haystack:
284 return search(haystack['metadata'], needles)
285 else:
286 return None
287 elif len(test) == 1:
288 value = list(test)[0]
289 if value == '':
290 return None
291 else:
292 return value
293 else:
294 raise ValueError(f"Too many values found: {len(test)} {test} {needles}")
296 if 'metadata' in dictionary:
297 metadata = dictionary['metadata']
299 if self._OBSTYPE != metadata['OBSTYPE']:
300 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
301 f"found {metadata['OBSTYPE']}")
303 self._instrument = search(dictionary, ['INSTRUME', 'instrument'])
304 self._raftName = search(dictionary, ['RAFTNAME'])
305 self._slotName = search(dictionary, ['SLOTNAME'])
306 self._detectorId = search(dictionary, ['DETECTOR', 'detectorId'])
307 self._detectorName = search(dictionary, ['DET_NAME', 'DETECTOR_NAME', 'detectorName'])
308 self._detectorSerial = search(dictionary, ['DET_SER', 'DETECTOR_SERIAL', 'detectorSerial'])
309 self._filter = search(dictionary, ['FILTER', 'filterName'])
310 self._calibId = search(dictionary, ['CALIB_ID'])
312 @classmethod
313 def readText(cls, filename, **kwargs):
314 """Read calibration representation from a yaml/ecsv file.
316 Parameters
317 ----------
318 filename : `str`
319 Name of the file containing the calibration definition.
320 kwargs : `dict` or collections.abc.Mapping`, optional
321 Set of key=value pairs to pass to the ``fromDict`` or
322 ``fromTable`` methods.
324 Returns
325 -------
326 calib : `~lsst.ip.isr.IsrCalibType`
327 Calibration class.
329 Raises
330 ------
331 RuntimeError :
332 Raised if the filename does not end in ".ecsv" or ".yaml".
333 """
334 if filename.endswith((".ecsv", ".ECSV")):
335 data = Table.read(filename, format='ascii.ecsv')
336 return cls.fromTable([data], **kwargs)
338 elif filename.endswith((".yaml", ".YAML")):
339 with open(filename, 'r') as f:
340 data = yaml.load(f, Loader=yaml.CLoader)
341 return cls.fromDict(data, **kwargs)
342 else:
343 raise RuntimeError(f"Unknown filename extension: {filename}")
345 def writeText(self, filename, format='auto'):
346 """Write the calibration data to a text file.
348 Parameters
349 ----------
350 filename : `str`
351 Name of the file to write.
352 format : `str`
353 Format to write the file as. Supported values are:
354 ``"auto"`` : Determine filetype from filename.
355 ``"yaml"`` : Write as yaml.
356 ``"ecsv"`` : Write as ecsv.
357 Returns
358 -------
359 used : `str`
360 The name of the file used to write the data. This may
361 differ from the input if the format is explicitly chosen.
363 Raises
364 ------
365 RuntimeError :
366 Raised if filename does not end in a known extension, or
367 if all information cannot be written.
369 Notes
370 -----
371 The file is written to YAML/ECSV format and will include any
372 associated metadata.
374 """
375 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
376 outDict = self.toDict()
377 path, ext = os.path.splitext(filename)
378 filename = path + ".yaml"
379 with open(filename, 'w') as f:
380 yaml.dump(outDict, f)
381 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
382 tableList = self.toTable()
383 if len(tableList) > 1:
384 # ECSV doesn't support multiple tables per file, so we
385 # can only write the first table.
386 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
388 table = tableList[0]
389 path, ext = os.path.splitext(filename)
390 filename = path + ".ecsv"
391 table.write(filename, format="ascii.ecsv")
392 else:
393 raise RuntimeError(f"Attempt to write to a file {filename} "
394 "that does not end in '.yaml' or '.ecsv'")
396 return filename
398 @classmethod
399 def readFits(cls, filename, **kwargs):
400 """Read calibration data from a FITS file.
402 Parameters
403 ----------
404 filename : `str`
405 Filename to read data from.
406 kwargs : `dict` or collections.abc.Mapping`, optional
407 Set of key=value pairs to pass to the ``fromTable``
408 method.
410 Returns
411 -------
412 calib : `lsst.ip.isr.IsrCalib`
413 Calibration contained within the file.
414 """
415 tableList = []
416 tableList.append(Table.read(filename, hdu=1))
417 extNum = 2 # Fits indices start at 1, we've read one already.
418 keepTrying = True
420 while keepTrying:
421 with warnings.catch_warnings():
422 warnings.simplefilter("error")
423 try:
424 newTable = Table.read(filename, hdu=extNum)
425 tableList.append(newTable)
426 extNum += 1
427 except Exception:
428 keepTrying = False
430 for table in tableList:
431 for k, v in table.meta.items():
432 if isinstance(v, fits.card.Undefined):
433 table.meta[k] = None
435 return cls.fromTable(tableList, **kwargs)
437 def writeFits(self, filename):
438 """Write calibration data to a FITS file.
440 Parameters
441 ----------
442 filename : `str`
443 Filename to write data to.
445 Returns
446 -------
447 used : `str`
448 The name of the file used to write the data.
450 """
451 tableList = self.toTable()
452 with warnings.catch_warnings():
453 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
454 astropyList = [fits.table_to_hdu(table) for table in tableList]
455 astropyList.insert(0, fits.PrimaryHDU())
457 writer = fits.HDUList(astropyList)
458 writer.writeto(filename, overwrite=True)
459 return filename
461 def fromDetector(self, detector):
462 """Modify the calibration parameters to match the supplied detector.
464 Parameters
465 ----------
466 detector : `lsst.afw.cameraGeom.Detector`
467 Detector to use to set parameters from.
469 Raises
470 ------
471 NotImplementedError
472 This needs to be implemented by subclasses for each
473 calibration type.
474 """
475 raise NotImplementedError("Must be implemented by subclass.")
477 @classmethod
478 def fromDict(cls, dictionary, **kwargs):
479 """Construct a calibration from a dictionary of properties.
481 Must be implemented by the specific calibration subclasses.
483 Parameters
484 ----------
485 dictionary : `dict`
486 Dictionary of properties.
487 kwargs : `dict` or collections.abc.Mapping`, optional
488 Set of key=value options.
490 Returns
491 ------
492 calib : `lsst.ip.isr.CalibType`
493 Constructed calibration.
495 Raises
496 ------
497 NotImplementedError :
498 Raised if not implemented.
499 """
500 raise NotImplementedError("Must be implemented by subclass.")
502 def toDict(self):
503 """Return a dictionary containing the calibration properties.
505 The dictionary should be able to be round-tripped through
506 `fromDict`.
508 Returns
509 -------
510 dictionary : `dict`
511 Dictionary of properties.
513 Raises
514 ------
515 NotImplementedError :
516 Raised if not implemented.
517 """
518 raise NotImplementedError("Must be implemented by subclass.")
520 @classmethod
521 def fromTable(cls, tableList, **kwargs):
522 """Construct a calibration from a dictionary of properties.
524 Must be implemented by the specific calibration subclasses.
526 Parameters
527 ----------
528 tableList : `list` [`lsst.afw.table.Table`]
529 List of tables of properties.
530 kwargs : `dict` or collections.abc.Mapping`, optional
531 Set of key=value options.
533 Returns
534 ------
535 calib : `lsst.ip.isr.CalibType`
536 Constructed calibration.
538 Raises
539 ------
540 NotImplementedError :
541 Raised if not implemented.
542 """
543 raise NotImplementedError("Must be implemented by subclass.")
545 def toTable(self):
546 """Return a list of tables containing the calibration properties.
548 The table list should be able to be round-tripped through
549 `fromDict`.
551 Returns
552 -------
553 tableList : `list` [`lsst.afw.table.Table`]
554 List of tables of properties.
556 Raises
557 ------
558 NotImplementedError :
559 Raised if not implemented.
560 """
561 raise NotImplementedError("Must be implemented by subclass.")
563 def validate(self, other=None):
564 """Validate that this calibration is defined and can be used.
566 Parameters
567 ----------
568 other : `object`, optional
569 Thing to validate against.
571 Returns
572 -------
573 valid : `bool`
574 Returns true if the calibration is valid and appropriate.
575 """
576 return False
578 def apply(self, target):
579 """Method to apply the calibration to the target object.
581 Parameters
582 ----------
583 target : `object`
584 Thing to validate against.
586 Returns
587 -------
588 valid : `bool`
589 Returns true if the calibration was applied correctly.
591 Raises
592 ------
593 NotImplementedError :
594 Raised if not implemented.
595 """
596 raise NotImplementedError("Must be implemented by subclass.")
599class IsrProvenance(IsrCalib):
600 """Class for the provenance of data used to construct calibration.
602 Provenance is not really a calibration, but we would like to
603 record this when constructing the calibration, and it provides an
604 example of the base calibration class.
606 Parameters
607 ----------
608 instrument : `str`, optional
609 Name of the instrument the data was taken with.
610 calibType : `str`, optional
611 Type of calibration this provenance was generated for.
612 detectorName : `str`, optional
613 Name of the detector this calibration is for.
614 detectorSerial : `str`, optional
615 Identifier for the detector.
617 """
618 _OBSTYPE = 'IsrProvenance'
620 def __init__(self, calibType="unknown",
621 **kwargs):
622 self.calibType = calibType
623 self.dimensions = set()
624 self.dataIdList = list()
626 super().__init__(**kwargs)
628 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList'])
630 def __str__(self):
631 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
633 def __eq__(self, other):
634 return super().__eq__(other)
636 def updateMetadata(self, setDate=False, **kwargs):
637 """Update calibration metadata.
639 Parameters
640 ----------
641 setDate : `bool, optional
642 Update the CALIBDATE fields in the metadata to the current
643 time. Defaults to False.
644 kwargs : `dict` or `collections.abc.Mapping`, optional
645 Other keyword parameters to set in the metadata.
646 """
647 kwargs['calibType'] = self.calibType
648 super().updateMetadata(setDate=setDate, **kwargs)
650 def fromDataIds(self, dataIdList):
651 """Update provenance from dataId List.
653 Parameters
654 ----------
655 dataIdList : `list` [`lsst.daf.butler.DataId`]
656 List of dataIds used in generating this calibration.
657 """
658 for dataId in dataIdList:
659 for key in dataId:
660 if key not in self.dimensions:
661 self.dimensions.add(key)
662 self.dataIdList.append(dataId)
664 @classmethod
665 def fromTable(cls, tableList):
666 """Construct provenance from table list.
668 Parameters
669 ----------
670 tableList : `list` [`lsst.afw.table.Table`]
671 List of tables to construct the provenance from.
673 Returns
674 -------
675 provenance : `lsst.ip.isr.IsrProvenance`
676 The provenance defined in the tables.
677 """
678 table = tableList[0]
679 metadata = table.meta
680 inDict = dict()
681 inDict['metadata'] = metadata
682 inDict['calibType'] = metadata['calibType']
683 inDict['dimensions'] = set()
684 inDict['dataIdList'] = list()
686 schema = dict()
687 for colName in table.columns:
688 schema[colName.lower()] = colName
689 inDict['dimensions'].add(colName.lower())
690 inDict['dimensions'] = sorted(inDict['dimensions'])
692 for row in table:
693 entry = dict()
694 for dim in sorted(inDict['dimensions']):
695 entry[dim] = row[schema[dim]]
696 inDict['dataIdList'].append(entry)
698 return cls.fromDict(inDict)
700 @classmethod
701 def fromDict(cls, dictionary):
702 """Construct provenance from a dictionary.
704 Parameters
705 ----------
706 dictionary : `dict`
707 Dictionary of provenance parameters.
709 Returns
710 -------
711 provenance : `lsst.ip.isr.IsrProvenance`
712 The provenance defined in the tables.
713 """
714 calib = cls()
715 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
716 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
717 f"found {dictionary['metadata']['OBSTYPE']}")
718 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary['metadata'])
720 # These properties should be in the metadata, but occasionally
721 # are found in the dictionary itself. Check both places,
722 # ending with `None` if neither contains the information.
723 calib.calibType = dictionary['calibType']
724 calib.dimensions = set(dictionary['dimensions'])
725 calib.dataIdList = dictionary['dataIdList']
727 calib.updateMetadata()
728 return calib
730 def toDict(self):
731 """Return a dictionary containing the provenance information.
733 Returns
734 -------
735 dictionary : `dict`
736 Dictionary of provenance.
737 """
738 self.updateMetadata()
740 outDict = {}
742 metadata = self.getMetadata()
743 outDict['metadata'] = metadata
744 outDict['detectorName'] = self._detectorName
745 outDict['detectorSerial'] = self._detectorSerial
746 outDict['detectorId'] = self._detectorId
747 outDict['instrument'] = self._instrument
748 outDict['calibType'] = self.calibType
749 outDict['dimensions'] = list(self.dimensions)
750 outDict['dataIdList'] = self.dataIdList
752 return outDict
754 def toTable(self):
755 """Return a list of tables containing the provenance.
757 This seems inefficient and slow, so this may not be the best
758 way to store the data.
760 Returns
761 -------
762 tableList : `list` [`lsst.afw.table.Table`]
763 List of tables containing the provenance information
765 """
766 tableList = []
767 self.updateMetadata(setDate=True, setCalibInfo=True)
769 catalog = Table(rows=self.dataIdList,
770 names=self.dimensions)
771 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
772 catalog.meta = filteredMetadata
773 tableList.append(catalog)
774 return tableList