29from astropy.table
import Table
30from astropy.io
import fits
33from lsst.utils.introspection
import get_full_type_name
34from lsst.utils
import doImport
37__all__ = [
"IsrCalib",
"IsrProvenance"]
41 """Generic calibration type.
43 Subclasses must implement the toDict, fromDict, toTable, fromTable
44 methods that allow the calibration information to be converted
45 from dictionaries
and afw tables. This will allow the calibration
46 to be persisted using the base
class read/write methods.
48 The validate method
is intended to provide a common way to check
49 that the calibration
is valid (internally consistent)
and
50 appropriate (usable
with the intended data). The apply method
is
51 intended to allow the calibration to be applied
in a consistent
57 Camera to extract metadata
from.
59 Detector to extract metadata
from.
60 log : `logging.Logger`, optional
67 def __init__(self, camera=None, detector=None, log=None, **kwargs):
83 "_detectorName",
"_detectorSerial",
"_detectorId",
84 "_filter",
"_calibId",
"_metadata"])
86 self.
log = log
if log
else logging.getLogger(__name__)
93 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
96 """Calibration equivalence.
98 Running ``calib.log.setLevel(0)`` enables debug statements to
99 identify problematic fields.
101 if not isinstance(other, self.__class__):
102 self.
log.debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
106 attrSelf = getattr(self, attr)
107 attrOther = getattr(other, attr)
109 if isinstance(attrSelf, dict):
111 if attrSelf.keys() != attrOther.keys():
112 self.
log.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
116 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
117 self.
log.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
124 if attrSelf[key] != attrOther[key]:
126 elif isinstance(attrSelf, np.ndarray):
128 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
129 self.
log.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
131 elif type(attrSelf) != type(attrOther):
132 if set([attrSelf, attrOther]) == set([
None,
""]):
135 self.
log.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
139 if attrSelf != attrOther:
140 self.
log.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
149 @requiredAttributes.setter
154 """Retrieve metadata associated with this calibration.
160 modified by the caller and the changes will be written to
166 """Store a copy of the supplied metadata with this calibration.
171 Metadata to associate with the calibration. Will be copied
and
172 overwrite existing metadata.
174 if metadata
is not None:
182 if isinstance(metadata, dict):
184 elif isinstance(metadata, PropertyList):
188 setCalibId=False, setCalibInfo=False, setDate=False,
190 """Update metadata keywords with new values.
195 Reference camera to use to set _instrument field.
197 Reference detector to use to set _detector* fields.
198 filterName : `str`, optional
199 Filter name to assign to this calibration.
200 setCalibId : `bool`, optional
201 Construct the _calibId field from other fields.
202 setCalibInfo : `bool`, optional
203 Set calibration parameters
from metadata.
204 setDate : `bool`, optional
205 Ensure the metadata CALIBDATE fields are set to the current
207 kwargs : `dict`
or `collections.abc.Mapping`, optional
208 Set of key=value pairs to assign to the metadata.
211 mdSupplemental = dict()
213 for k, v
in kwargs.items():
214 if isinstance(v, fits.card.Undefined):
239 date = datetime.datetime.now()
240 mdSupplemental[
"CALIBDATE"] = date.isoformat()
241 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
242 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
246 values.append(f
"instrument={self._instrument}")
if self.
_instrument else None
247 values.append(f
"raftName={self._raftName}")
if self.
_raftName else None
248 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName else None
249 values.append(f
"detector={self._detectorId}")
if self.
_detectorId else None
250 values.append(f
"filter={self._filter}")
if self.
_filter else None
252 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
253 values.append(f
"calibDate={calibDate}")
if calibDate
else None
265 self.
_metadata[
"CALIBCLS"] = get_full_type_name(self)
267 mdSupplemental.update(kwargs)
268 mdOriginal.update(mdSupplemental)
271 """Handle common keywords.
273 This isn't an ideal solution, but until all calibrations
274 expect to find everything in the metadata, they still need to
275 search through dictionaries.
280 Source
for the common keywords.
285 Raised
if the dictionary does
not match the expected OBSTYPE.
289 def search(haystack, needles):
290 """Search dictionary 'haystack' for an entry in 'needles'
292 test = [haystack.get(x) for x
in needles]
293 test = set([x
for x
in test
if x
is not None])
295 if "metadata" in haystack:
296 return search(haystack[
"metadata"], needles)
300 value = list(test)[0]
306 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
308 if "metadata" in dictionary:
309 metadata = dictionary[
"metadata"]
311 if self.
_OBSTYPE != metadata[
"OBSTYPE"]:
312 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
313 f
"found {metadata['OBSTYPE']}")
315 self.
_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
316 self.
_raftName = search(dictionary, [
"RAFTNAME"])
317 self.
_slotName = search(dictionary, [
"SLOTNAME"])
318 self.
_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
319 self.
_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
320 self.
_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
321 self.
_filter = search(dictionary, [
"FILTER",
"filterName"])
322 self.
_calibId = search(dictionary, [
"CALIB_ID"])
326 """Attempt to find calibration class in metadata.
331 Metadata possibly containing a calibration
class entry.
333 Message to include
in any errors.
337 calibClass : `object`
338 The
class to use to read the file contents. Should be an
344 Raised
if the resulting calibClass
is the base
345 `lsst.ip.isr.IsrClass` (which does
not implement the
348 calibClassName = metadata.get("CALIBCLS")
349 calibClass = doImport(calibClassName)
if calibClassName
is not None else cls
350 if calibClass
is IsrCalib:
351 raise ValueError(f
"Cannot use base class to read calibration data: {message}")
356 """Read calibration representation from a yaml/ecsv file.
361 Name of the file containing the calibration definition.
362 kwargs : `dict` or collections.abc.Mapping`, optional
363 Set of key=value pairs to
pass to the ``fromDict``
or
364 ``fromTable`` methods.
368 calib : `~lsst.ip.isr.IsrCalibType`
374 Raised
if the filename does
not end
in ".ecsv" or ".yaml".
376 if filename.endswith((
".ecsv",
".ECSV")):
377 data = Table.read(filename, format=
"ascii.ecsv")
379 return calibClass.fromTable([data], **kwargs)
380 elif filename.endswith((
".yaml",
".YAML")):
381 with open(filename,
"r")
as f:
382 data = yaml.load(f, Loader=yaml.CLoader)
384 return calibClass.fromDict(data, **kwargs)
386 raise RuntimeError(f
"Unknown filename extension: {filename}")
389 """Write the calibration data to a text file.
394 Name of the file to write.
396 Format to write the file as. Supported values are:
397 ``
"auto"`` : Determine filetype
from filename.
398 ``
"yaml"`` : Write
as yaml.
399 ``
"ecsv"`` : Write
as ecsv.
403 The name of the file used to write the data. This may
404 differ
from the input
if the format
is explicitly chosen.
409 Raised
if filename does
not end
in a known extension,
or
410 if all information cannot be written.
414 The file
is written to YAML/ECSV format
and will include any
417 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
419 path, ext = os.path.splitext(filename)
420 filename = path +
".yaml"
421 with open(filename,
"w")
as f:
422 yaml.dump(outDict, f)
423 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
425 if len(tableList) > 1:
428 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
431 path, ext = os.path.splitext(filename)
432 filename = path +
".ecsv"
433 table.write(filename, format=
"ascii.ecsv")
435 raise RuntimeError(f
"Attempt to write to a file {filename} "
436 "that does not end in '.yaml' or '.ecsv'")
442 """Read calibration data from a FITS file.
447 Filename to read data from.
448 kwargs : `dict`
or collections.abc.Mapping`, optional
449 Set of key=value pairs to
pass to the ``fromTable``
455 Calibration contained within the file.
458 tableList.append(Table.read(filename, hdu=1))
463 with warnings.catch_warnings():
464 warnings.simplefilter(
"error")
466 newTable = Table.read(filename, hdu=extNum)
467 tableList.append(newTable)
472 for table
in tableList:
473 for k, v
in table.meta.items():
474 if isinstance(v, fits.card.Undefined):
478 return calibClass.fromTable(tableList, **kwargs)
481 """Write calibration data to a FITS file.
486 Filename to write data to.
491 The name of the file used to write the data.
495 with warnings.catch_warnings():
496 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
497 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
498 astropyList.insert(0, fits.PrimaryHDU())
500 writer = fits.HDUList(astropyList)
501 writer.writeto(filename, overwrite=
True)
505 """Modify the calibration parameters to match the supplied detector.
510 Detector to use to set parameters from.
515 This needs to be implemented by subclasses
for each
518 raise NotImplementedError(
"Must be implemented by subclass.")
522 """Construct a calibration from a dictionary of properties.
524 Must be implemented by the specific calibration subclasses.
529 Dictionary of properties.
530 kwargs : `dict` or collections.abc.Mapping`, optional
531 Set of key=value options.
535 calib : `lsst.ip.isr.CalibType`
536 Constructed calibration.
540 NotImplementedError :
541 Raised
if not implemented.
543 raise NotImplementedError(
"Must be implemented by subclass.")
546 """Return a dictionary containing the calibration properties.
548 The dictionary should be able to be round-tripped through
554 Dictionary of properties.
558 NotImplementedError :
559 Raised if not implemented.
561 raise NotImplementedError(
"Must be implemented by subclass.")
565 """Construct a calibration from a dictionary of properties.
567 Must be implemented by the specific calibration subclasses.
571 tableList : `list` [`lsst.afw.table.Table`]
572 List of tables of properties.
573 kwargs : `dict` or collections.abc.Mapping`, optional
574 Set of key=value options.
578 calib : `lsst.ip.isr.CalibType`
579 Constructed calibration.
583 NotImplementedError :
584 Raised
if not implemented.
586 raise NotImplementedError(
"Must be implemented by subclass.")
589 """Return a list of tables containing the calibration properties.
591 The table list should be able to be round-tripped through
596 tableList : `list` [`lsst.afw.table.Table`]
597 List of tables of properties.
601 NotImplementedError :
602 Raised if not implemented.
604 raise NotImplementedError(
"Must be implemented by subclass.")
607 """Validate that this calibration is defined and can be used.
611 other : `object`, optional
612 Thing to validate against.
617 Returns true if the calibration
is valid
and appropriate.
622 """Method to apply the calibration to the target object.
627 Thing to validate against.
632 Returns true if the calibration was applied correctly.
636 NotImplementedError :
637 Raised
if not implemented.
639 raise NotImplementedError(
"Must be implemented by subclass.")
643 """Class for the provenance of data used to construct calibration.
645 Provenance is not really a calibration, but we would like to
646 record this when constructing the calibration,
and it provides an
647 example of the base calibration
class.
651 instrument : `str`, optional
652 Name of the instrument the data was taken
with.
653 calibType : `str`, optional
654 Type of calibration this provenance was generated
for.
655 detectorName : `str`, optional
656 Name of the detector this calibration
is for.
657 detectorSerial : `str`, optional
658 Identifier
for the detector.
661 _OBSTYPE = "IsrProvenance"
674 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
677 return super().
__eq__(other)
680 """Update calibration metadata.
684 setDate : `bool, optional
685 Update the CALIBDATE fields in the metadata to the current
686 time. Defaults to
False.
687 kwargs : `dict`
or `collections.abc.Mapping`, optional
688 Other keyword parameters to set
in the metadata.
694 """Update provenance from dataId List.
698 dataIdList : `list` [`lsst.daf.butler.DataId`]
699 List of dataIds used in generating this calibration.
701 for dataId
in dataIdList:
709 """Construct provenance from table list.
713 tableList : `list` [`lsst.afw.table.Table`]
714 List of tables to construct the provenance from.
719 The provenance defined
in the tables.
722 metadata = table.meta
724 inDict["metadata"] = metadata
725 inDict[
"calibType"] = metadata[
"calibType"]
726 inDict[
"dimensions"] = set()
727 inDict[
"dataIdList"] = list()
730 for colName
in table.columns:
731 schema[colName.lower()] = colName
732 inDict[
"dimensions"].add(colName.lower())
733 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
737 for dim
in sorted(inDict[
"dimensions"]):
738 entry[dim] = row[schema[dim]]
739 inDict[
"dataIdList"].append(entry)
745 """Construct provenance from a dictionary.
750 Dictionary of provenance parameters.
755 The provenance defined in the tables.
758 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
759 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
760 f
"found {dictionary['metadata']['OBSTYPE']}")
761 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
766 calib.calibType = dictionary[
"calibType"]
767 calib.dimensions = set(dictionary[
"dimensions"])
768 calib.dataIdList = dictionary[
"dataIdList"]
770 calib.updateMetadata()
774 """Return a dictionary containing the provenance information.
779 Dictionary of provenance.
786 outDict["metadata"] = metadata
798 """Return a list of tables containing the provenance.
800 This seems inefficient and slow, so this may
not be the best
801 way to store the data.
805 tableList : `list` [`lsst.afw.table.Table`]
806 List of tables containing the provenance information
814 filteredMetadata = {k: v
for k, v
in self.
getMetadata().
toDict().items()
if v
is not None}
815 catalog.meta = filteredMetadata
816 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def writeText(self, filename, format="auto")
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def determineCalibClass(cls, metadata, message)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)