29from astropy.table
import Table
30from astropy.io
import fits
33from lsst.utils.introspection
import get_full_type_name
34from lsst.utils
import doImport
37__all__ = [
"IsrCalib",
"IsrProvenance"]
41 """Generic calibration type.
43 Subclasses must implement the toDict, fromDict, toTable, fromTable
44 methods that allow the calibration information to be converted
45 from dictionaries
and afw tables. This will allow the calibration
46 to be persisted using the base
class read/write methods.
48 The validate method
is intended to provide a common way to check
49 that the calibration
is valid (internally consistent)
and
50 appropriate (usable
with the intended data). The apply method
is
51 intended to allow the calibration to be applied
in a consistent
57 Camera to extract metadata
from.
59 Detector to extract metadata
from.
60 log : `logging.Logger`, optional
67 def __init__(self, camera=None, detector=None, log=None, **kwargs):
83 "_detectorName",
"_detectorSerial",
"_detectorId",
84 "_filter",
"_calibId",
"_metadata"])
86 self.
loglog = log
if log
else logging.getLogger(__name__)
90 self.
updateMetadataupdateMetadata(camera=camera, detector=detector)
93 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
96 """Calibration equivalence.
98 Running ``calib.log.setLevel(0)`` enables debug statements to
99 identify problematic fields.
101 if not isinstance(other, self.__class__):
102 self.
loglog.debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
106 attrSelf = getattr(self, attr)
107 attrOther = getattr(other, attr)
109 if isinstance(attrSelf, dict):
111 if attrSelf.keys() != attrOther.keys():
112 self.
loglog.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
116 self.
loglog.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
118 elif isinstance(attrSelf, np.ndarray):
120 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
121 self.
loglog.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
123 elif type(attrSelf) != type(attrOther):
124 if set([attrSelf, attrOther]) == set([
None,
""]):
127 self.
loglog.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
131 if attrSelf != attrOther:
132 self.
loglog.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
141 @requiredAttributes.setter
146 """Retrieve metadata associated with this calibration.
152 modified by the caller and the changes will be written to
158 """Store a copy of the supplied metadata with this calibration.
163 Metadata to associate with the calibration. Will be copied
and
164 overwrite existing metadata.
166 if metadata
is not None:
174 if isinstance(metadata, dict):
176 elif isinstance(metadata, PropertyList):
180 setCalibId=False, setCalibInfo=False, setDate=False,
182 """Update metadata keywords with new values.
187 Reference camera to use to set _instrument field.
189 Reference detector to use to set _detector* fields.
190 filterName : `str`, optional
191 Filter name to assign to this calibration.
192 setCalibId : `bool`, optional
193 Construct the _calibId field from other fields.
194 setCalibInfo : `bool`, optional
195 Set calibration parameters
from metadata.
196 setDate : `bool`, optional
197 Ensure the metadata CALIBDATE fields are set to the current
199 kwargs : `dict`
or `collections.abc.Mapping`, optional
200 Set of key=value pairs to assign to the metadata.
203 mdSupplemental = dict()
205 for k, v
in kwargs.items():
206 if isinstance(v, fits.card.Undefined):
228 self.
_filter_filter = filterName
231 date = datetime.datetime.now()
232 mdSupplemental[
"CALIBDATE"] = date.isoformat()
233 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
234 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
238 values.append(f
"instrument={self._instrument}")
if self.
_instrument_instrument
else None
239 values.append(f
"raftName={self._raftName}")
if self.
_raftName_raftName
else None
240 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName_detectorName
else None
241 values.append(f
"detector={self._detectorId}")
if self.
_detectorId_detectorId
else None
242 values.append(f
"filter={self._filter}")
if self.
_filter_filter
else None
244 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
245 values.append(f
"calibDate={calibDate}")
if calibDate
else None
247 self.
_calibId_calibId =
" ".join(values)
257 self.
_metadata_metadata[
"CALIBCLS"] = get_full_type_name(self)
259 mdSupplemental.update(kwargs)
260 mdOriginal.update(mdSupplemental)
263 """Handle common keywords.
265 This isn't an ideal solution, but until all calibrations
266 expect to find everything in the metadata, they still need to
267 search through dictionaries.
272 Source
for the common keywords.
277 Raised
if the dictionary does
not match the expected OBSTYPE.
281 def search(haystack, needles):
282 """Search dictionary 'haystack' for an entry in 'needles'
284 test = [haystack.get(x) for x
in needles]
285 test = set([x
for x
in test
if x
is not None])
287 if "metadata" in haystack:
288 return search(haystack[
"metadata"], needles)
292 value = list(test)[0]
298 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
300 if "metadata" in dictionary:
301 metadata = dictionary[
"metadata"]
303 if self.
_OBSTYPE_OBSTYPE != metadata[
"OBSTYPE"]:
304 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
305 f
"found {metadata['OBSTYPE']}")
307 self.
_instrument_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
308 self.
_raftName_raftName = search(dictionary, [
"RAFTNAME"])
309 self.
_slotName_slotName = search(dictionary, [
"SLOTNAME"])
310 self.
_detectorId_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
311 self.
_detectorName_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
312 self.
_detectorSerial_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
313 self.
_filter_filter = search(dictionary, [
"FILTER",
"filterName"])
314 self.
_calibId_calibId = search(dictionary, [
"CALIB_ID"])
318 """Attempt to find calibration class in metadata.
323 Metadata possibly containing a calibration
class entry.
325 Message to include
in any errors.
329 calibClass : `object`
330 The
class to use to read the file contents. Should be an
336 Raised
if the resulting calibClass
is the base
337 `lsst.ip.isr.IsrClass` (which does
not implement the
340 calibClassName = metadata.get("CALIBCLS")
341 calibClass = doImport(calibClassName)
if calibClassName
is not None else cls
342 if calibClass
is IsrCalib:
343 raise ValueError(f
"Cannot use base class to read calibration data: {message}")
348 """Read calibration representation from a yaml/ecsv file.
353 Name of the file containing the calibration definition.
354 kwargs : `dict` or collections.abc.Mapping`, optional
355 Set of key=value pairs to
pass to the ``fromDict``
or
356 ``fromTable`` methods.
360 calib : `~lsst.ip.isr.IsrCalibType`
366 Raised
if the filename does
not end
in ".ecsv" or ".yaml".
368 if filename.endswith((
".ecsv",
".ECSV")):
369 data = Table.read(filename, format=
"ascii.ecsv")
371 return calibClass.fromTable([data], **kwargs)
372 elif filename.endswith((
".yaml",
".YAML")):
373 with open(filename,
"r")
as f:
374 data = yaml.load(f, Loader=yaml.CLoader)
376 return calibClass.fromDict(data, **kwargs)
378 raise RuntimeError(f
"Unknown filename extension: {filename}")
381 """Write the calibration data to a text file.
386 Name of the file to write.
388 Format to write the file as. Supported values are:
389 ``
"auto"`` : Determine filetype
from filename.
390 ``
"yaml"`` : Write
as yaml.
391 ``
"ecsv"`` : Write
as ecsv.
395 The name of the file used to write the data. This may
396 differ
from the input
if the format
is explicitly chosen.
401 Raised
if filename does
not end
in a known extension,
or
402 if all information cannot be written.
406 The file
is written to YAML/ECSV format
and will include any
409 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
410 outDict = self.
toDicttoDict()
411 path, ext = os.path.splitext(filename)
412 filename = path +
".yaml"
413 with open(filename,
"w")
as f:
414 yaml.dump(outDict, f)
415 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
416 tableList = self.
toTabletoTable()
417 if len(tableList) > 1:
420 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
423 path, ext = os.path.splitext(filename)
424 filename = path +
".ecsv"
425 table.write(filename, format=
"ascii.ecsv")
427 raise RuntimeError(f
"Attempt to write to a file {filename} "
428 "that does not end in '.yaml' or '.ecsv'")
434 """Read calibration data from a FITS file.
439 Filename to read data from.
440 kwargs : `dict`
or collections.abc.Mapping`, optional
441 Set of key=value pairs to
pass to the ``fromTable``
447 Calibration contained within the file.
450 tableList.append(Table.read(filename, hdu=1))
455 with warnings.catch_warnings():
456 warnings.simplefilter(
"error")
458 newTable = Table.read(filename, hdu=extNum)
459 tableList.append(newTable)
464 for table
in tableList:
465 for k, v
in table.meta.items():
466 if isinstance(v, fits.card.Undefined):
470 return calibClass.fromTable(tableList, **kwargs)
473 """Write calibration data to a FITS file.
478 Filename to write data to.
483 The name of the file used to write the data.
486 tableList = self.toTabletoTable()
487 with warnings.catch_warnings():
488 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
489 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
490 astropyList.insert(0, fits.PrimaryHDU())
492 writer = fits.HDUList(astropyList)
493 writer.writeto(filename, overwrite=
True)
497 """Modify the calibration parameters to match the supplied detector.
502 Detector to use to set parameters from.
507 This needs to be implemented by subclasses
for each
510 raise NotImplementedError(
"Must be implemented by subclass.")
514 """Construct a calibration from a dictionary of properties.
516 Must be implemented by the specific calibration subclasses.
521 Dictionary of properties.
522 kwargs : `dict` or collections.abc.Mapping`, optional
523 Set of key=value options.
527 calib : `lsst.ip.isr.CalibType`
528 Constructed calibration.
532 NotImplementedError :
533 Raised
if not implemented.
535 raise NotImplementedError(
"Must be implemented by subclass.")
538 """Return a dictionary containing the calibration properties.
540 The dictionary should be able to be round-tripped through
546 Dictionary of properties.
550 NotImplementedError :
551 Raised if not implemented.
553 raise NotImplementedError(
"Must be implemented by subclass.")
557 """Construct a calibration from a dictionary of properties.
559 Must be implemented by the specific calibration subclasses.
563 tableList : `list` [`lsst.afw.table.Table`]
564 List of tables of properties.
565 kwargs : `dict` or collections.abc.Mapping`, optional
566 Set of key=value options.
570 calib : `lsst.ip.isr.CalibType`
571 Constructed calibration.
575 NotImplementedError :
576 Raised
if not implemented.
578 raise NotImplementedError(
"Must be implemented by subclass.")
581 """Return a list of tables containing the calibration properties.
583 The table list should be able to be round-tripped through
588 tableList : `list` [`lsst.afw.table.Table`]
589 List of tables of properties.
593 NotImplementedError :
594 Raised if not implemented.
596 raise NotImplementedError(
"Must be implemented by subclass.")
599 """Validate that this calibration is defined and can be used.
603 other : `object`, optional
604 Thing to validate against.
609 Returns true if the calibration
is valid
and appropriate.
614 """Method to apply the calibration to the target object.
619 Thing to validate against.
624 Returns true if the calibration was applied correctly.
628 NotImplementedError :
629 Raised
if not implemented.
631 raise NotImplementedError(
"Must be implemented by subclass.")
635 """Class for the provenance of data used to construct calibration.
637 Provenance is not really a calibration, but we would like to
638 record this when constructing the calibration,
and it provides an
639 example of the base calibration
class.
643 instrument : `str`, optional
644 Name of the instrument the data was taken
with.
645 calibType : `str`, optional
646 Type of calibration this provenance was generated
for.
647 detectorName : `str`, optional
648 Name of the detector this calibration
is for.
649 detectorSerial : `str`, optional
650 Identifier
for the detector.
653 _OBSTYPE = "IsrProvenance"
666 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
669 return super().
__eq__(other)
672 """Update calibration metadata.
676 setDate : `bool, optional
677 Update the CALIBDATE fields in the metadata to the current
678 time. Defaults to
False.
679 kwargs : `dict`
or `collections.abc.Mapping`, optional
680 Other keyword parameters to set
in the metadata.
682 kwargs["calibType"] = self.
calibTypecalibType
686 """Update provenance from dataId List.
690 dataIdList : `list` [`lsst.daf.butler.DataId`]
691 List of dataIds used in generating this calibration.
693 for dataId
in dataIdList:
701 """Construct provenance from table list.
705 tableList : `list` [`lsst.afw.table.Table`]
706 List of tables to construct the provenance from.
711 The provenance defined
in the tables.
714 metadata = table.meta
716 inDict["metadata"] = metadata
717 inDict[
"calibType"] = metadata[
"calibType"]
718 inDict[
"dimensions"] = set()
719 inDict[
"dataIdList"] = list()
722 for colName
in table.columns:
723 schema[colName.lower()] = colName
724 inDict[
"dimensions"].add(colName.lower())
725 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
729 for dim
in sorted(inDict[
"dimensions"]):
730 entry[dim] = row[schema[dim]]
731 inDict[
"dataIdList"].append(entry)
737 """Construct provenance from a dictionary.
742 Dictionary of provenance parameters.
747 The provenance defined in the tables.
750 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
751 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
752 f
"found {dictionary['metadata']['OBSTYPE']}")
753 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
758 calib.calibType = dictionary[
"calibType"]
759 calib.dimensions = set(dictionary[
"dimensions"])
760 calib.dataIdList = dictionary[
"dataIdList"]
762 calib.updateMetadata()
766 """Return a dictionary containing the provenance information.
771 Dictionary of provenance.
778 outDict["metadata"] = metadata
781 outDict[
"detectorId"] = self.
_detectorId_detectorId
782 outDict[
"instrument"] = self.
_instrument_instrument
783 outDict[
"calibType"] = self.
calibTypecalibType
784 outDict[
"dimensions"] = list(self.
dimensionsdimensions)
785 outDict[
"dataIdList"] = self.
dataIdListdataIdList
790 """Return a list of tables containing the provenance.
792 This seems inefficient and slow, so this may
not be the best
793 way to store the data.
797 tableList : `list` [`lsst.afw.table.Table`]
798 List of tables containing the provenance information
804 catalog = Table(rows=self.
dataIdListdataIdList,
806 filteredMetadata = {k: v
for k, v
in self.
getMetadatagetMetadata().
toDict().items()
if v
is not None}
807 catalog.meta = filteredMetadata
808 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def writeText(self, filename, format="auto")
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def determineCalibClass(cls, metadata, message)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)