29from astropy.table
import Table
30from astropy.io
import fits
33from lsst.utils.introspection
import get_full_type_name
34from lsst.utils
import doImport
37__all__ = [
"IsrCalib",
"IsrProvenance"]
41 """Generic calibration type.
43 Subclasses must implement the toDict, fromDict, toTable, fromTable
44 methods that allow the calibration information to be converted
45 from dictionaries
and afw tables. This will allow the calibration
46 to be persisted using the base
class read/write methods.
48 The validate method
is intended to provide a common way to check
49 that the calibration
is valid (internally consistent)
and
50 appropriate (usable
with the intended data). The apply method
is
51 intended to allow the calibration to be applied
in a consistent
57 Camera to extract metadata
from.
59 Detector to extract metadata
from.
60 log : `logging.Logger`, optional
67 def __init__(self, camera=None, detector=None, log=None, **kwargs):
83 "_detectorName",
"_detectorSerial",
"_detectorId",
84 "_filter",
"_calibId",
"_metadata"])
86 self.
loglog = log
if log
else logging.getLogger(__name__)
90 self.
updateMetadataupdateMetadata(camera=camera, detector=detector)
93 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
96 """Calibration equivalence.
98 Running ``calib.log.setLevel(0)`` enables debug statements to
99 identify problematic fields.
101 if not isinstance(other, self.__class__):
102 self.
loglog.debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
106 attrSelf = getattr(self, attr)
107 attrOther = getattr(other, attr)
109 if isinstance(attrSelf, dict):
111 if attrSelf.keys() != attrOther.keys():
112 self.
loglog.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
116 self.
loglog.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
118 elif isinstance(attrSelf, np.ndarray):
120 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
121 self.
loglog.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
123 elif type(attrSelf) != type(attrOther):
124 if set([attrSelf, attrOther]) == set([
None,
""]):
127 self.
loglog.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
131 if attrSelf != attrOther:
132 self.
loglog.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
141 @requiredAttributes.setter
146 """Retrieve metadata associated with this calibration.
152 modified by the caller and the changes will be written to
158 """Store a copy of the supplied metadata with this calibration.
163 Metadata to associate with the calibration. Will be copied
and
164 overwrite existing metadata.
166 if metadata
is not None:
174 if isinstance(metadata, dict):
176 elif isinstance(metadata, PropertyList):
180 setCalibId=False, setCalibInfo=False, setDate=False,
182 """Update metadata keywords with new values.
187 Reference camera to use to set _instrument field.
189 Reference detector to use to set _detector* fields.
190 filterName : `str`, optional
191 Filter name to assign to this calibration.
192 setCalibId : `bool`, optional
193 Construct the _calibId field from other fields.
194 setCalibInfo : `bool`, optional
195 Set calibration parameters
from metadata.
196 setDate : `bool`, optional
197 Ensure the metadata CALIBDATE fields are set to the current datetime.
198 kwargs : `dict`
or `collections.abc.Mapping`, optional
199 Set of key=value pairs to assign to the metadata.
202 mdSupplemental = dict()
204 for k, v
in kwargs.items():
205 if isinstance(v, fits.card.Undefined):
227 self.
_filter_filter = filterName
230 date = datetime.datetime.now()
231 mdSupplemental[
"CALIBDATE"] = date.isoformat()
232 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
233 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
237 values.append(f
"instrument={self._instrument}")
if self.
_instrument_instrument
else None
238 values.append(f
"raftName={self._raftName}")
if self.
_raftName_raftName
else None
239 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName_detectorName
else None
240 values.append(f
"detector={self._detectorId}")
if self.
_detectorId_detectorId
else None
241 values.append(f
"filter={self._filter}")
if self.
_filter_filter
else None
243 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
244 values.append(f
"calibDate={calibDate}")
if calibDate
else None
246 self.
_calibId_calibId =
" ".join(values)
256 self.
_metadata_metadata[
"CALIBCLS"] = get_full_type_name(self)
258 mdSupplemental.update(kwargs)
259 mdOriginal.update(mdSupplemental)
262 """Handle common keywords.
264 This isn't an ideal solution, but until all calibrations
265 expect to find everything in the metadata, they still need to
266 search through dictionaries.
271 Source
for the common keywords.
276 Raised
if the dictionary does
not match the expected OBSTYPE.
280 def search(haystack, needles):
281 """Search dictionary 'haystack' for an entry in 'needles'
283 test = [haystack.get(x) for x
in needles]
284 test = set([x
for x
in test
if x
is not None])
286 if "metadata" in haystack:
287 return search(haystack[
"metadata"], needles)
291 value = list(test)[0]
297 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
299 if "metadata" in dictionary:
300 metadata = dictionary[
"metadata"]
302 if self.
_OBSTYPE_OBSTYPE != metadata[
"OBSTYPE"]:
303 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
304 f
"found {metadata['OBSTYPE']}")
306 self.
_instrument_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
307 self.
_raftName_raftName = search(dictionary, [
"RAFTNAME"])
308 self.
_slotName_slotName = search(dictionary, [
"SLOTNAME"])
309 self.
_detectorId_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
310 self.
_detectorName_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
311 self.
_detectorSerial_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
312 self.
_filter_filter = search(dictionary, [
"FILTER",
"filterName"])
313 self.
_calibId_calibId = search(dictionary, [
"CALIB_ID"])
317 """Attempt to find calibration class in metadata.
322 Metadata possibly containing a calibration
class entry.
324 Message to include
in any errors.
328 calibClass : `object`
329 The
class to use to read the file contents. Should be an
335 Raised
if the resulting calibClass
is the base
336 `lsst.ip.isr.IsrClass` (which does
not implement the
339 calibClassName = metadata.get("CALIBCLS")
340 calibClass = doImport(calibClassName)
if calibClassName
is not None else cls
341 if calibClass
is IsrCalib:
342 raise ValueError(f
"Cannot use base class to read calibration data: {msg}")
347 """Read calibration representation from a yaml/ecsv file.
352 Name of the file containing the calibration definition.
353 kwargs : `dict` or collections.abc.Mapping`, optional
354 Set of key=value pairs to
pass to the ``fromDict``
or
355 ``fromTable`` methods.
359 calib : `~lsst.ip.isr.IsrCalibType`
365 Raised
if the filename does
not end
in ".ecsv" or ".yaml".
367 if filename.endswith((
".ecsv",
".ECSV")):
368 data = Table.read(filename, format=
"ascii.ecsv")
370 return calibClass.fromTable([data], **kwargs)
371 elif filename.endswith((
".yaml",
".YAML")):
372 with open(filename,
"r")
as f:
373 data = yaml.load(f, Loader=yaml.CLoader)
375 return calibClass.fromDict(data, **kwargs)
377 raise RuntimeError(f
"Unknown filename extension: {filename}")
380 """Write the calibration data to a text file.
385 Name of the file to write.
387 Format to write the file as. Supported values are:
388 ``
"auto"`` : Determine filetype
from filename.
389 ``
"yaml"`` : Write
as yaml.
390 ``
"ecsv"`` : Write
as ecsv.
394 The name of the file used to write the data. This may
395 differ
from the input
if the format
is explicitly chosen.
400 Raised
if filename does
not end
in a known extension,
or
401 if all information cannot be written.
405 The file
is written to YAML/ECSV format
and will include any
408 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
409 outDict = self.
toDicttoDict()
410 path, ext = os.path.splitext(filename)
411 filename = path +
".yaml"
412 with open(filename,
"w")
as f:
413 yaml.dump(outDict, f)
414 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
415 tableList = self.
toTabletoTable()
416 if len(tableList) > 1:
419 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
422 path, ext = os.path.splitext(filename)
423 filename = path +
".ecsv"
424 table.write(filename, format=
"ascii.ecsv")
426 raise RuntimeError(f
"Attempt to write to a file {filename} "
427 "that does not end in '.yaml' or '.ecsv'")
433 """Read calibration data from a FITS file.
438 Filename to read data from.
439 kwargs : `dict`
or collections.abc.Mapping`, optional
440 Set of key=value pairs to
pass to the ``fromTable``
446 Calibration contained within the file.
449 tableList.append(Table.read(filename, hdu=1))
454 with warnings.catch_warnings():
455 warnings.simplefilter(
"error")
457 newTable = Table.read(filename, hdu=extNum)
458 tableList.append(newTable)
463 for table
in tableList:
464 for k, v
in table.meta.items():
465 if isinstance(v, fits.card.Undefined):
469 return calibClass.fromTable(tableList, **kwargs)
472 """Write calibration data to a FITS file.
477 Filename to write data to.
482 The name of the file used to write the data.
485 tableList = self.toTabletoTable()
486 with warnings.catch_warnings():
487 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
488 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
489 astropyList.insert(0, fits.PrimaryHDU())
491 writer = fits.HDUList(astropyList)
492 writer.writeto(filename, overwrite=
True)
496 """Modify the calibration parameters to match the supplied detector.
501 Detector to use to set parameters from.
506 This needs to be implemented by subclasses
for each
509 raise NotImplementedError(
"Must be implemented by subclass.")
513 """Construct a calibration from a dictionary of properties.
515 Must be implemented by the specific calibration subclasses.
520 Dictionary of properties.
521 kwargs : `dict` or collections.abc.Mapping`, optional
522 Set of key=value options.
526 calib : `lsst.ip.isr.CalibType`
527 Constructed calibration.
531 NotImplementedError :
532 Raised
if not implemented.
534 raise NotImplementedError(
"Must be implemented by subclass.")
537 """Return a dictionary containing the calibration properties.
539 The dictionary should be able to be round-tripped through
545 Dictionary of properties.
549 NotImplementedError :
550 Raised if not implemented.
552 raise NotImplementedError(
"Must be implemented by subclass.")
556 """Construct a calibration from a dictionary of properties.
558 Must be implemented by the specific calibration subclasses.
562 tableList : `list` [`lsst.afw.table.Table`]
563 List of tables of properties.
564 kwargs : `dict` or collections.abc.Mapping`, optional
565 Set of key=value options.
569 calib : `lsst.ip.isr.CalibType`
570 Constructed calibration.
574 NotImplementedError :
575 Raised
if not implemented.
577 raise NotImplementedError(
"Must be implemented by subclass.")
580 """Return a list of tables containing the calibration properties.
582 The table list should be able to be round-tripped through
587 tableList : `list` [`lsst.afw.table.Table`]
588 List of tables of properties.
592 NotImplementedError :
593 Raised if not implemented.
595 raise NotImplementedError(
"Must be implemented by subclass.")
598 """Validate that this calibration is defined and can be used.
602 other : `object`, optional
603 Thing to validate against.
608 Returns true if the calibration
is valid
and appropriate.
613 """Method to apply the calibration to the target object.
618 Thing to validate against.
623 Returns true if the calibration was applied correctly.
627 NotImplementedError :
628 Raised
if not implemented.
630 raise NotImplementedError(
"Must be implemented by subclass.")
634 """Class for the provenance of data used to construct calibration.
636 Provenance is not really a calibration, but we would like to
637 record this when constructing the calibration,
and it provides an
638 example of the base calibration
class.
642 instrument : `str`, optional
643 Name of the instrument the data was taken
with.
644 calibType : `str`, optional
645 Type of calibration this provenance was generated
for.
646 detectorName : `str`, optional
647 Name of the detector this calibration
is for.
648 detectorSerial : `str`, optional
649 Identifier
for the detector.
652 _OBSTYPE = "IsrProvenance"
665 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
668 return super().
__eq__(other)
671 """Update calibration metadata.
675 setDate : `bool, optional
676 Update the CALIBDATE fields in the metadata to the current
677 time. Defaults to
False.
678 kwargs : `dict`
or `collections.abc.Mapping`, optional
679 Other keyword parameters to set
in the metadata.
681 kwargs["calibType"] = self.
calibTypecalibType
685 """Update provenance from dataId List.
689 dataIdList : `list` [`lsst.daf.butler.DataId`]
690 List of dataIds used in generating this calibration.
692 for dataId
in dataIdList:
700 """Construct provenance from table list.
704 tableList : `list` [`lsst.afw.table.Table`]
705 List of tables to construct the provenance from.
710 The provenance defined
in the tables.
713 metadata = table.meta
715 inDict["metadata"] = metadata
716 inDict[
"calibType"] = metadata[
"calibType"]
717 inDict[
"dimensions"] = set()
718 inDict[
"dataIdList"] = list()
721 for colName
in table.columns:
722 schema[colName.lower()] = colName
723 inDict[
"dimensions"].add(colName.lower())
724 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
728 for dim
in sorted(inDict[
"dimensions"]):
729 entry[dim] = row[schema[dim]]
730 inDict[
"dataIdList"].append(entry)
736 """Construct provenance from a dictionary.
741 Dictionary of provenance parameters.
746 The provenance defined in the tables.
749 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
750 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
751 f
"found {dictionary['metadata']['OBSTYPE']}")
752 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
757 calib.calibType = dictionary[
"calibType"]
758 calib.dimensions = set(dictionary[
"dimensions"])
759 calib.dataIdList = dictionary[
"dataIdList"]
761 calib.updateMetadata()
765 """Return a dictionary containing the provenance information.
770 Dictionary of provenance.
777 outDict["metadata"] = metadata
780 outDict[
"detectorId"] = self.
_detectorId_detectorId
781 outDict[
"instrument"] = self.
_instrument_instrument
782 outDict[
"calibType"] = self.
calibTypecalibType
783 outDict[
"dimensions"] = list(self.
dimensionsdimensions)
784 outDict[
"dataIdList"] = self.
dataIdListdataIdList
789 """Return a list of tables containing the provenance.
791 This seems inefficient and slow, so this may
not be the best
792 way to store the data.
796 tableList : `list` [`lsst.afw.table.Table`]
797 List of tables containing the provenance information
803 catalog = Table(rows=self.
dataIdListdataIdList,
805 filteredMetadata = {k: v
for k, v
in self.
getMetadatagetMetadata().
toDict().items()
if v
is not None}
806 catalog.meta = filteredMetadata
807 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def writeText(self, filename, format="auto")
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def determineCalibClass(cls, metadata, message)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)