21__all__ = [
"IsrCalib",
"IsrProvenance"]
31from astropy.table
import Table
32from astropy.io
import fits
35from lsst.utils.introspection
import get_full_type_name
36from lsst.utils
import doImport
40 """Generic calibration type.
42 Subclasses must implement the toDict, fromDict, toTable, fromTable
43 methods that allow the calibration information to be converted
44 from dictionaries and afw tables. This will allow the calibration
45 to be persisted using the base class read/write methods.
47 The validate method is intended to provide a common way to check
48 that the calibration is valid (internally consistent) and
49 appropriate (usable with the intended data). The apply method is
50 intended to allow the calibration to be applied in a consistent
55 camera : `lsst.afw.cameraGeom.Camera`, optional
56 Camera to extract metadata from.
57 detector : `lsst.afw.cameraGeom.Detector`, optional
58 Detector to extract metadata from.
59 log : `logging.Logger`, optional
66 def __init__(self, camera=None, detector=None, log=None, **kwargs):
90 "_detectorName",
"_detectorSerial",
"_detectorId",
91 "_filter",
"_calibId",
"_seqfile",
"_seqname",
"_seqcksum",
94 self.
log = log
if log
else logging.getLogger(__name__)
101 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
104 """Calibration equivalence.
106 Running ``calib.log.setLevel(0)`` enables debug statements to
107 identify problematic fields.
109 if not isinstance(other, self.
__class__):
110 self.
log.debug(
"Incorrect class type: %s %s", self.
__class__, other.__class__)
114 attrSelf = getattr(self, attr)
115 attrOther = getattr(other, attr)
117 if isinstance(attrSelf, dict):
119 if attrSelf.keys() != attrOther.keys():
120 self.
log.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
124 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
125 self.
log.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
132 if attrSelf[key] != attrOther[key]:
134 elif isinstance(attrSelf, np.ndarray):
136 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
137 self.
log.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
139 elif type(attrSelf) != type(attrOther):
140 if set([attrSelf, attrOther]) == set([
None,
""]):
143 self.
log.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
147 if attrSelf != attrOther:
148 self.
log.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
157 @requiredAttributes.setter
162 """Retrieve metadata associated with this calibration.
166 meta : `lsst.daf.base.PropertyList`
167 Metadata. The returned `~lsst.daf.base.PropertyList` can be
168 modified by the caller and the changes will be written to
174 """Store a copy of the supplied metadata with this calibration.
178 metadata : `lsst.daf.base.PropertyList`
179 Metadata to associate with the calibration. Will be copied and
180 overwrite existing metadata.
182 if metadata
is not None:
190 if isinstance(metadata, dict):
192 elif isinstance(metadata, PropertyList):
196 setCalibId=False, setCalibInfo=False, setDate=False,
198 """Update metadata keywords with new values.
202 camera : `lsst.afw.cameraGeom.Camera`, optional
203 Reference camera to use to set ``_instrument`` field.
204 detector : `lsst.afw.cameraGeom.Detector`, optional
205 Reference detector to use to set ``_detector*`` fields.
206 filterName : `str`, optional
207 Filter name to assign to this calibration.
208 setCalibId : `bool`, optional
209 Construct the ``_calibId`` field from other fields.
210 setCalibInfo : `bool`, optional
211 Set calibration parameters from metadata.
212 setDate : `bool`, optional
213 Ensure the metadata ``CALIBDATE`` fields are set to the current
215 kwargs : `dict` or `collections.abc.Mapping`, optional
216 Set of ``key=value`` pairs to assign to the metadata.
219 mdSupplemental = dict()
221 for k, v
in kwargs.items():
222 if isinstance(v, fits.card.Undefined):
247 date = datetime.datetime.now()
248 mdSupplemental[
"CALIBDATE"] = date.isoformat()
249 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
250 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
254 values.append(f
"instrument={self._instrument}")
if self.
_instrument else None
255 values.append(f
"raftName={self._raftName}")
if self.
_raftName else None
256 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName else None
257 values.append(f
"detector={self._detectorId}")
if self.
_detectorId else None
258 values.append(f
"filter={self._filter}")
if self.
_filter else None
260 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
261 values.append(f
"calibDate={calibDate}")
if calibDate
else None
276 self.
_metadata[
"CALIBCLS"] = get_full_type_name(self)
278 mdSupplemental.update(kwargs)
279 mdOriginal.update(mdSupplemental)
282 """Extract and unify metadata information.
287 Exposures or other calibrations to scan.
293 keywords = [
"SEQNAME",
"SEQFILE",
"SEQCKSUM",
"ODP",
"AP0_RC"]
296 for exp
in exposures:
298 expMeta = exp.getMetadata()
299 except AttributeError:
304 if metadata[key] != expMeta[key]:
305 self.
log.warning(
"Metadata mismatch! Have: %s Found %s",
306 metadata[key], expMeta[key])
308 metadata[key] = expMeta[key]
312 """Handle common keywords.
314 This isn't an ideal solution, but until all calibrations
315 expect to find everything in the metadata, they still need to
316 search through dictionaries.
320 dictionary : `dict` or `lsst.daf.base.PropertyList`
321 Source for the common keywords.
326 Raised if the dictionary does not match the expected OBSTYPE.
329 def search(haystack, needles):
330 """Search dictionary 'haystack' for an entry in 'needles'
332 test = [haystack.get(x)
for x
in needles]
333 test = set([x
for x
in test
if x
is not None])
335 if "metadata" in haystack:
336 return search(haystack[
"metadata"], needles)
340 value = list(test)[0]
346 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
348 if "metadata" in dictionary:
349 metadata = dictionary[
"metadata"]
351 if self.
_OBSTYPE != metadata[
"OBSTYPE"]:
352 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
353 f
"found {metadata['OBSTYPE']}")
355 self.
_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
356 self.
_raftName = search(dictionary, [
"RAFTNAME"])
357 self.
_slotName = search(dictionary, [
"SLOTNAME"])
358 self.
_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
359 self.
_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
360 self.
_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
361 self.
_filter = search(dictionary, [
"FILTER",
"filterName"])
362 self.
_calibId = search(dictionary, [
"CALIB_ID"])
363 self.
_seqfile = search(dictionary, [
"SEQFILE"])
364 self.
_seqname = search(dictionary, [
"SEQNAME"])
365 self.
_seqcksum = search(dictionary, [
"SEQCKSUM"])
369 """Attempt to find calibration class in metadata.
373 metadata : `dict` or `lsst.daf.base.PropertyList`
374 Metadata possibly containing a calibration class entry.
376 Message to include in any errors.
380 calibClass : `object`
381 The class to use to read the file contents. Should be an
382 `lsst.ip.isr.IsrCalib` subclass.
387 Raised if the resulting calibClass is the base
388 `lsst.ip.isr.IsrClass` (which does not implement the
391 calibClassName = metadata.get(
"CALIBCLS")
392 calibClass = doImport(calibClassName)
if calibClassName
is not None else cls
393 if calibClass
is IsrCalib:
394 raise ValueError(f
"Cannot use base class to read calibration data: {message}")
399 """Read calibration representation from a yaml/ecsv file.
404 Name of the file containing the calibration definition.
405 kwargs : `dict` or collections.abc.Mapping`, optional
406 Set of key=value pairs to pass to the ``fromDict`` or
407 ``fromTable`` methods.
411 calib : `~lsst.ip.isr.IsrCalibType`
417 Raised if the filename does not end in ".ecsv" or ".yaml".
419 if filename.endswith((
".ecsv",
".ECSV")):
420 data = Table.read(filename, format=
"ascii.ecsv")
422 return calibClass.fromTable([data], **kwargs)
423 elif filename.endswith((
".yaml",
".YAML")):
424 with open(filename,
"r")
as f:
425 data = yaml.load(f, Loader=yaml.CLoader)
427 return calibClass.fromDict(data, **kwargs)
429 raise RuntimeError(f
"Unknown filename extension: {filename}")
432 """Write the calibration data to a text file.
437 Name of the file to write.
439 Format to write the file as. Supported values are:
440 ``"auto"`` : Determine filetype from filename.
441 ``"yaml"`` : Write as yaml.
442 ``"ecsv"`` : Write as ecsv.
447 The name of the file used to write the data. This may
448 differ from the input if the format is explicitly chosen.
453 Raised if filename does not end in a known extension, or
454 if all information cannot be written.
458 The file is written to YAML/ECSV format and will include any
461 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
463 path, ext = os.path.splitext(filename)
464 filename = path +
".yaml"
465 with open(filename,
"w")
as f:
466 yaml.dump(outDict, f)
467 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
469 if len(tableList) > 1:
472 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
475 path, ext = os.path.splitext(filename)
476 filename = path +
".ecsv"
477 table.write(filename, format=
"ascii.ecsv")
479 raise RuntimeError(f
"Attempt to write to a file {filename} "
480 "that does not end in '.yaml' or '.ecsv'")
486 """Read calibration data from a FITS file.
491 Filename to read data from.
492 kwargs : `dict` or collections.abc.Mapping`, optional
493 Set of key=value pairs to pass to the ``fromTable``
498 calib : `lsst.ip.isr.IsrCalib`
499 Calibration contained within the file.
502 tableList.append(Table.read(filename, hdu=1, mask_invalid=
False))
507 with warnings.catch_warnings():
508 warnings.simplefilter(
"error")
510 newTable = Table.read(filename, hdu=extNum, mask_invalid=
False)
511 tableList.append(newTable)
516 for table
in tableList:
517 for k, v
in table.meta.items():
518 if isinstance(v, fits.card.Undefined):
522 return calibClass.fromTable(tableList, **kwargs)
525 """Write calibration data to a FITS file.
530 Filename to write data to.
535 The name of the file used to write the data.
538 with warnings.catch_warnings():
539 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
540 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
541 astropyList.insert(0, fits.PrimaryHDU())
543 writer = fits.HDUList(astropyList)
544 writer.writeto(filename, overwrite=
True)
548 """Modify the calibration parameters to match the supplied detector.
552 detector : `lsst.afw.cameraGeom.Detector`
553 Detector to use to set parameters from.
558 Raised if not implemented by a subclass.
559 This needs to be implemented by subclasses for each
562 raise NotImplementedError(
"Must be implemented by subclass.")
566 """Construct a calibration from a dictionary of properties.
568 Must be implemented by the specific calibration subclasses.
573 Dictionary of properties.
574 kwargs : `dict` or collections.abc.Mapping`, optional
575 Set of key=value options.
579 calib : `lsst.ip.isr.CalibType`
580 Constructed calibration.
585 Raised if not implemented.
587 raise NotImplementedError(
"Must be implemented by subclass.")
590 """Return a dictionary containing the calibration properties.
592 The dictionary should be able to be round-tripped through
598 Dictionary of properties.
603 Raised if not implemented.
605 raise NotImplementedError(
"Must be implemented by subclass.")
609 """Construct a calibration from a dictionary of properties.
611 Must be implemented by the specific calibration subclasses.
615 tableList : `list` [`lsst.afw.table.Table`]
616 List of tables of properties.
617 kwargs : `dict` or collections.abc.Mapping`, optional
618 Set of key=value options.
622 calib : `lsst.ip.isr.CalibType`
623 Constructed calibration.
628 Raised if not implemented.
630 raise NotImplementedError(
"Must be implemented by subclass.")
633 """Return a list of tables containing the calibration properties.
635 The table list should be able to be round-tripped through
640 tableList : `list` [`lsst.afw.table.Table`]
641 List of tables of properties.
646 Raised if not implemented.
648 raise NotImplementedError(
"Must be implemented by subclass.")
651 """Validate that this calibration is defined and can be used.
655 other : `object`, optional
656 Thing to validate against.
661 Returns true if the calibration is valid and appropriate.
666 """Method to apply the calibration to the target object.
671 Thing to validate against.
676 Returns true if the calibration was applied correctly.
681 Raised if not implemented.
683 raise NotImplementedError(
"Must be implemented by subclass.")
687 """Class for the provenance of data used to construct calibration.
689 Provenance is not really a calibration, but we would like to
690 record this when constructing the calibration, and it provides an
691 example of the base calibration class.
695 instrument : `str`, optional
696 Name of the instrument the data was taken with.
697 calibType : `str`, optional
698 Type of calibration this provenance was generated for.
699 detectorName : `str`, optional
700 Name of the detector this calibration is for.
701 detectorSerial : `str`, optional
702 Identifier for the detector.
705 _OBSTYPE =
"IsrProvenance"
718 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
721 return super().
__eq__(other)
724 """Update calibration metadata.
728 setDate : `bool`, optional
729 Update the ``CALIBDATE`` fields in the metadata to the current
730 time. Defaults to False.
731 kwargs : `dict` or `collections.abc.Mapping`, optional
732 Other keyword parameters to set in the metadata.
738 """Update provenance from dataId List.
742 dataIdList : `list` [`lsst.daf.butler.DataId`]
743 List of dataIds used in generating this calibration.
745 for dataId
in dataIdList:
753 """Construct provenance from table list.
757 tableList : `list` [`lsst.afw.table.Table`]
758 List of tables to construct the provenance from.
762 provenance : `lsst.ip.isr.IsrProvenance`
763 The provenance defined in the tables.
766 metadata = table.meta
768 inDict[
"metadata"] = metadata
769 inDict[
"calibType"] = metadata[
"calibType"]
770 inDict[
"dimensions"] = set()
771 inDict[
"dataIdList"] = list()
774 for colName
in table.columns:
775 schema[colName.lower()] = colName
776 inDict[
"dimensions"].add(colName.lower())
777 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
781 for dim
in sorted(inDict[
"dimensions"]):
782 entry[dim] = row[schema[dim]]
783 inDict[
"dataIdList"].append(entry)
789 """Construct provenance from a dictionary.
794 Dictionary of provenance parameters.
798 provenance : `lsst.ip.isr.IsrProvenance`
799 The provenance defined in the tables.
802 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
803 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
804 f
"found {dictionary['metadata']['OBSTYPE']}")
805 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
810 calib.calibType = dictionary[
"calibType"]
811 calib.dimensions = set(dictionary[
"dimensions"])
812 calib.dataIdList = dictionary[
"dataIdList"]
814 calib.updateMetadata()
818 """Return a dictionary containing the provenance information.
823 Dictionary of provenance.
830 outDict[
"metadata"] = metadata
842 """Return a list of tables containing the provenance.
844 This seems inefficient and slow, so this may not be the best
845 way to store the data.
849 tableList : `list` [`lsst.afw.table.Table`]
850 List of tables containing the provenance information
857 filteredMetadata = {k: v
for k, v
in self.
getMetadata().
toDict().items()
if v
is not None}
858 catalog.meta = filteredMetadata
859 tableList.append(catalog)
fromDict(cls, dictionary, **kwargs)
writeText(self, filename, format="auto")
readText(cls, filename, **kwargs)
calibInfoFromDict(self, dictionary)
validate(self, other=None)
updateMetadataFromExposures(self, exposures)
writeFits(self, filename)
setMetadata(self, metadata)
determineCalibClass(cls, metadata, message)
__init__(self, camera=None, detector=None, log=None, **kwargs)
requiredAttributes(self, value)
fromDetector(self, detector)
updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
fromTable(cls, tableList, **kwargs)
readFits(cls, filename, **kwargs)
fromTable(cls, tableList)
fromDataIds(self, dataIdList)
__init__(self, calibType="unknown", **kwargs)
updateMetadata(self, setDate=False, **kwargs)
fromDict(cls, dictionary)