21__all__ = [
"IsrCalib",
"IsrProvenance"]
31from astropy.table
import Table
32from astropy.io
import fits
35from lsst.utils.introspection
import get_full_type_name
36from lsst.utils
import doImport
40 """Generic calibration type.
42 Subclasses must implement the toDict, fromDict, toTable, fromTable
43 methods that allow the calibration information to be converted
44 from dictionaries
and afw tables. This will allow the calibration
45 to be persisted using the base
class read/write methods.
47 The validate method
is intended to provide a common way to check
48 that the calibration
is valid (internally consistent)
and
49 appropriate (usable
with the intended data). The apply method
is
50 intended to allow the calibration to be applied
in a consistent
56 Camera to extract metadata
from.
58 Detector to extract metadata
from.
59 log : `logging.Logger`, optional
66 def __init__(self, camera=None, detector=None, log=None, **kwargs):
82 "_detectorName",
"_detectorSerial",
"_detectorId",
83 "_filter",
"_calibId",
"_metadata"])
85 self.
log = log
if log
else logging.getLogger(__name__)
92 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
95 """Calibration equivalence.
97 Running ``calib.log.setLevel(0)`` enables debug statements to
98 identify problematic fields.
100 if not isinstance(other, self.__class__):
101 self.
log.debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
105 attrSelf = getattr(self, attr)
106 attrOther = getattr(other, attr)
108 if isinstance(attrSelf, dict):
110 if attrSelf.keys() != attrOther.keys():
111 self.
log.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
116 self.
log.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
123 if attrSelf[key] != attrOther[key]:
125 elif isinstance(attrSelf, np.ndarray):
127 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
128 self.
log.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
130 elif type(attrSelf) != type(attrOther):
131 if set([attrSelf, attrOther]) == set([
None,
""]):
134 self.
log.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
138 if attrSelf != attrOther:
139 self.
log.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
148 @requiredAttributes.setter
153 """Retrieve metadata associated with this calibration.
159 modified by the caller and the changes will be written to
165 """Store a copy of the supplied metadata with this calibration.
170 Metadata to associate with the calibration. Will be copied
and
171 overwrite existing metadata.
173 if metadata
is not None:
181 if isinstance(metadata, dict):
183 elif isinstance(metadata, PropertyList):
187 setCalibId=False, setCalibInfo=False, setDate=False,
189 """Update metadata keywords with new values.
194 Reference camera to use to set ``_instrument`` field.
196 Reference detector to use to set ``_detector*`` fields.
197 filterName : `str`, optional
198 Filter name to assign to this calibration.
199 setCalibId : `bool`, optional
200 Construct the ``_calibId`` field from other fields.
201 setCalibInfo : `bool`, optional
202 Set calibration parameters
from metadata.
203 setDate : `bool`, optional
204 Ensure the metadata ``CALIBDATE`` fields are set to the current
206 kwargs : `dict`
or `collections.abc.Mapping`, optional
207 Set of ``key=value`` pairs to assign to the metadata.
210 mdSupplemental = dict()
212 for k, v
in kwargs.items():
213 if isinstance(v, fits.card.Undefined):
238 date = datetime.datetime.now()
239 mdSupplemental[
"CALIBDATE"] = date.isoformat()
240 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
241 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
245 values.append(f
"instrument={self._instrument}")
if self.
_instrument else None
246 values.append(f
"raftName={self._raftName}")
if self.
_raftName else None
247 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName else None
248 values.append(f
"detector={self._detectorId}")
if self.
_detectorId else None
249 values.append(f
"filter={self._filter}")
if self.
_filter else None
251 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
252 values.append(f
"calibDate={calibDate}")
if calibDate
else None
264 self.
_metadata[
"CALIBCLS"] = get_full_type_name(self)
266 mdSupplemental.update(kwargs)
267 mdOriginal.update(mdSupplemental)
270 """Handle common keywords.
272 This isn't an ideal solution, but until all calibrations
273 expect to find everything in the metadata, they still need to
274 search through dictionaries.
279 Source
for the common keywords.
284 Raised
if the dictionary does
not match the expected OBSTYPE.
287 def search(haystack, needles):
288 """Search dictionary 'haystack' for an entry in 'needles'
290 test = [haystack.get(x) for x
in needles]
291 test = set([x
for x
in test
if x
is not None])
293 if "metadata" in haystack:
294 return search(haystack[
"metadata"], needles)
298 value = list(test)[0]
304 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
306 if "metadata" in dictionary:
307 metadata = dictionary[
"metadata"]
309 if self.
_OBSTYPE != metadata[
"OBSTYPE"]:
310 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
311 f
"found {metadata['OBSTYPE']}")
313 self.
_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
314 self.
_raftName = search(dictionary, [
"RAFTNAME"])
315 self.
_slotName = search(dictionary, [
"SLOTNAME"])
316 self.
_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
317 self.
_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
318 self.
_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
319 self.
_filter = search(dictionary, [
"FILTER",
"filterName"])
320 self.
_calibId = search(dictionary, [
"CALIB_ID"])
324 """Attempt to find calibration class in metadata.
329 Metadata possibly containing a calibration
class entry.
331 Message to include
in any errors.
335 calibClass : `object`
336 The
class to use to read the file contents. Should be an
342 Raised
if the resulting calibClass
is the base
343 `lsst.ip.isr.IsrClass` (which does
not implement the
346 calibClassName = metadata.get("CALIBCLS")
347 calibClass = doImport(calibClassName)
if calibClassName
is not None else cls
348 if calibClass
is IsrCalib:
349 raise ValueError(f
"Cannot use base class to read calibration data: {message}")
354 """Read calibration representation from a yaml/ecsv file.
359 Name of the file containing the calibration definition.
360 kwargs : `dict` or collections.abc.Mapping`, optional
361 Set of key=value pairs to
pass to the ``fromDict``
or
362 ``fromTable`` methods.
366 calib : `~lsst.ip.isr.IsrCalibType`
372 Raised
if the filename does
not end
in ".ecsv" or ".yaml".
374 if filename.endswith((
".ecsv",
".ECSV")):
375 data = Table.read(filename, format=
"ascii.ecsv")
377 return calibClass.fromTable([data], **kwargs)
378 elif filename.endswith((
".yaml",
".YAML")):
379 with open(filename,
"r")
as f:
380 data = yaml.load(f, Loader=yaml.CLoader)
382 return calibClass.fromDict(data, **kwargs)
384 raise RuntimeError(f
"Unknown filename extension: {filename}")
387 """Write the calibration data to a text file.
392 Name of the file to write.
394 Format to write the file as. Supported values are:
395 ``
"auto"`` : Determine filetype
from filename.
396 ``
"yaml"`` : Write
as yaml.
397 ``
"ecsv"`` : Write
as ecsv.
402 The name of the file used to write the data. This may
403 differ
from the input
if the format
is explicitly chosen.
408 Raised
if filename does
not end
in a known extension,
or
409 if all information cannot be written.
413 The file
is written to YAML/ECSV format
and will include any
416 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
418 path, ext = os.path.splitext(filename)
419 filename = path +
".yaml"
420 with open(filename,
"w")
as f:
421 yaml.dump(outDict, f)
422 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
424 if len(tableList) > 1:
427 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
430 path, ext = os.path.splitext(filename)
431 filename = path +
".ecsv"
432 table.write(filename, format=
"ascii.ecsv")
434 raise RuntimeError(f
"Attempt to write to a file {filename} "
435 "that does not end in '.yaml' or '.ecsv'")
441 """Read calibration data from a FITS file.
446 Filename to read data from.
447 kwargs : `dict`
or collections.abc.Mapping`, optional
448 Set of key=value pairs to
pass to the ``fromTable``
454 Calibration contained within the file.
457 tableList.append(Table.read(filename, hdu=1))
462 with warnings.catch_warnings():
463 warnings.simplefilter(
"error")
465 newTable = Table.read(filename, hdu=extNum)
466 tableList.append(newTable)
471 for table
in tableList:
472 for k, v
in table.meta.items():
473 if isinstance(v, fits.card.Undefined):
477 return calibClass.fromTable(tableList, **kwargs)
480 """Write calibration data to a FITS file.
485 Filename to write data to.
490 The name of the file used to write the data.
493 with warnings.catch_warnings():
494 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
495 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
496 astropyList.insert(0, fits.PrimaryHDU())
498 writer = fits.HDUList(astropyList)
499 writer.writeto(filename, overwrite=
True)
503 """Modify the calibration parameters to match the supplied detector.
508 Detector to use to set parameters from.
513 Raised
if not implemented by a subclass.
514 This needs to be implemented by subclasses
for each
517 raise NotImplementedError(
"Must be implemented by subclass.")
521 """Construct a calibration from a dictionary of properties.
523 Must be implemented by the specific calibration subclasses.
528 Dictionary of properties.
529 kwargs : `dict` or collections.abc.Mapping`, optional
530 Set of key=value options.
534 calib : `lsst.ip.isr.CalibType`
535 Constructed calibration.
540 Raised
if not implemented.
542 raise NotImplementedError(
"Must be implemented by subclass.")
545 """Return a dictionary containing the calibration properties.
547 The dictionary should be able to be round-tripped through
553 Dictionary of properties.
558 Raised if not implemented.
560 raise NotImplementedError(
"Must be implemented by subclass.")
564 """Construct a calibration from a dictionary of properties.
566 Must be implemented by the specific calibration subclasses.
570 tableList : `list` [`lsst.afw.table.Table`]
571 List of tables of properties.
572 kwargs : `dict` or collections.abc.Mapping`, optional
573 Set of key=value options.
577 calib : `lsst.ip.isr.CalibType`
578 Constructed calibration.
583 Raised
if not implemented.
585 raise NotImplementedError(
"Must be implemented by subclass.")
588 """Return a list of tables containing the calibration properties.
590 The table list should be able to be round-tripped through
595 tableList : `list` [`lsst.afw.table.Table`]
596 List of tables of properties.
601 Raised if not implemented.
603 raise NotImplementedError(
"Must be implemented by subclass.")
606 """Validate that this calibration is defined and can be used.
610 other : `object`, optional
611 Thing to validate against.
616 Returns true if the calibration
is valid
and appropriate.
621 """Method to apply the calibration to the target object.
626 Thing to validate against.
631 Returns true if the calibration was applied correctly.
636 Raised
if not implemented.
638 raise NotImplementedError(
"Must be implemented by subclass.")
642 """Class for the provenance of data used to construct calibration.
644 Provenance is not really a calibration, but we would like to
645 record this when constructing the calibration,
and it provides an
646 example of the base calibration
class.
650 instrument : `str`, optional
651 Name of the instrument the data was taken
with.
652 calibType : `str`, optional
653 Type of calibration this provenance was generated
for.
654 detectorName : `str`, optional
655 Name of the detector this calibration
is for.
656 detectorSerial : `str`, optional
657 Identifier
for the detector.
660 _OBSTYPE = "IsrProvenance"
673 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
676 return super().
__eq__(other)
679 """Update calibration metadata.
683 setDate : `bool`, optional
684 Update the ``CALIBDATE`` fields in the metadata to the current
685 time. Defaults to
False.
686 kwargs : `dict`
or `collections.abc.Mapping`, optional
687 Other keyword parameters to set
in the metadata.
693 """Update provenance from dataId List.
697 dataIdList : `list` [`lsst.daf.butler.DataId`]
698 List of dataIds used in generating this calibration.
700 for dataId
in dataIdList:
708 """Construct provenance from table list.
712 tableList : `list` [`lsst.afw.table.Table`]
713 List of tables to construct the provenance from.
718 The provenance defined
in the tables.
721 metadata = table.meta
723 inDict["metadata"] = metadata
724 inDict[
"calibType"] = metadata[
"calibType"]
725 inDict[
"dimensions"] = set()
726 inDict[
"dataIdList"] = list()
729 for colName
in table.columns:
730 schema[colName.lower()] = colName
731 inDict[
"dimensions"].add(colName.lower())
732 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
736 for dim
in sorted(inDict[
"dimensions"]):
737 entry[dim] = row[schema[dim]]
738 inDict[
"dataIdList"].append(entry)
744 """Construct provenance from a dictionary.
749 Dictionary of provenance parameters.
754 The provenance defined in the tables.
757 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
758 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
759 f
"found {dictionary['metadata']['OBSTYPE']}")
760 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
765 calib.calibType = dictionary[
"calibType"]
766 calib.dimensions = set(dictionary[
"dimensions"])
767 calib.dataIdList = dictionary[
"dataIdList"]
769 calib.updateMetadata()
773 """Return a dictionary containing the provenance information.
778 Dictionary of provenance.
785 outDict["metadata"] = metadata
797 """Return a list of tables containing the provenance.
799 This seems inefficient and slow, so this may
not be the best
800 way to store the data.
804 tableList : `list` [`lsst.afw.table.Table`]
805 List of tables containing the provenance information
812 filteredMetadata = {k: v
for k, v
in self.
getMetadata().
toDict().items()
if v
is not None}
813 catalog.meta = filteredMetadata
814 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def writeText(self, filename, format="auto")
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def determineCalibClass(cls, metadata, message)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)