28 from astropy.table
import Table
29 from astropy.io
import fits
35 __all__ = [
"IsrCalib",
"IsrProvenance"]
39 """Generic calibration type.
41 Subclasses must implement the toDict, fromDict, toTable, fromTable
42 methods that allow the calibration information to be converted
43 from dictionaries and afw tables. This will allow the calibration
44 to be persisted using the base class read/write methods.
46 The validate method is intended to provide a common way to check
47 that the calibration is valid (internally consistent) and
48 appropriate (usable with the intended data). The apply method is
49 intended to allow the calibration to be applied in a consistent
54 camera : `lsst.afw.cameraGeom.Camera`, optional
55 Camera to extract metadata from.
56 detector : `lsst.afw.cameraGeom.Detector`, optional
57 Detector to extract metadata from.
58 log : `lsst.log.Log`, optional
65 def __init__(self, camera=None, detector=None, log=None, **kwargs):
81 '_detectorName',
'_detectorSerial',
'_detectorId',
82 '_filter',
'_calibId',
'_metadata'])
84 self.
loglog = log
if log
else Log.getLogger(__name__.partition(
".")[2])
88 self.
updateMetadataupdateMetadata(camera=camera, detector=detector)
91 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
94 """Calibration equivalence.
96 Running ``calib.log.setLevel(0)`` enables debug statements to
97 identify problematic fields.
99 if not isinstance(other, self.__class__):
100 self.
loglog.debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
104 attrSelf = getattr(self, attr)
105 attrOther = getattr(other, attr)
107 if isinstance(attrSelf, dict):
109 if attrSelf.keys() != attrOther.keys():
110 self.
loglog.debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
113 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
114 self.
loglog.debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
116 elif isinstance(attrSelf, np.ndarray):
118 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
119 self.
loglog.debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
121 elif type(attrSelf) != type(attrOther):
122 if set([attrSelf, attrOther]) == set([
None,
""]):
125 self.
loglog.debug(
"Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
129 if attrSelf != attrOther:
130 self.
loglog.debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
139 @requiredAttributes.setter
144 """Retrieve metadata associated with this calibration.
148 meta : `lsst.daf.base.PropertyList`
149 Metadata. The returned `~lsst.daf.base.PropertyList` can be
150 modified by the caller and the changes will be written to
156 """Store a copy of the supplied metadata with this calibration.
160 metadata : `lsst.daf.base.PropertyList`
161 Metadata to associate with the calibration. Will be copied and
162 overwrite existing metadata.
164 if metadata
is not None:
172 if isinstance(metadata, dict):
174 elif isinstance(metadata, PropertyList):
178 setCalibId=False, setCalibInfo=False, setDate=False,
180 """Update metadata keywords with new values.
184 camera : `lsst.afw.cameraGeom.Camera`, optional
185 Reference camera to use to set _instrument field.
186 detector : `lsst.afw.cameraGeom.Detector`, optional
187 Reference detector to use to set _detector* fields.
188 filterName : `str`, optional
189 Filter name to assign to this calibration.
190 setCalibId : `bool`, optional
191 Construct the _calibId field from other fields.
192 setCalibInfo : `bool`, optional
193 Set calibration parameters from metadata.
194 setDate : `bool`, optional
195 Ensure the metadata CALIBDATE fields are set to the current datetime.
196 kwargs : `dict` or `collections.abc.Mapping`, optional
197 Set of key=value pairs to assign to the metadata.
200 mdSupplemental = dict()
202 for k, v
in kwargs.items():
203 if isinstance(v, fits.card.Undefined):
225 self.
_filter_filter = filterName
228 date = datetime.datetime.now()
229 mdSupplemental[
'CALIBDATE'] = date.isoformat()
230 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat()
231 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat()
235 values.append(f
"instrument={self._instrument}")
if self.
_instrument_instrument
else None
236 values.append(f
"raftName={self._raftName}")
if self.
_raftName_raftName
else None
237 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName_detectorName
else None
238 values.append(f
"detector={self._detectorId}")
if self.
_detectorId_detectorId
else None
239 values.append(f
"filter={self._filter}")
if self.
_filter_filter
else None
241 calibDate = mdOriginal.get(
'CALIBDATE', mdSupplemental.get(
'CALIBDATE',
None))
242 values.append(f
"calibDate={calibDate}")
if calibDate
else None
244 self.
_calibId_calibId =
" ".join(values)
255 mdSupplemental.update(kwargs)
256 mdOriginal.update(mdSupplemental)
259 """Handle common keywords.
261 This isn't an ideal solution, but until all calibrations
262 expect to find everything in the metadata, they still need to
263 search through dictionaries.
267 dictionary : `dict` or `lsst.daf.base.PropertyList`
268 Source for the common keywords.
273 Raised if the dictionary does not match the expected OBSTYPE.
277 def search(haystack, needles):
278 """Search dictionary 'haystack' for an entry in 'needles'
280 test = [haystack.get(x)
for x
in needles]
281 test = set([x
for x
in test
if x
is not None])
283 if 'metadata' in haystack:
284 return search(haystack[
'metadata'], needles)
288 value = list(test)[0]
294 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
296 if 'metadata' in dictionary:
297 metadata = dictionary[
'metadata']
299 if self.
_OBSTYPE_OBSTYPE != metadata[
'OBSTYPE']:
300 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
301 f
"found {metadata['OBSTYPE']}")
303 self.
_instrument_instrument = search(dictionary, [
'INSTRUME',
'instrument'])
304 self.
_raftName_raftName = search(dictionary, [
'RAFTNAME'])
305 self.
_slotName_slotName = search(dictionary, [
'SLOTNAME'])
306 self.
_detectorId_detectorId = search(dictionary, [
'DETECTOR',
'detectorId'])
307 self.
_detectorName_detectorName = search(dictionary, [
'DET_NAME',
'DETECTOR_NAME',
'detectorName'])
308 self.
_detectorSerial_detectorSerial = search(dictionary, [
'DET_SER',
'DETECTOR_SERIAL',
'detectorSerial'])
309 self.
_filter_filter = search(dictionary, [
'FILTER',
'filterName'])
310 self.
_calibId_calibId = search(dictionary, [
'CALIB_ID'])
314 """Read calibration representation from a yaml/ecsv file.
319 Name of the file containing the calibration definition.
320 kwargs : `dict` or collections.abc.Mapping`, optional
321 Set of key=value pairs to pass to the ``fromDict`` or
322 ``fromTable`` methods.
326 calib : `~lsst.ip.isr.IsrCalibType`
332 Raised if the filename does not end in ".ecsv" or ".yaml".
334 if filename.endswith((
".ecsv",
".ECSV")):
335 data = Table.read(filename, format=
'ascii.ecsv')
336 return cls.
fromTablefromTable([data], **kwargs)
338 elif filename.endswith((
".yaml",
".YAML")):
339 with open(filename,
'r')
as f:
340 data = yaml.load(f, Loader=yaml.CLoader)
341 return cls.
fromDictfromDict(data, **kwargs)
343 raise RuntimeError(f
"Unknown filename extension: {filename}")
346 """Write the calibration data to a text file.
351 Name of the file to write.
353 Format to write the file as. Supported values are:
354 ``"auto"`` : Determine filetype from filename.
355 ``"yaml"`` : Write as yaml.
356 ``"ecsv"`` : Write as ecsv.
360 The name of the file used to write the data. This may
361 differ from the input if the format is explicitly chosen.
366 Raised if filename does not end in a known extension, or
367 if all information cannot be written.
371 The file is written to YAML/ECSV format and will include any
375 if format ==
'yaml' or (format ==
'auto' and filename.lower().endswith((
".yaml",
".YAML"))):
376 outDict = self.
toDicttoDict()
377 path, ext = os.path.splitext(filename)
378 filename = path +
".yaml"
379 with open(filename,
'w')
as f:
380 yaml.dump(outDict, f)
381 elif format ==
'ecsv' or (format ==
'auto' and filename.lower().endswith((
".ecsv",
".ECSV"))):
382 tableList = self.
toTabletoTable()
383 if len(tableList) > 1:
386 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
389 path, ext = os.path.splitext(filename)
390 filename = path +
".ecsv"
391 table.write(filename, format=
"ascii.ecsv")
393 raise RuntimeError(f
"Attempt to write to a file {filename} "
394 "that does not end in '.yaml' or '.ecsv'")
400 """Read calibration data from a FITS file.
405 Filename to read data from.
406 kwargs : `dict` or collections.abc.Mapping`, optional
407 Set of key=value pairs to pass to the ``fromTable``
412 calib : `lsst.ip.isr.IsrCalib`
413 Calibration contained within the file.
416 tableList.append(Table.read(filename, hdu=1))
421 with warnings.catch_warnings():
422 warnings.simplefilter(
"error")
424 newTable = Table.read(filename, hdu=extNum)
425 tableList.append(newTable)
430 for table
in tableList:
431 for k, v
in table.meta.items():
432 if isinstance(v, fits.card.Undefined):
435 return cls.
fromTablefromTable(tableList, **kwargs)
438 """Write calibration data to a FITS file.
443 Filename to write data to.
448 The name of the file used to write the data.
451 tableList = self.
toTabletoTable()
452 with warnings.catch_warnings():
453 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
454 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
455 astropyList.insert(0, fits.PrimaryHDU())
457 writer = fits.HDUList(astropyList)
458 writer.writeto(filename, overwrite=
True)
462 """Modify the calibration parameters to match the supplied detector.
466 detector : `lsst.afw.cameraGeom.Detector`
467 Detector to use to set parameters from.
472 This needs to be implemented by subclasses for each
475 raise NotImplementedError(
"Must be implemented by subclass.")
479 """Construct a calibration from a dictionary of properties.
481 Must be implemented by the specific calibration subclasses.
486 Dictionary of properties.
487 kwargs : `dict` or collections.abc.Mapping`, optional
488 Set of key=value options.
492 calib : `lsst.ip.isr.CalibType`
493 Constructed calibration.
497 NotImplementedError :
498 Raised if not implemented.
500 raise NotImplementedError(
"Must be implemented by subclass.")
503 """Return a dictionary containing the calibration properties.
505 The dictionary should be able to be round-tripped through
511 Dictionary of properties.
515 NotImplementedError :
516 Raised if not implemented.
518 raise NotImplementedError(
"Must be implemented by subclass.")
522 """Construct a calibration from a dictionary of properties.
524 Must be implemented by the specific calibration subclasses.
528 tableList : `list` [`lsst.afw.table.Table`]
529 List of tables of properties.
530 kwargs : `dict` or collections.abc.Mapping`, optional
531 Set of key=value options.
535 calib : `lsst.ip.isr.CalibType`
536 Constructed calibration.
540 NotImplementedError :
541 Raised if not implemented.
543 raise NotImplementedError(
"Must be implemented by subclass.")
546 """Return a list of tables containing the calibration properties.
548 The table list should be able to be round-tripped through
553 tableList : `list` [`lsst.afw.table.Table`]
554 List of tables of properties.
558 NotImplementedError :
559 Raised if not implemented.
561 raise NotImplementedError(
"Must be implemented by subclass.")
564 """Validate that this calibration is defined and can be used.
568 other : `object`, optional
569 Thing to validate against.
574 Returns true if the calibration is valid and appropriate.
579 """Method to apply the calibration to the target object.
584 Thing to validate against.
589 Returns true if the calibration was applied correctly.
593 NotImplementedError :
594 Raised if not implemented.
596 raise NotImplementedError(
"Must be implemented by subclass.")
600 """Class for the provenance of data used to construct calibration.
602 Provenance is not really a calibration, but we would like to
603 record this when constructing the calibration, and it provides an
604 example of the base calibration class.
608 instrument : `str`, optional
609 Name of the instrument the data was taken with.
610 calibType : `str`, optional
611 Type of calibration this provenance was generated for.
612 detectorName : `str`, optional
613 Name of the detector this calibration is for.
614 detectorSerial : `str`, optional
615 Identifier for the detector.
618 _OBSTYPE =
'IsrProvenance'
631 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
634 return super().
__eq__(other)
637 """Update calibration metadata.
641 setDate : `bool, optional
642 Update the CALIBDATE fields in the metadata to the current
643 time. Defaults to False.
644 kwargs : `dict` or `collections.abc.Mapping`, optional
645 Other keyword parameters to set in the metadata.
647 kwargs[
'calibType'] = self.
calibTypecalibType
651 """Update provenance from dataId List.
655 dataIdList : `list` [`lsst.daf.butler.DataId`]
656 List of dataIds used in generating this calibration.
658 for dataId
in dataIdList:
666 """Construct provenance from table list.
670 tableList : `list` [`lsst.afw.table.Table`]
671 List of tables to construct the provenance from.
675 provenance : `lsst.ip.isr.IsrProvenance`
676 The provenance defined in the tables.
679 metadata = table.meta
681 inDict[
'metadata'] = metadata
682 inDict[
'calibType'] = metadata[
'calibType']
683 inDict[
'dimensions'] = set()
684 inDict[
'dataIdList'] = list()
687 for colName
in table.columns:
688 schema[colName.lower()] = colName
689 inDict[
'dimensions'].add(colName.lower())
690 inDict[
'dimensions'] = sorted(inDict[
'dimensions'])
694 for dim
in sorted(inDict[
'dimensions']):
695 entry[dim] = row[schema[dim]]
696 inDict[
'dataIdList'].append(entry)
702 """Construct provenance from a dictionary.
707 Dictionary of provenance parameters.
711 provenance : `lsst.ip.isr.IsrProvenance`
712 The provenance defined in the tables.
715 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
716 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
717 f
"found {dictionary['metadata']['OBSTYPE']}")
718 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
'metadata'])
723 calib.calibType = dictionary[
'calibType']
724 calib.dimensions = set(dictionary[
'dimensions'])
725 calib.dataIdList = dictionary[
'dataIdList']
727 calib.updateMetadata()
731 """Return a dictionary containing the provenance information.
736 Dictionary of provenance.
743 outDict[
'metadata'] = metadata
746 outDict[
'detectorId'] = self.
_detectorId_detectorId
747 outDict[
'instrument'] = self.
_instrument_instrument
748 outDict[
'calibType'] = self.
calibTypecalibType
749 outDict[
'dimensions'] = list(self.
dimensionsdimensions)
750 outDict[
'dataIdList'] = self.
dataIdListdataIdList
755 """Return a list of tables containing the provenance.
757 This seems inefficient and slow, so this may not be the best
758 way to store the data.
762 tableList : `list` [`lsst.afw.table.Table`]
763 List of tables containing the provenance information
769 catalog = Table(rows=self.
dataIdListdataIdList,
771 filteredMetadata = {k: v
for k, v
in self.
getMetadatagetMetadata().
toDict().items()
if v
is not None}
772 catalog.meta = filteredMetadata
773 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def writeText(self, filename, format='auto')
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)