26 from astropy.table
import Table
27 from astropy.io
import fits
33 __all__ = [
"IsrCalib",
"IsrProvenance"]
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
63 def __init__(self, camera=None, detector=None, log=None, **kwargs):
79 '_detectorName',
'_detectorSerial',
'_detectorId',
80 '_filter',
'_calibId',
'_metadata'])
82 self.
loglog = log
if log
else Log.getLogger(__name__.partition(
".")[2])
86 self.
updateMetadataupdateMetadata(camera=camera, detector=detector)
89 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
92 """Calibration equivalence.
94 Subclasses will need to check specific sub-properties. The
95 default is only to check common entries.
97 if not isinstance(other, self.__class__):
101 if getattr(self, attr) != getattr(other, attr):
110 @requiredAttributes.setter
115 """Retrieve metadata associated with this calibration.
119 meta : `lsst.daf.base.PropertyList`
120 Metadata. The returned `~lsst.daf.base.PropertyList` can be
121 modified by the caller and the changes will be written to
127 """Store a copy of the supplied metadata with this calibration.
131 metadata : `lsst.daf.base.PropertyList`
132 Metadata to associate with the calibration. Will be copied and
133 overwrite existing metadata.
135 if metadata
is not None:
143 if isinstance(metadata, dict):
145 elif isinstance(metadata, PropertyList):
149 setCalibId=False, setCalibInfo=False, setDate=False,
151 """Update metadata keywords with new values.
155 camera : `lsst.afw.cameraGeom.Camera`, optional
156 Reference camera to use to set _instrument field.
157 detector : `lsst.afw.cameraGeom.Detector`, optional
158 Reference detector to use to set _detector* fields.
159 filterName : `str`, optional
160 Filter name to assign to this calibration.
161 setCalibId : `bool`, optional
162 Construct the _calibId field from other fields.
163 setCalibInfo : `bool`, optional
164 Set calibration parameters from metadata.
165 setDate : `bool`, optional
166 Ensure the metadata CALIBDATE fields are set to the current datetime.
167 kwargs : `dict` or `collections.abc.Mapping`, optional
168 Set of key=value pairs to assign to the metadata.
171 mdSupplemental = dict()
173 for k, v
in kwargs.items():
174 if isinstance(v, fits.card.Undefined):
196 self.
_filter_filter = filterName
199 date = datetime.datetime.now()
200 mdSupplemental[
'CALIBDATE'] = date.isoformat()
201 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat()
202 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat()
206 values.append(f
"instrument={self._instrument}")
if self.
_instrument_instrument
else None
207 values.append(f
"raftName={self._raftName}")
if self.
_raftName_raftName
else None
208 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName_detectorName
else None
209 values.append(f
"detector={self._detectorId}")
if self.
_detectorId_detectorId
else None
210 values.append(f
"filter={self._filter}")
if self.
_filter_filter
else None
212 calibDate = mdOriginal.get(
'CALIBDATE', mdSupplemental.get(
'CALIBDATE',
None))
213 values.append(f
"calibDate={calibDate}")
if calibDate
else None
215 self.
_calibId_calibId =
" ".join(values)
226 mdSupplemental.update(kwargs)
227 mdOriginal.update(mdSupplemental)
230 """Handle common keywords.
232 This isn't an ideal solution, but until all calibrations
233 expect to find everything in the metadata, they still need to
234 search through dictionaries.
238 dictionary : `dict` or `lsst.daf.base.PropertyList`
239 Source for the common keywords.
244 Raised if the dictionary does not match the expected OBSTYPE.
248 def search(haystack, needles):
249 """Search dictionary 'haystack' for an entry in 'needles'
251 test = [haystack.get(x)
for x
in needles]
252 test = set([x
for x
in test
if x
is not None])
254 if 'metadata' in haystack:
255 return search(haystack[
'metadata'], needles)
259 value = list(test)[0]
265 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
267 if 'metadata' in dictionary:
268 metadata = dictionary[
'metadata']
270 if self.
_OBSTYPE_OBSTYPE != metadata[
'OBSTYPE']:
271 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
272 f
"found {metadata['OBSTYPE']}")
274 self.
_instrument_instrument = search(dictionary, [
'INSTRUME',
'instrument'])
275 self.
_raftName_raftName = search(dictionary, [
'RAFTNAME'])
276 self.
_slotName_slotName = search(dictionary, [
'SLOTNAME'])
277 self.
_detectorId_detectorId = search(dictionary, [
'DETECTOR',
'detectorId'])
278 self.
_detectorName_detectorName = search(dictionary, [
'DET_NAME',
'DETECTOR_NAME',
'detectorName'])
279 self.
_detectorSerial_detectorSerial = search(dictionary, [
'DET_SER',
'DETECTOR_SERIAL',
'detectorSerial'])
280 self.
_filter_filter = search(dictionary, [
'FILTER',
'filterName'])
281 self.
_calibId_calibId = search(dictionary, [
'CALIB_ID'])
285 """Read calibration representation from a yaml/ecsv file.
290 Name of the file containing the calibration definition.
291 kwargs : `dict` or collections.abc.Mapping`, optional
292 Set of key=value pairs to pass to the ``fromDict`` or
293 ``fromTable`` methods.
297 calib : `~lsst.ip.isr.IsrCalibType`
303 Raised if the filename does not end in ".ecsv" or ".yaml".
305 if filename.endswith((
".ecsv",
".ECSV")):
306 data = Table.read(filename, format=
'ascii.ecsv')
307 return cls.
fromTablefromTable([data], **kwargs)
309 elif filename.endswith((
".yaml",
".YAML")):
310 with open(filename,
'r')
as f:
311 data = yaml.load(f, Loader=yaml.CLoader)
312 return cls.
fromDictfromDict(data, **kwargs)
314 raise RuntimeError(f
"Unknown filename extension: {filename}")
317 """Write the calibration data to a text file.
322 Name of the file to write.
324 Format to write the file as. Supported values are:
325 ``"auto"`` : Determine filetype from filename.
326 ``"yaml"`` : Write as yaml.
327 ``"ecsv"`` : Write as ecsv.
331 The name of the file used to write the data. This may
332 differ from the input if the format is explicitly chosen.
337 Raised if filename does not end in a known extension, or
338 if all information cannot be written.
342 The file is written to YAML/ECSV format and will include any
346 if format ==
'yaml' or (format ==
'auto' and filename.lower().endswith((
".yaml",
".YAML"))):
347 outDict = self.
toDicttoDict()
348 path, ext = os.path.splitext(filename)
349 filename = path +
".yaml"
350 with open(filename,
'w')
as f:
351 yaml.dump(outDict, f)
352 elif format ==
'ecsv' or (format ==
'auto' and filename.lower().endswith((
".ecsv",
".ECSV"))):
353 tableList = self.
toTabletoTable()
354 if len(tableList) > 1:
357 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
360 path, ext = os.path.splitext(filename)
361 filename = path +
".ecsv"
362 table.write(filename, format=
"ascii.ecsv")
364 raise RuntimeError(f
"Attempt to write to a file {filename} "
365 "that does not end in '.yaml' or '.ecsv'")
371 """Read calibration data from a FITS file.
376 Filename to read data from.
377 kwargs : `dict` or collections.abc.Mapping`, optional
378 Set of key=value pairs to pass to the ``fromTable``
383 calib : `lsst.ip.isr.IsrCalib`
384 Calibration contained within the file.
387 tableList.append(Table.read(filename, hdu=1))
392 with warnings.catch_warnings():
393 warnings.simplefilter(
"error")
395 newTable = Table.read(filename, hdu=extNum)
396 tableList.append(newTable)
401 for table
in tableList:
402 for k, v
in table.meta.items():
403 if isinstance(v, fits.card.Undefined):
406 return cls.
fromTablefromTable(tableList, **kwargs)
409 """Write calibration data to a FITS file.
414 Filename to write data to.
419 The name of the file used to write the data.
422 tableList = self.
toTabletoTable()
423 with warnings.catch_warnings():
424 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
425 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
426 astropyList.insert(0, fits.PrimaryHDU())
428 writer = fits.HDUList(astropyList)
429 writer.writeto(filename, overwrite=
True)
433 """Modify the calibration parameters to match the supplied detector.
437 detector : `lsst.afw.cameraGeom.Detector`
438 Detector to use to set parameters from.
443 This needs to be implemented by subclasses for each
446 raise NotImplementedError(
"Must be implemented by subclass.")
450 """Construct a calibration from a dictionary of properties.
452 Must be implemented by the specific calibration subclasses.
457 Dictionary of properties.
458 kwargs : `dict` or collections.abc.Mapping`, optional
459 Set of key=value options.
463 calib : `lsst.ip.isr.CalibType`
464 Constructed calibration.
468 NotImplementedError :
469 Raised if not implemented.
471 raise NotImplementedError(
"Must be implemented by subclass.")
474 """Return a dictionary containing the calibration properties.
476 The dictionary should be able to be round-tripped through
482 Dictionary of properties.
486 NotImplementedError :
487 Raised if not implemented.
489 raise NotImplementedError(
"Must be implemented by subclass.")
493 """Construct a calibration from a dictionary of properties.
495 Must be implemented by the specific calibration subclasses.
499 tableList : `list` [`lsst.afw.table.Table`]
500 List of tables of properties.
501 kwargs : `dict` or collections.abc.Mapping`, optional
502 Set of key=value options.
506 calib : `lsst.ip.isr.CalibType`
507 Constructed calibration.
511 NotImplementedError :
512 Raised if not implemented.
514 raise NotImplementedError(
"Must be implemented by subclass.")
517 """Return a list of tables containing the calibration properties.
519 The table list should be able to be round-tripped through
524 tableList : `list` [`lsst.afw.table.Table`]
525 List of tables of properties.
529 NotImplementedError :
530 Raised if not implemented.
532 raise NotImplementedError(
"Must be implemented by subclass.")
535 """Validate that this calibration is defined and can be used.
539 other : `object`, optional
540 Thing to validate against.
545 Returns true if the calibration is valid and appropriate.
550 """Method to apply the calibration to the target object.
555 Thing to validate against.
560 Returns true if the calibration was applied correctly.
564 NotImplementedError :
565 Raised if not implemented.
567 raise NotImplementedError(
"Must be implemented by subclass.")
571 """Class for the provenance of data used to construct calibration.
573 Provenance is not really a calibration, but we would like to
574 record this when constructing the calibration, and it provides an
575 example of the base calibration class.
579 instrument : `str`, optional
580 Name of the instrument the data was taken with.
581 calibType : `str`, optional
582 Type of calibration this provenance was generated for.
583 detectorName : `str`, optional
584 Name of the detector this calibration is for.
585 detectorSerial : `str`, optional
586 Identifier for the detector.
589 _OBSTYPE =
'IsrProvenance'
602 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
605 return super().
__eq__(other)
608 """Update calibration metadata.
612 setDate : `bool, optional
613 Update the CALIBDATE fields in the metadata to the current
614 time. Defaults to False.
615 kwargs : `dict` or `collections.abc.Mapping`, optional
616 Other keyword parameters to set in the metadata.
618 kwargs[
'calibType'] = self.
calibTypecalibType
622 """Update provenance from dataId List.
626 dataIdList : `list` [`lsst.daf.butler.DataId`]
627 List of dataIds used in generating this calibration.
629 for dataId
in dataIdList:
637 """Construct provenance from table list.
641 tableList : `list` [`lsst.afw.table.Table`]
642 List of tables to construct the provenance from.
646 provenance : `lsst.ip.isr.IsrProvenance`
647 The provenance defined in the tables.
650 metadata = table.meta
652 inDict[
'metadata'] = metadata
653 inDict[
'calibType'] = metadata[
'calibType']
654 inDict[
'dimensions'] = set()
655 inDict[
'dataIdList'] = list()
658 for colName
in table.columns:
659 schema[colName.lower()] = colName
660 inDict[
'dimensions'].add(colName.lower())
661 inDict[
'dimensions'] = sorted(inDict[
'dimensions'])
665 for dim
in sorted(inDict[
'dimensions']):
666 entry[dim] = row[schema[dim]]
667 inDict[
'dataIdList'].append(entry)
673 """Construct provenance from a dictionary.
678 Dictionary of provenance parameters.
682 provenance : `lsst.ip.isr.IsrProvenance`
683 The provenance defined in the tables.
686 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
687 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
688 f
"found {dictionary['metadata']['OBSTYPE']}")
689 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
'metadata'])
694 calib.calibType = dictionary[
'calibType']
695 calib.dimensions = set(dictionary[
'dimensions'])
696 calib.dataIdList = dictionary[
'dataIdList']
698 calib.updateMetadata()
702 """Return a dictionary containing the provenance information.
707 Dictionary of provenance.
714 outDict[
'metadata'] = metadata
717 outDict[
'detectorId'] = self.
_detectorId_detectorId
718 outDict[
'instrument'] = self.
_instrument_instrument
719 outDict[
'calibType'] = self.
calibTypecalibType
720 outDict[
'dimensions'] = list(self.
dimensionsdimensions)
721 outDict[
'dataIdList'] = self.
dataIdListdataIdList
726 """Return a list of tables containing the provenance.
728 This seems inefficient and slow, so this may not be the best
729 way to store the data.
733 tableList : `list` [`lsst.afw.table.Table`]
734 List of tables containing the provenance information
740 catalog = Table(rows=self.
dataIdListdataIdList,
742 filteredMetadata = {k: v
for k, v
in self.
getMetadatagetMetadata().
toDict().items()
if v
is not None}
743 catalog.meta = filteredMetadata
744 tableList.append(catalog)
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def writeText(self, filename, format='auto')
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)