26 from astropy.table
import Table
27 from astropy.io
import fits
33 __all__ = [
"IsrCalib",
"IsrProvenance"]
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
63 def __init__(self, camera=None, detector=None, log=None, **kwargs):
79 '_detectorName',
'_detectorSerial',
'_detectorId',
80 '_filter',
'_calibId',
'_metadata'])
82 self.
log = log
if log
else Log.getLogger(__name__.partition(
".")[2])
89 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
92 """Calibration equivalence.
94 Subclasses will need to check specific sub-properties. The
95 default is only to check common entries.
97 if not isinstance(other, self.__class__):
101 if getattr(self, attr) != getattr(other, attr):
110 @requiredAttributes.setter
115 """Retrieve metadata associated with this calibration.
119 meta : `lsst.daf.base.PropertyList`
120 Metadata. The returned `~lsst.daf.base.PropertyList` can be
121 modified by the caller and the changes will be written to
127 """Store a copy of the supplied metadata with this calibration.
131 metadata : `lsst.daf.base.PropertyList`
132 Metadata to associate with the calibration. Will be copied and
133 overwrite existing metadata.
135 if metadata
is not None:
143 if isinstance(metadata, dict):
145 elif isinstance(metadata, PropertyList):
149 setCalibId=False, setCalibInfo=False, setDate=False,
151 """Update metadata keywords with new values.
155 camera : `lsst.afw.cameraGeom.Camera`, optional
156 Reference camera to use to set _instrument field.
157 detector : `lsst.afw.cameraGeom.Detector`, optional
158 Reference detector to use to set _detector* fields.
159 filterName : `str`, optional
160 Filter name to assign to this calibration.
161 setCalibId : `bool` optional
162 Construct the _calibId field from other fields.
163 setDate : `bool`, optional
164 Ensure the metadata CALIBDATE fields are set to the current datetime.
165 kwargs : `dict` or `collections.abc.Mapping`, optional
166 Set of key=value pairs to assign to the metadata.
169 mdSupplemental = dict()
191 date = datetime.datetime.now()
192 mdSupplemental[
'CALIBDATE'] = date.isoformat()
193 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat()
194 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat()
198 values.append(f
"instrument={self._instrument}")
if self.
_instrument else None
199 values.append(f
"raftName={self._raftName}")
if self.
_raftName else None
200 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName else None
201 values.append(f
"detector={self._detectorId}")
if self.
_detectorId else None
202 values.append(f
"filter={self._filter}")
if self.
_filter else None
204 calibDate = mdOriginal.get(
'CALIBDATE', mdSupplemental.get(
'CALIBDATE',
None))
205 values.append(f
"calibDate={calibDate}")
if calibDate
else None
218 mdSupplemental.update(kwargs)
219 mdOriginal.update(mdSupplemental)
222 """Handle common keywords.
224 This isn't an ideal solution, but until all calibrations
225 expect to find everything in the metadata, they still need to
226 search through dictionaries.
230 dictionary : `dict` or `lsst.daf.base.PropertyList`
231 Source for the common keywords.
236 Raised if the dictionary does not match the expected OBSTYPE.
240 def search(haystack, needles):
241 """Search dictionary 'haystack' for an entry in 'needles'
243 test = [haystack.get(x)
for x
in needles]
244 test = set([x
for x
in test
if x
is not None])
246 if 'metadata' in haystack:
247 return search(haystack[
'metadata'], needles)
251 value = list(test)[0]
257 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
259 if 'metadata' in dictionary:
260 metadata = dictionary[
'metadata']
262 if self.
_OBSTYPE != metadata[
'OBSTYPE']:
263 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
264 f
"found {metadata['OBSTYPE']}")
266 self.
_instrument = search(dictionary, [
'INSTRUME',
'instrument'])
267 self.
_raftName = search(dictionary, [
'RAFTNAME'])
268 self.
_slotName = search(dictionary, [
'SLOTNAME'])
269 self.
_detectorId = search(dictionary, [
'DETECTOR',
'detectorId'])
270 self.
_detectorName = search(dictionary, [
'DET_NAME',
'DETECTOR_NAME',
'detectorName'])
271 self.
_detectorSerial = search(dictionary, [
'DET_SER',
'DETECTOR_SERIAL',
'detectorSerial'])
272 self.
_filter = search(dictionary, [
'FILTER',
'filterName'])
273 self.
_calibId = search(dictionary, [
'CALIB_ID'])
277 """Read calibration representation from a yaml/ecsv file.
282 Name of the file containing the calibration definition.
286 calib : `~lsst.ip.isr.IsrCalibType`
292 Raised if the filename does not end in ".ecsv" or ".yaml".
294 if filename.endswith((
".ecsv",
".ECSV")):
295 data = Table.read(filename, format=
'ascii.ecsv')
298 elif filename.endswith((
".yaml",
".YAML")):
299 with open(filename,
'r')
as f:
300 data = yaml.load(f, Loader=yaml.CLoader)
303 raise RuntimeError(f
"Unknown filename extension: {filename}")
306 """Write the calibration data to a text file.
311 Name of the file to write.
313 Format to write the file as. Supported values are:
314 ``"auto"`` : Determine filetype from filename.
315 ``"yaml"`` : Write as yaml.
316 ``"ecsv"`` : Write as ecsv.
320 The name of the file used to write the data. This may
321 differ from the input if the format is explicitly chosen.
326 Raised if filename does not end in a known extension, or
327 if all information cannot be written.
331 The file is written to YAML/ECSV format and will include any
335 if format ==
'yaml' or (format ==
'auto' and filename.lower().endswith((
".yaml",
".YAML"))):
337 path, ext = os.path.splitext(filename)
338 filename = path +
".yaml"
339 with open(filename,
'w')
as f:
340 yaml.dump(outDict, f)
341 elif format ==
'ecsv' or (format ==
'auto' and filename.lower().endswith((
".ecsv",
".ECSV"))):
343 if len(tableList) > 1:
346 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
349 path, ext = os.path.splitext(filename)
350 filename = path +
".ecsv"
351 table.write(filename, format=
"ascii.ecsv")
353 raise RuntimeError(f
"Attempt to write to a file {filename} "
354 "that does not end in '.yaml' or '.ecsv'")
360 """Read calibration data from a FITS file.
365 Filename to read data from.
369 calib : `lsst.ip.isr.IsrCalib`
370 Calibration contained within the file.
373 tableList.append(Table.read(filename, hdu=1))
378 with warnings.catch_warnings():
379 warnings.simplefilter(
"error")
381 newTable = Table.read(filename, hdu=extNum)
382 tableList.append(newTable)
387 for table
in tableList:
388 for k, v
in table.meta.items():
389 if isinstance(v, fits.card.Undefined):
395 """Write calibration data to a FITS file.
400 Filename to write data to.
405 The name of the file used to write the data.
409 with warnings.catch_warnings():
410 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
411 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
412 astropyList.insert(0, fits.PrimaryHDU())
414 writer = fits.HDUList(astropyList)
415 writer.writeto(filename, overwrite=
True)
419 """Modify the calibration parameters to match the supplied detector.
423 detector : `lsst.afw.cameraGeom.Detector`
424 Detector to use to set parameters from.
429 This needs to be implemented by subclasses for each
432 raise NotImplementedError(
"Must be implemented by subclass.")
436 """Construct a calibration from a dictionary of properties.
438 Must be implemented by the specific calibration subclasses.
443 Dictionary of properties.
447 calib : `lsst.ip.isr.CalibType`
448 Constructed calibration.
452 NotImplementedError :
453 Raised if not implemented.
455 raise NotImplementedError(
"Must be implemented by subclass.")
458 """Return a dictionary containing the calibration properties.
460 The dictionary should be able to be round-tripped through
466 Dictionary of properties.
470 NotImplementedError :
471 Raised if not implemented.
473 raise NotImplementedError(
"Must be implemented by subclass.")
477 """Construct a calibration from a dictionary of properties.
479 Must be implemented by the specific calibration subclasses.
483 tableList : `list` [`lsst.afw.table.Table`]
484 List of tables of properties.
488 calib : `lsst.ip.isr.CalibType`
489 Constructed calibration.
493 NotImplementedError :
494 Raised if not implemented.
496 raise NotImplementedError(
"Must be implemented by subclass.")
499 """Return a list of tables containing the calibration properties.
501 The table list should be able to be round-tripped through
506 tableList : `list` [`lsst.afw.table.Table`]
507 List of tables of properties.
511 NotImplementedError :
512 Raised if not implemented.
514 raise NotImplementedError(
"Must be implemented by subclass.")
517 """Validate that this calibration is defined and can be used.
521 other : `object`, optional
522 Thing to validate against.
527 Returns true if the calibration is valid and appropriate.
532 """Method to apply the calibration to the target object.
537 Thing to validate against.
542 Returns true if the calibration was applied correctly.
546 NotImplementedError :
547 Raised if not implemented.
549 raise NotImplementedError(
"Must be implemented by subclass.")
553 """Class for the provenance of data used to construct calibration.
555 Provenance is not really a calibration, but we would like to
556 record this when constructing the calibration, and it provides an
557 example of the base calibration class.
561 instrument : `str`, optional
562 Name of the instrument the data was taken with.
563 calibType : `str`, optional
564 Type of calibration this provenance was generated for.
565 detectorName : `str`, optional
566 Name of the detector this calibration is for.
567 detectorSerial : `str`, optional
568 Identifier for the detector.
571 _OBSTYPE =
'IsrProvenance'
584 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
587 return super().
__eq__(other)
590 """Update calibration metadata.
594 setDate : `bool, optional
595 Update the CALIBDATE fields in the metadata to the current
596 time. Defaults to False.
597 kwargs : `dict` or `collections.abc.Mapping`, optional
598 Other keyword parameters to set in the metadata.
604 """Update provenance from dataId List.
608 dataIdList : `list` [`lsst.daf.butler.DataId`]
609 List of dataIds used in generating this calibration.
611 for dataId
in dataIdList:
619 """Construct provenance from table list.
623 tableList : `list` [`lsst.afw.table.Table`]
624 List of tables to construct the provenance from.
628 provenance : `lsst.ip.isr.IsrProvenance`
629 The provenance defined in the tables.
632 metadata = table.meta
634 inDict[
'metadata'] = metadata
635 inDict[
'calibType'] = metadata[
'calibType']
636 inDict[
'dimensions'] = set()
637 inDict[
'dataIdList'] = list()
640 for colName
in table.columns:
641 schema[colName.lower()] = colName
642 inDict[
'dimensions'].add(colName.lower())
643 inDict[
'dimensions'] = sorted(inDict[
'dimensions'])
647 for dim
in sorted(inDict[
'dimensions']):
648 entry[dim] = row[schema[dim]]
649 inDict[
'dataIdList'].append(entry)
655 """Construct provenance from a dictionary.
660 Dictionary of provenance parameters.
664 provenance : `lsst.ip.isr.IsrProvenance`
665 The provenance defined in the tables.
668 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
669 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
670 f
"found {dictionary['metadata']['OBSTYPE']}")
672 calib.setMetadata(dictionary[
'metadata'])
677 calib.calibType = dictionary[
'calibType']
678 calib.dimensions = set(dictionary[
'dimensions'])
679 calib.dataIdList = dictionary[
'dataIdList']
681 calib.updateMetadata()
685 """Return a dictionary containing the provenance information.
690 Dictionary of provenance.
697 outDict[
'metadata'] = metadata
709 """Return a list of tables containing the provenance.
711 This seems inefficient and slow, so this may not be the best
712 way to store the data.
716 tableList : `list` [`lsst.afw.table.Table`]
717 List of tables containing the provenance information
724 filteredMetadata = {k: v
for k, v
in self.
getMetadata().
toDict().items()
if v
is not None}
725 catalog.meta = filteredMetadata
726 tableList.append(catalog)