26 from astropy.table
import Table
27 from astropy.io
import fits
33 __all__ = [
"IsrCalib",
"IsrProvenance"]
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
63 def __init__(self, camera=None, detector=None, log=None, **kwargs):
79 '_detectorName',
'_detectorSerial',
'_detectorId',
80 '_filter',
'_calibId',
'_metadata'])
82 self.
log = log
if log
else Log.getLogger(__name__.partition(
".")[2])
89 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
92 """Calibration equivalence.
94 Subclasses will need to check specific sub-properties. The
95 default is only to check common entries.
97 if not isinstance(other, self.__class__):
101 if getattr(self, attr) != getattr(other, attr):
110 @requiredAttributes.setter
115 """Retrieve metadata associated with this calibration.
119 meta : `lsst.daf.base.PropertyList`
120 Metadata. The returned `~lsst.daf.base.PropertyList` can be
121 modified by the caller and the changes will be written to
127 """Store a copy of the supplied metadata with this calibration.
131 metadata : `lsst.daf.base.PropertyList`
132 Metadata to associate with the calibration. Will be copied and
133 overwrite existing metadata.
135 if metadata
is not None:
143 if isinstance(metadata, dict):
145 elif isinstance(metadata, PropertyList):
149 setCalibId=False, setCalibInfo=False, setDate=False,
151 """Update metadata keywords with new values.
155 camera : `lsst.afw.cameraGeom.Camera`, optional
156 Reference camera to use to set _instrument field.
157 detector : `lsst.afw.cameraGeom.Detector`, optional
158 Reference detector to use to set _detector* fields.
159 filterName : `str`, optional
160 Filter name to assign to this calibration.
161 setCalibId : `bool`, optional
162 Construct the _calibId field from other fields.
163 setCalibInfo : `bool`, optional
164 Set calibration parameters from metadata.
165 setDate : `bool`, optional
166 Ensure the metadata CALIBDATE fields are set to the current datetime.
167 kwargs : `dict` or `collections.abc.Mapping`, optional
168 Set of key=value pairs to assign to the metadata.
171 mdSupplemental = dict()
173 for k, v
in kwargs.items():
174 if isinstance(v, fits.card.Undefined):
197 date = datetime.datetime.now()
198 mdSupplemental[
'CALIBDATE'] = date.isoformat()
199 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat()
200 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat()
204 values.append(f
"instrument={self._instrument}")
if self.
_instrument else None
205 values.append(f
"raftName={self._raftName}")
if self.
_raftName else None
206 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName else None
207 values.append(f
"detector={self._detectorId}")
if self.
_detectorId else None
208 values.append(f
"filter={self._filter}")
if self.
_filter else None
210 calibDate = mdOriginal.get(
'CALIBDATE', mdSupplemental.get(
'CALIBDATE',
None))
211 values.append(f
"calibDate={calibDate}")
if calibDate
else None
224 mdSupplemental.update(kwargs)
225 mdOriginal.update(mdSupplemental)
228 """Handle common keywords.
230 This isn't an ideal solution, but until all calibrations
231 expect to find everything in the metadata, they still need to
232 search through dictionaries.
236 dictionary : `dict` or `lsst.daf.base.PropertyList`
237 Source for the common keywords.
242 Raised if the dictionary does not match the expected OBSTYPE.
246 def search(haystack, needles):
247 """Search dictionary 'haystack' for an entry in 'needles'
249 test = [haystack.get(x)
for x
in needles]
250 test = set([x
for x
in test
if x
is not None])
252 if 'metadata' in haystack:
253 return search(haystack[
'metadata'], needles)
257 value = list(test)[0]
263 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
265 if 'metadata' in dictionary:
266 metadata = dictionary[
'metadata']
268 if self.
_OBSTYPE != metadata[
'OBSTYPE']:
269 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
270 f
"found {metadata['OBSTYPE']}")
272 self.
_instrument = search(dictionary, [
'INSTRUME',
'instrument'])
273 self.
_raftName = search(dictionary, [
'RAFTNAME'])
274 self.
_slotName = search(dictionary, [
'SLOTNAME'])
275 self.
_detectorId = search(dictionary, [
'DETECTOR',
'detectorId'])
276 self.
_detectorName = search(dictionary, [
'DET_NAME',
'DETECTOR_NAME',
'detectorName'])
277 self.
_detectorSerial = search(dictionary, [
'DET_SER',
'DETECTOR_SERIAL',
'detectorSerial'])
278 self.
_filter = search(dictionary, [
'FILTER',
'filterName'])
279 self.
_calibId = search(dictionary, [
'CALIB_ID'])
283 """Read calibration representation from a yaml/ecsv file.
288 Name of the file containing the calibration definition.
289 kwargs : `dict` or collections.abc.Mapping`, optional
290 Set of key=value pairs to pass to the ``fromDict`` or
291 ``fromTable`` methods.
295 calib : `~lsst.ip.isr.IsrCalibType`
301 Raised if the filename does not end in ".ecsv" or ".yaml".
303 if filename.endswith((
".ecsv",
".ECSV")):
304 data = Table.read(filename, format=
'ascii.ecsv')
307 elif filename.endswith((
".yaml",
".YAML")):
308 with open(filename,
'r')
as f:
309 data = yaml.load(f, Loader=yaml.CLoader)
312 raise RuntimeError(f
"Unknown filename extension: {filename}")
315 """Write the calibration data to a text file.
320 Name of the file to write.
322 Format to write the file as. Supported values are:
323 ``"auto"`` : Determine filetype from filename.
324 ``"yaml"`` : Write as yaml.
325 ``"ecsv"`` : Write as ecsv.
329 The name of the file used to write the data. This may
330 differ from the input if the format is explicitly chosen.
335 Raised if filename does not end in a known extension, or
336 if all information cannot be written.
340 The file is written to YAML/ECSV format and will include any
344 if format ==
'yaml' or (format ==
'auto' and filename.lower().endswith((
".yaml",
".YAML"))):
346 path, ext = os.path.splitext(filename)
347 filename = path +
".yaml"
348 with open(filename,
'w')
as f:
349 yaml.dump(outDict, f)
350 elif format ==
'ecsv' or (format ==
'auto' and filename.lower().endswith((
".ecsv",
".ECSV"))):
352 if len(tableList) > 1:
355 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
358 path, ext = os.path.splitext(filename)
359 filename = path +
".ecsv"
360 table.write(filename, format=
"ascii.ecsv")
362 raise RuntimeError(f
"Attempt to write to a file {filename} "
363 "that does not end in '.yaml' or '.ecsv'")
369 """Read calibration data from a FITS file.
374 Filename to read data from.
375 kwargs : `dict` or collections.abc.Mapping`, optional
376 Set of key=value pairs to pass to the ``fromTable``
381 calib : `lsst.ip.isr.IsrCalib`
382 Calibration contained within the file.
385 tableList.append(Table.read(filename, hdu=1))
390 with warnings.catch_warnings():
391 warnings.simplefilter(
"error")
393 newTable = Table.read(filename, hdu=extNum)
394 tableList.append(newTable)
399 for table
in tableList:
400 for k, v
in table.meta.items():
401 if isinstance(v, fits.card.Undefined):
404 return cls.
fromTable(tableList, **kwargs)
407 """Write calibration data to a FITS file.
412 Filename to write data to.
417 The name of the file used to write the data.
421 with warnings.catch_warnings():
422 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
423 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
424 astropyList.insert(0, fits.PrimaryHDU())
426 writer = fits.HDUList(astropyList)
427 writer.writeto(filename, overwrite=
True)
431 """Modify the calibration parameters to match the supplied detector.
435 detector : `lsst.afw.cameraGeom.Detector`
436 Detector to use to set parameters from.
441 This needs to be implemented by subclasses for each
444 raise NotImplementedError(
"Must be implemented by subclass.")
448 """Construct a calibration from a dictionary of properties.
450 Must be implemented by the specific calibration subclasses.
455 Dictionary of properties.
456 kwargs : `dict` or collections.abc.Mapping`, optional
457 Set of key=value options.
461 calib : `lsst.ip.isr.CalibType`
462 Constructed calibration.
466 NotImplementedError :
467 Raised if not implemented.
469 raise NotImplementedError(
"Must be implemented by subclass.")
472 """Return a dictionary containing the calibration properties.
474 The dictionary should be able to be round-tripped through
480 Dictionary of properties.
484 NotImplementedError :
485 Raised if not implemented.
487 raise NotImplementedError(
"Must be implemented by subclass.")
491 """Construct a calibration from a dictionary of properties.
493 Must be implemented by the specific calibration subclasses.
497 tableList : `list` [`lsst.afw.table.Table`]
498 List of tables of properties.
499 kwargs : `dict` or collections.abc.Mapping`, optional
500 Set of key=value options.
504 calib : `lsst.ip.isr.CalibType`
505 Constructed calibration.
509 NotImplementedError :
510 Raised if not implemented.
512 raise NotImplementedError(
"Must be implemented by subclass.")
515 """Return a list of tables containing the calibration properties.
517 The table list should be able to be round-tripped through
522 tableList : `list` [`lsst.afw.table.Table`]
523 List of tables of properties.
527 NotImplementedError :
528 Raised if not implemented.
530 raise NotImplementedError(
"Must be implemented by subclass.")
533 """Validate that this calibration is defined and can be used.
537 other : `object`, optional
538 Thing to validate against.
543 Returns true if the calibration is valid and appropriate.
548 """Method to apply the calibration to the target object.
553 Thing to validate against.
558 Returns true if the calibration was applied correctly.
562 NotImplementedError :
563 Raised if not implemented.
565 raise NotImplementedError(
"Must be implemented by subclass.")
569 """Class for the provenance of data used to construct calibration.
571 Provenance is not really a calibration, but we would like to
572 record this when constructing the calibration, and it provides an
573 example of the base calibration class.
577 instrument : `str`, optional
578 Name of the instrument the data was taken with.
579 calibType : `str`, optional
580 Type of calibration this provenance was generated for.
581 detectorName : `str`, optional
582 Name of the detector this calibration is for.
583 detectorSerial : `str`, optional
584 Identifier for the detector.
587 _OBSTYPE =
'IsrProvenance'
600 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
603 return super().
__eq__(other)
606 """Update calibration metadata.
610 setDate : `bool, optional
611 Update the CALIBDATE fields in the metadata to the current
612 time. Defaults to False.
613 kwargs : `dict` or `collections.abc.Mapping`, optional
614 Other keyword parameters to set in the metadata.
620 """Update provenance from dataId List.
624 dataIdList : `list` [`lsst.daf.butler.DataId`]
625 List of dataIds used in generating this calibration.
627 for dataId
in dataIdList:
635 """Construct provenance from table list.
639 tableList : `list` [`lsst.afw.table.Table`]
640 List of tables to construct the provenance from.
644 provenance : `lsst.ip.isr.IsrProvenance`
645 The provenance defined in the tables.
648 metadata = table.meta
650 inDict[
'metadata'] = metadata
651 inDict[
'calibType'] = metadata[
'calibType']
652 inDict[
'dimensions'] = set()
653 inDict[
'dataIdList'] = list()
656 for colName
in table.columns:
657 schema[colName.lower()] = colName
658 inDict[
'dimensions'].add(colName.lower())
659 inDict[
'dimensions'] = sorted(inDict[
'dimensions'])
663 for dim
in sorted(inDict[
'dimensions']):
664 entry[dim] = row[schema[dim]]
665 inDict[
'dataIdList'].append(entry)
671 """Construct provenance from a dictionary.
676 Dictionary of provenance parameters.
680 provenance : `lsst.ip.isr.IsrProvenance`
681 The provenance defined in the tables.
684 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
685 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
686 f
"found {dictionary['metadata']['OBSTYPE']}")
687 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
'metadata'])
692 calib.calibType = dictionary[
'calibType']
693 calib.dimensions = set(dictionary[
'dimensions'])
694 calib.dataIdList = dictionary[
'dataIdList']
696 calib.updateMetadata()
700 """Return a dictionary containing the provenance information.
705 Dictionary of provenance.
712 outDict[
'metadata'] = metadata
724 """Return a list of tables containing the provenance.
726 This seems inefficient and slow, so this may not be the best
727 way to store the data.
731 tableList : `list` [`lsst.afw.table.Table`]
732 List of tables containing the provenance information
740 filteredMetadata = {k: v
for k, v
in self.
getMetadata().
toDict().items()
if v
is not None}
741 catalog.meta = filteredMetadata
742 tableList.append(catalog)