Coverage for python/lsst/ip/isr/calibType.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import datetime
23import os.path
24import warnings
25import yaml
26from astropy.table import Table
27from astropy.io import fits
29from lsst.log import Log
30from lsst.daf.base import PropertyList
33__all__ = ["IsrCalib", "IsrProvenance"]
36class IsrCalib(abc.ABC):
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
48 manner.
50 Parameters
51 ----------
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
57 Log for messages.
58 """
59 _OBSTYPE = 'generic'
60 _SCHEMA = 'NO SCHEMA'
61 _VERSION = 0
63 def __init__(self, camera=None, detector=None, log=None, **kwargs):
64 self._instrument = None
65 self._raftName = None
66 self._slotName = None
67 self._detectorName = None
68 self._detectorSerial = None
69 self._detectorId = None
70 self._filter = None
71 self._calibId = None
72 self._metadata = PropertyList()
73 self.setMetadata(PropertyList())
74 self.calibInfoFromDict(kwargs)
76 # Define the required attributes for this calibration.
77 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
78 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName',
79 '_detectorName', '_detectorSerial', '_detectorId',
80 '_filter', '_calibId', '_metadata'])
82 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
84 if detector:
85 self.fromDetector(detector)
86 self.updateMetadata(camera=camera, detector=detector)
88 def __str__(self):
89 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
91 def __eq__(self, other):
92 """Calibration equivalence.
94 Subclasses will need to check specific sub-properties. The
95 default is only to check common entries.
96 """
97 if not isinstance(other, self.__class__):
98 return False
100 for attr in self._requiredAttributes:
101 if getattr(self, attr) != getattr(other, attr):
102 return False
104 return True
106 @property
107 def requiredAttributes(self):
108 return self._requiredAttributes
110 @requiredAttributes.setter
111 def requiredAttributes(self, value):
112 self._requiredAttributes = value
114 def getMetadata(self):
115 """Retrieve metadata associated with this calibration.
117 Returns
118 -------
119 meta : `lsst.daf.base.PropertyList`
120 Metadata. The returned `~lsst.daf.base.PropertyList` can be
121 modified by the caller and the changes will be written to
122 external files.
123 """
124 return self._metadata
126 def setMetadata(self, metadata):
127 """Store a copy of the supplied metadata with this calibration.
129 Parameters
130 ----------
131 metadata : `lsst.daf.base.PropertyList`
132 Metadata to associate with the calibration. Will be copied and
133 overwrite existing metadata.
134 """
135 if metadata is not None:
136 self._metadata.update(metadata)
138 # Ensure that we have the obs type required by calibration ingest
139 self._metadata["OBSTYPE"] = self._OBSTYPE
140 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
141 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
143 if isinstance(metadata, dict):
144 self.calibInfoFromDict(metadata)
145 elif isinstance(metadata, PropertyList):
146 self.calibInfoFromDict(metadata.toDict())
148 def updateMetadata(self, camera=None, detector=None, filterName=None,
149 setCalibId=False, setCalibInfo=False, setDate=False,
150 **kwargs):
151 """Update metadata keywords with new values.
153 Parameters
154 ----------
155 camera : `lsst.afw.cameraGeom.Camera`, optional
156 Reference camera to use to set _instrument field.
157 detector : `lsst.afw.cameraGeom.Detector`, optional
158 Reference detector to use to set _detector* fields.
159 filterName : `str`, optional
160 Filter name to assign to this calibration.
161 setCalibId : `bool` optional
162 Construct the _calibId field from other fields.
163 setDate : `bool`, optional
164 Ensure the metadata CALIBDATE fields are set to the current datetime.
165 kwargs : `dict` or `collections.abc.Mapping`, optional
166 Set of key=value pairs to assign to the metadata.
167 """
168 mdOriginal = self.getMetadata()
169 mdSupplemental = dict()
171 if setCalibInfo:
172 self.calibInfoFromDict(kwargs)
174 if camera:
175 self._instrument = camera.getName()
177 if detector:
178 self._detectorName = detector.getName()
179 self._detectorSerial = detector.getSerial()
180 self._detectorId = detector.getId()
181 if "_" in self._detectorName:
182 (self._raftName, self._slotName) = self._detectorName.split("_")
184 if filterName:
185 # If set via:
186 # exposure.getInfo().getFilter().getName()
187 # then this will hold the abstract filter.
188 self._filter = filterName
190 if setDate:
191 date = datetime.datetime.now()
192 mdSupplemental['CALIBDATE'] = date.isoformat()
193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
196 if setCalibId:
197 values = []
198 values.append(f"instrument={self._instrument}") if self._instrument else None
199 values.append(f"raftName={self._raftName}") if self._raftName else None
200 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
201 values.append(f"detector={self._detectorId}") if self._detectorId else None
202 values.append(f"filter={self._filter}") if self._filter else None
204 calibDate = mdOriginal.get('CALIBDATE', mdSupplemental.get('CALIBDATE', None))
205 values.append(f"calibDate={calibDate}") if calibDate else None
207 self._calibId = " ".join(values)
209 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
210 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
211 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
212 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None
213 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
214 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
215 self._metadata["FILTER"] = self._filter if self._filter else None
216 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
218 mdSupplemental.update(kwargs)
219 mdOriginal.update(mdSupplemental)
221 def calibInfoFromDict(self, dictionary):
222 """Handle common keywords.
224 This isn't an ideal solution, but until all calibrations
225 expect to find everything in the metadata, they still need to
226 search through dictionaries.
228 Parameters
229 ----------
230 dictionary : `dict` or `lsst.daf.base.PropertyList`
231 Source for the common keywords.
233 Raises
234 ------
235 RuntimeError :
236 Raised if the dictionary does not match the expected OBSTYPE.
238 """
240 def search(haystack, needles):
241 """Search dictionary 'haystack' for an entry in 'needles'
242 """
243 test = [haystack.get(x) for x in needles]
244 test = set([x for x in test if x is not None])
245 if len(test) == 0:
246 if 'metadata' in haystack:
247 return search(haystack['metadata'], needles)
248 else:
249 return None
250 elif len(test) == 1:
251 value = list(test)[0]
252 if value == '':
253 return None
254 else:
255 return value
256 else:
257 raise ValueError(f"Too many values found: {len(test)} {test} {needles}")
259 if 'metadata' in dictionary:
260 metadata = dictionary['metadata']
262 if self._OBSTYPE != metadata['OBSTYPE']:
263 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
264 f"found {metadata['OBSTYPE']}")
266 self._instrument = search(dictionary, ['INSTRUME', 'instrument'])
267 self._raftName = search(dictionary, ['RAFTNAME'])
268 self._slotName = search(dictionary, ['SLOTNAME'])
269 self._detectorId = search(dictionary, ['DETECTOR', 'detectorId'])
270 self._detectorName = search(dictionary, ['DET_NAME', 'DETECTOR_NAME', 'detectorName'])
271 self._detectorSerial = search(dictionary, ['DET_SER', 'DETECTOR_SERIAL', 'detectorSerial'])
272 self._filter = search(dictionary, ['FILTER', 'filterName'])
273 self._calibId = search(dictionary, ['CALIB_ID'])
275 @classmethod
276 def readText(cls, filename):
277 """Read calibration representation from a yaml/ecsv file.
279 Parameters
280 ----------
281 filename : `str`
282 Name of the file containing the calibration definition.
284 Returns
285 -------
286 calib : `~lsst.ip.isr.IsrCalibType`
287 Calibration class.
289 Raises
290 ------
291 RuntimeError :
292 Raised if the filename does not end in ".ecsv" or ".yaml".
293 """
294 if filename.endswith((".ecsv", ".ECSV")):
295 data = Table.read(filename, format='ascii.ecsv')
296 return cls.fromTable([data])
298 elif filename.endswith((".yaml", ".YAML")):
299 with open(filename, 'r') as f:
300 data = yaml.load(f, Loader=yaml.CLoader)
301 return cls.fromDict(data)
302 else:
303 raise RuntimeError(f"Unknown filename extension: {filename}")
305 def writeText(self, filename, format='auto'):
306 """Write the calibration data to a text file.
308 Parameters
309 ----------
310 filename : `str`
311 Name of the file to write.
312 format : `str`
313 Format to write the file as. Supported values are:
314 ``"auto"`` : Determine filetype from filename.
315 ``"yaml"`` : Write as yaml.
316 ``"ecsv"`` : Write as ecsv.
317 Returns
318 -------
319 used : `str`
320 The name of the file used to write the data. This may
321 differ from the input if the format is explicitly chosen.
323 Raises
324 ------
325 RuntimeError :
326 Raised if filename does not end in a known extension, or
327 if all information cannot be written.
329 Notes
330 -----
331 The file is written to YAML/ECSV format and will include any
332 associated metadata.
334 """
335 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
336 outDict = self.toDict()
337 path, ext = os.path.splitext(filename)
338 filename = path + ".yaml"
339 with open(filename, 'w') as f:
340 yaml.dump(outDict, f)
341 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
342 tableList = self.toTable()
343 if len(tableList) > 1:
344 # ECSV doesn't support multiple tables per file, so we
345 # can only write the first table.
346 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
348 table = tableList[0]
349 path, ext = os.path.splitext(filename)
350 filename = path + ".ecsv"
351 table.write(filename, format="ascii.ecsv")
352 else:
353 raise RuntimeError(f"Attempt to write to a file {filename} "
354 "that does not end in '.yaml' or '.ecsv'")
356 return filename
358 @classmethod
359 def readFits(cls, filename):
360 """Read calibration data from a FITS file.
362 Parameters
363 ----------
364 filename : `str`
365 Filename to read data from.
367 Returns
368 -------
369 calib : `lsst.ip.isr.IsrCalib`
370 Calibration contained within the file.
371 """
372 tableList = []
373 tableList.append(Table.read(filename, hdu=1))
374 extNum = 2 # Fits indices start at 1, we've read one already.
375 keepTrying = True
377 while keepTrying:
378 with warnings.catch_warnings():
379 warnings.simplefilter("error")
380 try:
381 newTable = Table.read(filename, hdu=extNum)
382 tableList.append(newTable)
383 extNum += 1
384 except Exception:
385 keepTrying = False
387 for table in tableList:
388 for k, v in table.meta.items():
389 if isinstance(v, fits.card.Undefined):
390 table.meta[k] = None
392 return cls.fromTable(tableList)
394 def writeFits(self, filename):
395 """Write calibration data to a FITS file.
397 Parameters
398 ----------
399 filename : `str`
400 Filename to write data to.
402 Returns
403 -------
404 used : `str`
405 The name of the file used to write the data.
407 """
408 tableList = self.toTable()
409 with warnings.catch_warnings():
410 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
411 astropyList = [fits.table_to_hdu(table) for table in tableList]
412 astropyList.insert(0, fits.PrimaryHDU())
414 writer = fits.HDUList(astropyList)
415 writer.writeto(filename, overwrite=True)
416 return filename
418 def fromDetector(self, detector):
419 """Modify the calibration parameters to match the supplied detector.
421 Parameters
422 ----------
423 detector : `lsst.afw.cameraGeom.Detector`
424 Detector to use to set parameters from.
426 Raises
427 ------
428 NotImplementedError
429 This needs to be implemented by subclasses for each
430 calibration type.
431 """
432 raise NotImplementedError("Must be implemented by subclass.")
434 @classmethod
435 def fromDict(cls, dictionary):
436 """Construct a calibration from a dictionary of properties.
438 Must be implemented by the specific calibration subclasses.
440 Parameters
441 ----------
442 dictionary : `dict`
443 Dictionary of properties.
445 Returns
446 ------
447 calib : `lsst.ip.isr.CalibType`
448 Constructed calibration.
450 Raises
451 ------
452 NotImplementedError :
453 Raised if not implemented.
454 """
455 raise NotImplementedError("Must be implemented by subclass.")
457 def toDict(self):
458 """Return a dictionary containing the calibration properties.
460 The dictionary should be able to be round-tripped through
461 `fromDict`.
463 Returns
464 -------
465 dictionary : `dict`
466 Dictionary of properties.
468 Raises
469 ------
470 NotImplementedError :
471 Raised if not implemented.
472 """
473 raise NotImplementedError("Must be implemented by subclass.")
475 @classmethod
476 def fromTable(cls, tableList):
477 """Construct a calibration from a dictionary of properties.
479 Must be implemented by the specific calibration subclasses.
481 Parameters
482 ----------
483 tableList : `list` [`lsst.afw.table.Table`]
484 List of tables of properties.
486 Returns
487 ------
488 calib : `lsst.ip.isr.CalibType`
489 Constructed calibration.
491 Raises
492 ------
493 NotImplementedError :
494 Raised if not implemented.
495 """
496 raise NotImplementedError("Must be implemented by subclass.")
498 def toTable(self):
499 """Return a list of tables containing the calibration properties.
501 The table list should be able to be round-tripped through
502 `fromDict`.
504 Returns
505 -------
506 tableList : `list` [`lsst.afw.table.Table`]
507 List of tables of properties.
509 Raises
510 ------
511 NotImplementedError :
512 Raised if not implemented.
513 """
514 raise NotImplementedError("Must be implemented by subclass.")
516 def validate(self, other=None):
517 """Validate that this calibration is defined and can be used.
519 Parameters
520 ----------
521 other : `object`, optional
522 Thing to validate against.
524 Returns
525 -------
526 valid : `bool`
527 Returns true if the calibration is valid and appropriate.
528 """
529 return False
531 def apply(self, target):
532 """Method to apply the calibration to the target object.
534 Parameters
535 ----------
536 target : `object`
537 Thing to validate against.
539 Returns
540 -------
541 valid : `bool`
542 Returns true if the calibration was applied correctly.
544 Raises
545 ------
546 NotImplementedError :
547 Raised if not implemented.
548 """
549 raise NotImplementedError("Must be implemented by subclass.")
552class IsrProvenance(IsrCalib):
553 """Class for the provenance of data used to construct calibration.
555 Provenance is not really a calibration, but we would like to
556 record this when constructing the calibration, and it provides an
557 example of the base calibration class.
559 Parameters
560 ----------
561 instrument : `str`, optional
562 Name of the instrument the data was taken with.
563 calibType : `str`, optional
564 Type of calibration this provenance was generated for.
565 detectorName : `str`, optional
566 Name of the detector this calibration is for.
567 detectorSerial : `str`, optional
568 Identifier for the detector.
570 """
571 _OBSTYPE = 'IsrProvenance'
573 def __init__(self, calibType="unknown",
574 **kwargs):
575 self.calibType = calibType
576 self.dimensions = set()
577 self.dataIdList = list()
579 super().__init__(**kwargs)
581 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList'])
583 def __str__(self):
584 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
586 def __eq__(self, other):
587 return super().__eq__(other)
589 def updateMetadata(self, setDate=False, **kwargs):
590 """Update calibration metadata.
592 Parameters
593 ----------
594 setDate : `bool, optional
595 Update the CALIBDATE fields in the metadata to the current
596 time. Defaults to False.
597 kwargs : `dict` or `collections.abc.Mapping`, optional
598 Other keyword parameters to set in the metadata.
599 """
600 kwargs['calibType'] = self.calibType
601 super().updateMetadata(setDate=setDate, **kwargs)
603 def fromDataIds(self, dataIdList):
604 """Update provenance from dataId List.
606 Parameters
607 ----------
608 dataIdList : `list` [`lsst.daf.butler.DataId`]
609 List of dataIds used in generating this calibration.
610 """
611 for dataId in dataIdList:
612 for key in dataId:
613 if key not in self.dimensions:
614 self.dimensions.add(key)
615 self.dataIdList.append(dataId)
617 @classmethod
618 def fromTable(cls, tableList):
619 """Construct provenance from table list.
621 Parameters
622 ----------
623 tableList : `list` [`lsst.afw.table.Table`]
624 List of tables to construct the provenance from.
626 Returns
627 -------
628 provenance : `lsst.ip.isr.IsrProvenance`
629 The provenance defined in the tables.
630 """
631 table = tableList[0]
632 metadata = table.meta
633 inDict = dict()
634 inDict['metadata'] = metadata
635 inDict['calibType'] = metadata['calibType']
636 inDict['dimensions'] = set()
637 inDict['dataIdList'] = list()
639 schema = dict()
640 for colName in table.columns:
641 schema[colName.lower()] = colName
642 inDict['dimensions'].add(colName.lower())
643 inDict['dimensions'] = sorted(inDict['dimensions'])
645 for row in table:
646 entry = dict()
647 for dim in sorted(inDict['dimensions']):
648 entry[dim] = row[schema[dim]]
649 inDict['dataIdList'].append(entry)
651 return cls.fromDict(inDict)
653 @classmethod
654 def fromDict(cls, dictionary):
655 """Construct provenance from a dictionary.
657 Parameters
658 ----------
659 dictionary : `dict`
660 Dictionary of provenance parameters.
662 Returns
663 -------
664 provenance : `lsst.ip.isr.IsrProvenance`
665 The provenance defined in the tables.
666 """
667 calib = cls()
668 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
669 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
670 f"found {dictionary['metadata']['OBSTYPE']}")
672 calib.setMetadata(dictionary['metadata'])
674 # These properties should be in the metadata, but occasionally
675 # are found in the dictionary itself. Check both places,
676 # ending with `None` if neither contains the information.
677 calib.calibType = dictionary['calibType']
678 calib.dimensions = set(dictionary['dimensions'])
679 calib.dataIdList = dictionary['dataIdList']
681 calib.updateMetadata()
682 return calib
684 def toDict(self):
685 """Return a dictionary containing the provenance information.
687 Returns
688 -------
689 dictionary : `dict`
690 Dictionary of provenance.
691 """
692 self.updateMetadata()
694 outDict = {}
696 metadata = self.getMetadata()
697 outDict['metadata'] = metadata
698 outDict['detectorName'] = self._detectorName
699 outDict['detectorSerial'] = self._detectorSerial
700 outDict['detectorId'] = self._detectorId
701 outDict['instrument'] = self._instrument
702 outDict['calibType'] = self.calibType
703 outDict['dimensions'] = list(self.dimensions)
704 outDict['dataIdList'] = self.dataIdList
706 return outDict
708 def toTable(self):
709 """Return a list of tables containing the provenance.
711 This seems inefficient and slow, so this may not be the best
712 way to store the data.
714 Returns
715 -------
716 tableList : `list` [`lsst.afw.table.Table`]
717 List of tables containing the provenance information
719 """
720 tableList = []
721 self.updateMetadata()
722 catalog = Table(rows=self.dataIdList,
723 names=self.dimensions)
724 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
725 catalog.meta = filteredMetadata
726 tableList.append(catalog)
727 return tableList