Coverage for python/lsst/ip/isr/calibType.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import datetime
23import os.path
24import warnings
25import yaml
26from astropy.table import Table
27from astropy.io import fits
29from lsst.log import Log
30from lsst.daf.base import PropertyList
33__all__ = ["IsrCalib", "IsrProvenance"]
36class IsrCalib(abc.ABC):
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
48 manner.
50 Parameters
51 ----------
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
57 Log for messages.
58 """
59 _OBSTYPE = 'generic'
60 _SCHEMA = 'NO SCHEMA'
61 _VERSION = 0
63 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs):
64 self._instrument = None
65 self._raftName = None
66 self._slotName = None
67 self._detectorName = detectorName
68 self._detectorSerial = None
69 self._detectorId = detectorId
70 self._filter = None
71 self._calibId = None
72 self._metadata = PropertyList()
73 self.setMetadata(PropertyList())
75 # Define the required attributes for this calibration.
76 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
77 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName',
78 '_detectorName', '_detectorSerial', '_detectorId',
79 '_filter', '_calibId', '_metadata'])
81 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
83 if detector:
84 self.fromDetector(detector)
85 self.updateMetadata(camera=camera, detector=detector, setDate=False)
87 def __str__(self):
88 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
90 def __eq__(self, other):
91 """Calibration equivalence.
93 Subclasses will need to check specific sub-properties. The
94 default is only to check common entries.
95 """
96 if not isinstance(other, self.__class__):
97 return False
99 for attr in self._requiredAttributes:
100 if getattr(self, attr) != getattr(other, attr):
101 return False
103 return True
105 @property
106 def requiredAttributes(self):
107 return self._requiredAttributes
109 @requiredAttributes.setter
110 def requiredAttributes(self, value):
111 self._requiredAttributes = value
113 def getMetadata(self):
114 """Retrieve metadata associated with this calibration.
116 Returns
117 -------
118 meta : `lsst.daf.base.PropertyList`
119 Metadata. The returned `~lsst.daf.base.PropertyList` can be
120 modified by the caller and the changes will be written to
121 external files.
122 """
123 return self._metadata
125 def setMetadata(self, metadata):
126 """Store a copy of the supplied metadata with this calibration.
128 Parameters
129 ----------
130 metadata : `lsst.daf.base.PropertyList`
131 Metadata to associate with the calibration. Will be copied and
132 overwrite existing metadata.
133 """
134 if metadata is not None:
135 self._metadata.update(metadata)
137 # Ensure that we have the obs type required by calibration ingest
138 self._metadata["OBSTYPE"] = self._OBSTYPE
139 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
140 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
142 def updateMetadata(self, camera=None, detector=None, filterName=None,
143 setCalibId=False, setDate=False,
144 **kwargs):
145 """Update metadata keywords with new values.
147 Parameters
148 ----------
149 camera : `lsst.afw.cameraGeom.Camera`, optional
150 Reference camera to use to set _instrument field.
151 detector : `lsst.afw.cameraGeom.Detector`, optional
152 Reference detector to use to set _detector* fields.
153 filterName : `str`, optional
154 Filter name to assign to this calibration.
155 setCalibId : `bool` optional
156 Construct the _calibId field from other fields.
157 setDate : `bool`, optional
158 Ensure the metadata CALIBDATE fields are set to the current datetime.
159 kwargs : `dict` or `collections.abc.Mapping`, optional
160 Set of key=value pairs to assign to the metadata.
161 """
162 mdOriginal = self.getMetadata()
163 mdSupplemental = dict()
165 if camera:
166 self._instrument = camera.getName()
168 if detector:
169 self._detectorName = detector.getName()
170 self._detectorSerial = detector.getSerial()
171 self._detectorId = detector.getId()
172 if "_" in self._detectorName:
173 (self._raftName, self._slotName) = self._detectorName.split("_")
175 if filterName:
176 # If set via:
177 # exposure.getInfo().getFilter().getName()
178 # then this will hold the abstract filter.
179 self._filter = filterName
181 if setCalibId:
182 values = []
183 values.append(f"instrument={self._instrument}") if self._instrument else None
184 values.append(f"raftName={self._raftName}") if self._raftName else None
185 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
186 values.append(f"detector={self.detector}") if self._detector else None
187 values.append(f"filter={self._filter}") if self._filter else None
188 self._calibId = " ".join(values)
190 if setDate:
191 date = datetime.datetime.now()
192 mdSupplemental['CALIBDATE'] = date.isoformat()
193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
196 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
197 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
198 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
199 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None
200 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
201 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
202 self._metadata["FILTER"] = self._filter if self._filter else None
203 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
205 mdSupplemental.update(kwargs)
206 mdOriginal.update(mdSupplemental)
208 @classmethod
209 def readText(cls, filename):
210 """Read calibration representation from a yaml/ecsv file.
212 Parameters
213 ----------
214 filename : `str`
215 Name of the file containing the calibration definition.
217 Returns
218 -------
219 calib : `~lsst.ip.isr.IsrCalibType`
220 Calibration class.
222 Raises
223 ------
224 RuntimeError :
225 Raised if the filename does not end in ".ecsv" or ".yaml".
226 """
227 if filename.endswith((".ecsv", ".ECSV")):
228 data = Table.read(filename, format='ascii.ecsv')
229 return cls.fromTable([data])
231 elif filename.endswith((".yaml", ".YAML")):
232 with open(filename, 'r') as f:
233 data = yaml.load(f, Loader=yaml.CLoader)
234 return cls.fromDict(data)
235 else:
236 raise RuntimeError(f"Unknown filename extension: {filename}")
238 def writeText(self, filename, format='auto'):
239 """Write the calibration data to a text file.
241 Parameters
242 ----------
243 filename : `str`
244 Name of the file to write.
245 format : `str`
246 Format to write the file as. Supported values are:
247 ``"auto"`` : Determine filetype from filename.
248 ``"yaml"`` : Write as yaml.
249 ``"ecsv"`` : Write as ecsv.
250 Returns
251 -------
252 used : `str`
253 The name of the file used to write the data. This may
254 differ from the input if the format is explicitly chosen.
256 Raises
257 ------
258 RuntimeError :
259 Raised if filename does not end in a known extension, or
260 if all information cannot be written.
262 Notes
263 -----
264 The file is written to YAML/ECSV format and will include any
265 associated metadata.
267 """
268 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
269 outDict = self.toDict()
270 path, ext = os.path.splitext(filename)
271 filename = path + ".yaml"
272 with open(filename, 'w') as f:
273 yaml.dump(outDict, f)
274 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
275 tableList = self.toTable()
276 if len(tableList) > 1:
277 # ECSV doesn't support multiple tables per file, so we
278 # can only write the first table.
279 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
281 table = tableList[0]
282 path, ext = os.path.splitext(filename)
283 filename = path + ".ecsv"
284 table.write(filename, format="ascii.ecsv")
285 else:
286 raise RuntimeError(f"Attempt to write to a file {filename} "
287 "that does not end in '.yaml' or '.ecsv'")
289 return filename
291 @classmethod
292 def readFits(cls, filename):
293 """Read calibration data from a FITS file.
295 Parameters
296 ----------
297 filename : `str`
298 Filename to read data from.
300 Returns
301 -------
302 calib : `lsst.ip.isr.IsrCalib`
303 Calibration contained within the file.
304 """
305 tableList = []
306 tableList.append(Table.read(filename, hdu=1))
307 extNum = 2 # Fits indices start at 1, we've read one already.
308 keepTrying = True
310 while keepTrying:
311 with warnings.catch_warnings():
312 warnings.simplefilter("error")
313 try:
314 newTable = Table.read(filename, hdu=extNum)
315 tableList.append(newTable)
316 extNum += 1
317 except Exception:
318 keepTrying = False
320 return cls.fromTable(tableList)
322 def writeFits(self, filename):
323 """Write calibration data to a FITS file.
325 Parameters
326 ----------
327 filename : `str`
328 Filename to write data to.
330 Returns
331 -------
332 used : `str`
333 The name of the file used to write the data.
335 """
336 tableList = self.toTable()
337 with warnings.catch_warnings():
338 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
339 astropyList = [fits.table_to_hdu(table) for table in tableList]
340 astropyList.insert(0, fits.PrimaryHDU())
342 writer = fits.HDUList(astropyList)
343 writer.writeto(filename, overwrite=True)
344 return filename
346 def fromDetector(self, detector):
347 """Modify the calibration parameters to match the supplied detector.
349 Parameters
350 ----------
351 detector : `lsst.afw.cameraGeom.Detector`
352 Detector to use to set parameters from.
354 Raises
355 ------
356 NotImplementedError
357 This needs to be implemented by subclasses for each
358 calibration type.
359 """
360 raise NotImplementedError("Must be implemented by subclass.")
362 @classmethod
363 def fromDict(cls, dictionary):
364 """Construct a calibration from a dictionary of properties.
366 Must be implemented by the specific calibration subclasses.
368 Parameters
369 ----------
370 dictionary : `dict`
371 Dictionary of properties.
373 Returns
374 ------
375 calib : `lsst.ip.isr.CalibType`
376 Constructed calibration.
378 Raises
379 ------
380 NotImplementedError :
381 Raised if not implemented.
382 """
383 raise NotImplementedError("Must be implemented by subclass.")
385 def toDict(self):
386 """Return a dictionary containing the calibration properties.
388 The dictionary should be able to be round-tripped through
389 `fromDict`.
391 Returns
392 -------
393 dictionary : `dict`
394 Dictionary of properties.
396 Raises
397 ------
398 NotImplementedError :
399 Raised if not implemented.
400 """
401 raise NotImplementedError("Must be implemented by subclass.")
403 @classmethod
404 def fromTable(cls, tableList):
405 """Construct a calibration from a dictionary of properties.
407 Must be implemented by the specific calibration subclasses.
409 Parameters
410 ----------
411 tableList : `list` [`lsst.afw.table.Table`]
412 List of tables of properties.
414 Returns
415 ------
416 calib : `lsst.ip.isr.CalibType`
417 Constructed calibration.
419 Raises
420 ------
421 NotImplementedError :
422 Raised if not implemented.
423 """
424 raise NotImplementedError("Must be implemented by subclass.")
426 def toTable(self):
427 """Return a list of tables containing the calibration properties.
429 The table list should be able to be round-tripped through
430 `fromDict`.
432 Returns
433 -------
434 tableList : `list` [`lsst.afw.table.Table`]
435 List of tables of properties.
437 Raises
438 ------
439 NotImplementedError :
440 Raised if not implemented.
441 """
442 raise NotImplementedError("Must be implemented by subclass.")
444 def validate(self, other=None):
445 """Validate that this calibration is defined and can be used.
447 Parameters
448 ----------
449 other : `object`, optional
450 Thing to validate against.
452 Returns
453 -------
454 valid : `bool`
455 Returns true if the calibration is valid and appropriate.
456 """
457 return False
459 def apply(self, target):
460 """Method to apply the calibration to the target object.
462 Parameters
463 ----------
464 target : `object`
465 Thing to validate against.
467 Returns
468 -------
469 valid : `bool`
470 Returns true if the calibration was applied correctly.
472 Raises
473 ------
474 NotImplementedError :
475 Raised if not implemented.
476 """
477 raise NotImplementedError("Must be implemented by subclass.")
480class IsrProvenance(IsrCalib):
481 """Class for the provenance of data used to construct calibration.
483 Provenance is not really a calibration, but we would like to
484 record this when constructing the calibration, and it provides an
485 example of the base calibration class.
487 Parameters
488 ----------
489 instrument : `str`, optional
490 Name of the instrument the data was taken with.
491 calibType : `str`, optional
492 Type of calibration this provenance was generated for.
493 detectorName : `str`, optional
494 Name of the detector this calibration is for.
495 detectorSerial : `str`, optional
496 Identifier for the detector.
498 """
499 _OBSTYPE = 'IsrProvenance'
501 def __init__(self, calibType="unknown",
502 **kwargs):
503 self.calibType = calibType
504 self.dimensions = set()
505 self.dataIdList = list()
507 super().__init__(**kwargs)
509 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList'])
511 def __str__(self):
512 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
514 def __eq__(self, other):
515 return super().__eq__(other)
517 def updateMetadata(self, setDate=False, **kwargs):
518 """Update calibration metadata.
520 Parameters
521 ----------
522 setDate : `bool, optional
523 Update the CALIBDATE fields in the metadata to the current
524 time. Defaults to False.
525 kwargs : `dict` or `collections.abc.Mapping`, optional
526 Other keyword parameters to set in the metadata.
527 """
528 kwargs['calibType'] = self.calibType
529 super().updateMetadata(setDate=setDate, **kwargs)
531 def fromDataIds(self, dataIdList):
532 """Update provenance from dataId List.
534 Parameters
535 ----------
536 dataIdList : `list` [`lsst.daf.butler.DataId`]
537 List of dataIds used in generating this calibration.
538 """
539 for dataId in dataIdList:
540 for key in dataId:
541 if key not in self.dimensions:
542 self.dimensions.add(key)
543 self.dataIdList.append(dataId)
545 @classmethod
546 def fromTable(cls, tableList):
547 """Construct provenance from table list.
549 Parameters
550 ----------
551 tableList : `list` [`lsst.afw.table.Table`]
552 List of tables to construct the provenance from.
554 Returns
555 -------
556 provenance : `lsst.ip.isr.IsrProvenance`
557 The provenance defined in the tables.
558 """
559 table = tableList[0]
560 metadata = table.meta
561 inDict = dict()
562 inDict['metadata'] = metadata
563 inDict['detectorName'] = metadata.get('DET_NAME', None)
564 inDict['detectorSerial'] = metadata.get('DET_SER', None)
565 inDict['instrument'] = metadata.get('INSTRUME', None)
566 inDict['calibType'] = metadata['calibType']
567 inDict['dimensions'] = set()
568 inDict['dataIdList'] = list()
570 schema = dict()
571 for colName in table.columns:
572 schema[colName.lower()] = colName
573 inDict['dimensions'].add(colName.lower())
574 inDict['dimensions'] = sorted(inDict['dimensions'])
576 for row in table:
577 entry = dict()
578 for dim in sorted(inDict['dimensions']):
579 entry[dim] = row[schema[dim]]
580 inDict['dataIdList'].append(entry)
582 return cls.fromDict(inDict)
584 @classmethod
585 def fromDict(cls, dictionary):
586 """Construct provenance from a dictionary.
588 Parameters
589 ----------
590 dictionary : `dict`
591 Dictionary of provenance parameters.
593 Returns
594 -------
595 provenance : `lsst.ip.isr.IsrProvenance`
596 The provenance defined in the tables.
597 """
598 calib = cls()
599 calib.updateMetadata(setDate=False, **dictionary['metadata'])
601 # These properties should be in the metadata, but occasionally
602 # are found in the dictionary itself. Check both places,
603 # ending with `None` if neither contains the information.
604 calib._detectorName = dictionary.get('detectorName',
605 dictionary['metadata'].get('DET_NAME', None))
606 calib._detectorSerial = dictionary.get('detectorSerial',
607 dictionary['metadata'].get('DET_SER', None))
608 calib._instrument = dictionary.get('instrument',
609 dictionary['metadata'].get('INSTRUME', None))
610 calib.calibType = dictionary['calibType']
611 calib.dimensions = set(dictionary['dimensions'])
612 calib.dataIdList = dictionary['dataIdList']
614 calib.updateMetadata()
615 return calib
617 def toDict(self):
618 """Return a dictionary containing the provenance information.
620 Returns
621 -------
622 dictionary : `dict`
623 Dictionary of provenance.
624 """
625 self.updateMetadata(setDate=True)
627 outDict = {}
629 metadata = self.getMetadata()
630 outDict['metadata'] = metadata
631 outDict['detectorName'] = self._detectorName
632 outDict['detectorSerial'] = self._detectorSerial
633 outDict['instrument'] = self._instrument
634 outDict['calibType'] = self.calibType
635 outDict['dimensions'] = list(self.dimensions)
636 outDict['dataIdList'] = self.dataIdList
638 return outDict
640 def toTable(self):
641 """Return a list of tables containing the provenance.
643 This seems inefficient and slow, so this may not be the best
644 way to store the data.
646 Returns
647 -------
648 tableList : `list` [`lsst.afw.table.Table`]
649 List of tables containing the provenance information
651 """
652 tableList = []
653 self.updateMetadata(setDate=True)
654 catalog = Table(rows=self.dataIdList,
655 names=self.dimensions)
656 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
657 catalog.meta = filteredMetadata
658 tableList.append(catalog)
659 return tableList