Coverage for python/lsst/ip/isr/calibType.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import datetime
23import os.path
24import warnings
25import yaml
26from astropy.table import Table
27from astropy.io import fits
29from lsst.log import Log
30from lsst.daf.base import PropertyList
33__all__ = ["IsrCalib", "IsrProvenance"]
36class IsrCalib(abc.ABC):
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
48 manner.
50 Parameters
51 ----------
52 camera : `lsst.afw.cameraGeom.Camera`, optional
53 Camera to extract metadata from.
54 detector : `lsst.afw.cameraGeom.Detector`, optional
55 Detector to extract metadata from.
56 log : `lsst.log.Log`, optional
57 Log for messages.
58 """
59 _OBSTYPE = 'generic'
60 _SCHEMA = 'NO SCHEMA'
61 _VERSION = 0
63 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs):
64 self._instrument = None
65 self._raftName = None
66 self._slotName = None
67 self._detectorName = detectorName
68 self._detectorSerial = None
69 self._detectorId = detectorId
70 self._filter = None
71 self._calibId = None
72 self._metadata = PropertyList()
73 self.setMetadata(PropertyList())
75 # Define the required attributes for this calibration.
76 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
77 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName',
78 '_detectorName', '_detectorSerial', '_detectorId',
79 '_filter', '_calibId', '_metadata'])
81 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
83 if detector:
84 self.fromDetector(detector)
85 self.updateMetadata(camera=camera, detector=detector, setDate=False)
87 def __str__(self):
88 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
90 def __eq__(self, other):
91 """Calibration equivalence.
93 Subclasses will need to check specific sub-properties. The
94 default is only to check common entries.
95 """
96 if not isinstance(other, self.__class__):
97 return False
99 for attr in self._requiredAttributes:
100 if getattr(self, attr) != getattr(other, attr):
101 return False
103 return True
105 @property
106 def requiredAttributes(self):
107 return self._requiredAttributes
109 @requiredAttributes.setter
110 def requiredAttributes(self, value):
111 self._requiredAttributes = value
113 def getMetadata(self):
114 """Retrieve metadata associated with this calibration.
116 Returns
117 -------
118 meta : `lsst.daf.base.PropertyList`
119 Metadata. The returned `~lsst.daf.base.PropertyList` can be
120 modified by the caller and the changes will be written to
121 external files.
122 """
123 return self._metadata
125 def setMetadata(self, metadata):
126 """Store a copy of the supplied metadata with this calibration.
128 Parameters
129 ----------
130 metadata : `lsst.daf.base.PropertyList`
131 Metadata to associate with the calibration. Will be copied and
132 overwrite existing metadata.
133 """
134 if metadata is not None:
135 self._metadata.update(metadata)
137 # Ensure that we have the obs type required by calibration ingest
138 self._metadata["OBSTYPE"] = self._OBSTYPE
139 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
140 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
142 def updateMetadata(self, camera=None, detector=None, filterName=None,
143 setCalibId=False, setDate=False,
144 **kwargs):
145 """Update metadata keywords with new values.
147 Parameters
148 ----------
149 camera : `lsst.afw.cameraGeom.Camera`, optional
150 Reference camera to use to set _instrument field.
151 detector : `lsst.afw.cameraGeom.Detector`, optional
152 Reference detector to use to set _detector* fields.
153 filterName : `str`, optional
154 Filter name to assign to this calibration.
155 setCalibId : `bool` optional
156 Construct the _calibId field from other fields.
157 setDate : `bool`, optional
158 Ensure the metadata CALIBDATE fields are set to the current datetime.
159 kwargs : `dict` or `collections.abc.Mapping`, optional
160 Set of key=value pairs to assign to the metadata.
161 """
162 mdOriginal = self.getMetadata()
163 mdSupplemental = dict()
165 if camera:
166 self._instrument = camera.getName()
168 if detector:
169 self._detectorName = detector.getName()
170 self._detectorSerial = detector.getSerial()
171 self._detectorId = detector.getId()
172 if "_" in self._detectorName:
173 (self._raftName, self._slotName) = self._detectorName.split("_")
175 if filterName:
176 # If set via:
177 # exposure.getInfo().getFilter().getName()
178 # then this will hold the abstract filter.
179 self._filter = filterName
181 if setCalibId:
182 values = []
183 values.append(f"instrument={self._instrument}") if self._instrument else None
184 values.append(f"raftName={self._raftName}") if self._raftName else None
185 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
186 values.append(f"detector={self.detector}") if self._detector else None
187 values.append(f"filter={self._filter}") if self._filter else None
188 self._calibId = " ".join(values)
190 if setDate:
191 date = datetime.datetime.now()
192 mdSupplemental['CALIBDATE'] = date.isoformat()
193 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
194 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
196 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
197 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
198 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
199 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None
200 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
201 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
202 self._metadata["FILTER"] = self._filter if self._filter else None
203 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
205 mdSupplemental.update(kwargs)
206 mdOriginal.update(mdSupplemental)
208 @classmethod
209 def readText(cls, filename):
210 """Read calibration representation from a yaml/ecsv file.
212 Parameters
213 ----------
214 filename : `str`
215 Name of the file containing the calibration definition.
217 Returns
218 -------
219 calib : `~lsst.ip.isr.IsrCalibType`
220 Calibration class.
222 Raises
223 ------
224 RuntimeError :
225 Raised if the filename does not end in ".ecsv" or ".yaml".
226 """
227 if filename.endswith((".ecsv", ".ECSV")):
228 data = Table.read(filename, format='ascii.ecsv')
229 return cls.fromTable([data])
231 elif filename.endswith((".yaml", ".YAML")):
232 with open(filename, 'r') as f:
233 data = yaml.load(f, Loader=yaml.CLoader)
234 return cls.fromDict(data)
235 else:
236 raise RuntimeError(f"Unknown filename extension: {filename}")
238 def writeText(self, filename, format='auto'):
239 """Write the calibration data to a text file.
241 Parameters
242 ----------
243 filename : `str`
244 Name of the file to write.
245 format : `str`
246 Format to write the file as. Supported values are:
247 ``"auto"`` : Determine filetype from filename.
248 ``"yaml"`` : Write as yaml.
249 ``"ecsv"`` : Write as ecsv.
250 Returns
251 -------
252 used : `str`
253 The name of the file used to write the data. This may
254 differ from the input if the format is explicitly chosen.
256 Raises
257 ------
258 RuntimeError :
259 Raised if filename does not end in a known extension, or
260 if all information cannot be written.
262 Notes
263 -----
264 The file is written to YAML/ECSV format and will include any
265 associated metadata.
267 """
268 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
269 outDict = self.toDict()
270 path, ext = os.path.splitext(filename)
271 filename = path + ".yaml"
272 with open(filename, 'w') as f:
273 yaml.dump(outDict, f)
274 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
275 tableList = self.toTable()
276 if len(tableList) > 1:
277 # ECSV doesn't support multiple tables per file, so we
278 # can only write the first table.
279 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
281 table = tableList[0]
282 path, ext = os.path.splitext(filename)
283 filename = path + ".ecsv"
284 table.write(filename, format="ascii.ecsv")
285 else:
286 raise RuntimeError(f"Attempt to write to a file {filename} "
287 "that does not end in '.yaml' or '.ecsv'")
289 return filename
291 @classmethod
292 def readFits(cls, filename):
293 """Read calibration data from a FITS file.
295 Parameters
296 ----------
297 filename : `str`
298 Filename to read data from.
300 Returns
301 -------
302 calib : `lsst.ip.isr.IsrCalib`
303 Calibration contained within the file.
304 """
305 tableList = []
306 tableList.append(Table.read(filename, hdu=1))
307 extNum = 2 # Fits indices start at 1, we've read one already.
308 try:
309 with warnings.catch_warnings("error"):
310 newTable = Table.read(filename, hdu=extNum)
311 tableList.append(newTable)
312 extNum += 1
313 except Exception:
314 pass
316 return cls.fromTable(tableList)
318 def writeFits(self, filename):
319 """Write calibration data to a FITS file.
321 Parameters
322 ----------
323 filename : `str`
324 Filename to write data to.
326 Returns
327 -------
328 used : `str`
329 The name of the file used to write the data.
331 """
332 tableList = self.toTable()
333 with warnings.catch_warnings():
334 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
335 astropyList = [fits.table_to_hdu(table) for table in tableList]
336 astropyList.insert(0, fits.PrimaryHDU())
338 writer = fits.HDUList(astropyList)
339 writer.writeto(filename, overwrite=True)
340 return filename
342 def fromDetector(self, detector):
343 """Modify the calibration parameters to match the supplied detector.
345 Parameters
346 ----------
347 detector : `lsst.afw.cameraGeom.Detector`
348 Detector to use to set parameters from.
350 Raises
351 ------
352 NotImplementedError
353 This needs to be implemented by subclasses for each
354 calibration type.
355 """
356 raise NotImplementedError("Must be implemented by subclass.")
358 @classmethod
359 def fromDict(cls, dictionary):
360 """Construct a calibration from a dictionary of properties.
362 Must be implemented by the specific calibration subclasses.
364 Parameters
365 ----------
366 dictionary : `dict`
367 Dictionary of properties.
369 Returns
370 ------
371 calib : `lsst.ip.isr.CalibType`
372 Constructed calibration.
374 Raises
375 ------
376 NotImplementedError :
377 Raised if not implemented.
378 """
379 raise NotImplementedError("Must be implemented by subclass.")
381 def toDict(self):
382 """Return a dictionary containing the calibration properties.
384 The dictionary should be able to be round-tripped through
385 `fromDict`.
387 Returns
388 -------
389 dictionary : `dict`
390 Dictionary of properties.
392 Raises
393 ------
394 NotImplementedError :
395 Raised if not implemented.
396 """
397 raise NotImplementedError("Must be implemented by subclass.")
399 @classmethod
400 def fromTable(cls, tableList):
401 """Construct a calibration from a dictionary of properties.
403 Must be implemented by the specific calibration subclasses.
405 Parameters
406 ----------
407 tableList : `list` [`lsst.afw.table.Table`]
408 List of tables of properties.
410 Returns
411 ------
412 calib : `lsst.ip.isr.CalibType`
413 Constructed calibration.
415 Raises
416 ------
417 NotImplementedError :
418 Raised if not implemented.
419 """
420 raise NotImplementedError("Must be implemented by subclass.")
422 def toTable(self):
423 """Return a list of tables containing the calibration properties.
425 The table list should be able to be round-tripped through
426 `fromDict`.
428 Returns
429 -------
430 tableList : `list` [`lsst.afw.table.Table`]
431 List of tables of properties.
433 Raises
434 ------
435 NotImplementedError :
436 Raised if not implemented.
437 """
438 raise NotImplementedError("Must be implemented by subclass.")
440 def validate(self, other=None):
441 """Validate that this calibration is defined and can be used.
443 Parameters
444 ----------
445 other : `object`, optional
446 Thing to validate against.
448 Returns
449 -------
450 valid : `bool`
451 Returns true if the calibration is valid and appropriate.
452 """
453 return False
455 def apply(self, target):
456 """Method to apply the calibration to the target object.
458 Parameters
459 ----------
460 target : `object`
461 Thing to validate against.
463 Returns
464 -------
465 valid : `bool`
466 Returns true if the calibration was applied correctly.
468 Raises
469 ------
470 NotImplementedError :
471 Raised if not implemented.
472 """
473 raise NotImplementedError("Must be implemented by subclass.")
476class IsrProvenance(IsrCalib):
477 """Class for the provenance of data used to construct calibration.
479 Provenance is not really a calibration, but we would like to
480 record this when constructing the calibration, and it provides an
481 example of the base calibration class.
483 Parameters
484 ----------
485 instrument : `str`, optional
486 Name of the instrument the data was taken with.
487 calibType : `str`, optional
488 Type of calibration this provenance was generated for.
489 detectorName : `str`, optional
490 Name of the detector this calibration is for.
491 detectorSerial : `str`, optional
492 Identifier for the detector.
494 """
495 _OBSTYPE = 'IsrProvenance'
497 def __init__(self, calibType="unknown",
498 **kwargs):
499 self.calibType = calibType
500 self.dimensions = set()
501 self.dataIdList = list()
503 super().__init__(**kwargs)
505 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList'])
507 def __str__(self):
508 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
510 def __eq__(self, other):
511 return super().__eq__(other)
513 def updateMetadata(self, setDate=False, **kwargs):
514 """Update calibration metadata.
516 Parameters
517 ----------
518 setDate : `bool, optional
519 Update the CALIBDATE fields in the metadata to the current
520 time. Defaults to False.
521 kwargs : `dict` or `collections.abc.Mapping`, optional
522 Other keyword parameters to set in the metadata.
523 """
524 kwargs['calibType'] = self.calibType
525 super().updateMetadata(setDate=setDate, **kwargs)
527 def fromDataIds(self, dataIdList):
528 """Update provenance from dataId List.
530 Parameters
531 ----------
532 dataIdList : `list` [`lsst.daf.butler.DataId`]
533 List of dataIds used in generating this calibration.
534 """
535 for dataId in dataIdList:
536 for key in dataId:
537 if key not in self.dimensions:
538 self.dimensions.add(key)
539 self.dataIdList.append(dataId)
541 @classmethod
542 def fromTable(cls, tableList):
543 """Construct provenance from table list.
545 Parameters
546 ----------
547 tableList : `list` [`lsst.afw.table.Table`]
548 List of tables to construct the provenance from.
550 Returns
551 -------
552 provenance : `lsst.ip.isr.IsrProvenance`
553 The provenance defined in the tables.
554 """
555 table = tableList[0]
556 metadata = table.meta
557 inDict = dict()
558 inDict['metadata'] = metadata
559 inDict['detectorName'] = metadata.get('DET_NAME', None)
560 inDict['detectorSerial'] = metadata.get('DET_SER', None)
561 inDict['instrument'] = metadata.get('INSTRUME', None)
562 inDict['calibType'] = metadata['calibType']
563 inDict['dimensions'] = set()
564 inDict['dataIdList'] = list()
566 schema = dict()
567 for colName in table.columns:
568 schema[colName.lower()] = colName
569 inDict['dimensions'].add(colName.lower())
570 inDict['dimensions'] = sorted(inDict['dimensions'])
572 for row in table:
573 entry = dict()
574 for dim in sorted(inDict['dimensions']):
575 entry[dim] = row[schema[dim]]
576 inDict['dataIdList'].append(entry)
578 return cls.fromDict(inDict)
580 @classmethod
581 def fromDict(cls, dictionary):
582 """Construct provenance from a dictionary.
584 Parameters
585 ----------
586 dictionary : `dict`
587 Dictionary of provenance parameters.
589 Returns
590 -------
591 provenance : `lsst.ip.isr.IsrProvenance`
592 The provenance defined in the tables.
593 """
594 calib = cls()
595 calib.updateMetadata(setDate=False, **dictionary['metadata'])
597 # These properties should be in the metadata, but occasionally
598 # are found in the dictionary itself. Check both places,
599 # ending with `None` if neither contains the information.
600 calib._detectorName = dictionary.get('detectorName',
601 dictionary['metadata'].get('DET_NAME', None))
602 calib._detectorSerial = dictionary.get('detectorSerial',
603 dictionary['metadata'].get('DET_SER', None))
604 calib._instrument = dictionary.get('instrument',
605 dictionary['metadata'].get('INSTRUME', None))
606 calib.calibType = dictionary['calibType']
607 calib.dimensions = set(dictionary['dimensions'])
608 calib.dataIdList = dictionary['dataIdList']
610 calib.updateMetadata()
611 return calib
613 def toDict(self):
614 """Return a dictionary containing the provenance information.
616 Returns
617 -------
618 dictionary : `dict`
619 Dictionary of provenance.
620 """
621 self.updateMetadata(setDate=True)
623 outDict = {}
625 metadata = self.getMetadata()
626 outDict['metadata'] = metadata
627 outDict['detectorName'] = self._detectorName
628 outDict['detectorSerial'] = self._detectorSerial
629 outDict['instrument'] = self._instrument
630 outDict['calibType'] = self.calibType
631 outDict['dimensions'] = list(self.dimensions)
632 outDict['dataIdList'] = self.dataIdList
634 return outDict
636 def toTable(self):
637 """Return a list of tables containing the provenance.
639 This seems inefficient and slow, so this may not be the best
640 way to store the data.
642 Returns
643 -------
644 tableList : `list` [`lsst.afw.table.Table`]
645 List of tables containing the provenance information
647 """
648 tableList = []
649 self.updateMetadata(setDate=True)
650 catalog = Table(rows=self.dataIdList,
651 names=self.dimensions)
652 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
653 catalog.meta = filteredMetadata
654 tableList.append(catalog)
655 return tableList