Coverage for python/lsst/ip/isr/calibType.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import copy
23import datetime
24import os.path
25import warnings
26import yaml
27from astropy.table import Table
28from astropy.io import fits
30from lsst.log import Log
31from lsst.daf.base import PropertyList
34__all__ = ["IsrCalib", "IsrProvenance"]
37class IsrCalib(abc.ABC):
38 """Generic calibration type.
40 Subclasses must implement the toDict, fromDict, toTable, fromTable
41 methods that allow the calibration information to be converted
42 from dictionaries and afw tables. This will allow the calibration
43 to be persisted using the base class read/write methods.
45 The validate method is intended to provide a common way to check
46 that the calibration is valid (internally consistent) and
47 appropriate (usable with the intended data). The apply method is
48 intended to allow the calibration to be applied in a consistent
49 manner.
51 Parameters
52 ----------
53 camera : `lsst.afw.cameraGeom.Camera`, optional
54 Camera to extract metadata from.
55 detector : `lsst.afw.cameraGeom.Detector`, optional
56 Detector to extract metadata from.
57 log : `lsst.log.Log`, optional
58 Log for messages.
59 """
60 _OBSTYPE = 'generic'
61 _SCHEMA = 'NO SCHEMA'
62 _VERSION = 0
64 def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs):
65 self._instrument = None
66 self._raftName = None
67 self._slotName = None
68 self._detectorName = detectorName
69 self._detectorSerial = None
70 self._detectorId = detectorId
71 self._filter = None
72 self._calibId = None
74 self.setMetadata(PropertyList())
76 # Define the required attributes for this calibration.
77 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
78 self.requiredAttributes.update(['_instrument', '_raftName', '_slotName',
79 '_detectorName', '_detectorSerial', '_detectorId',
80 '_filter', '_calibId', '_metadata'])
82 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
84 if detector:
85 self.fromDetector(detector)
86 self.updateMetadata(camera=camera, detector=detector, setDate=False)
88 def __str__(self):
89 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
91 def __eq__(self, other):
92 """Calibration equivalence.
94 Subclasses will need to check specific sub-properties. The
95 default is only to check common entries.
96 """
97 if not isinstance(other, self.__class__):
98 return False
100 for attr in self._requiredAttributes:
101 if getattr(self, attr) != getattr(other, attr):
102 return False
104 return True
106 @property
107 def requiredAttributes(self):
108 return self._requiredAttributes
110 @requiredAttributes.setter
111 def requiredAttributes(self, value):
112 self._requiredAttributes = value
114 def getMetadata(self):
115 """Retrieve metadata associated with this calibration.
117 Returns
118 -------
119 meta : `lsst.daf.base.PropertyList`
120 Metadata. The returned `~lsst.daf.base.PropertyList` can be
121 modified by the caller and the changes will be written to
122 external files.
123 """
124 return self._metadata
126 def setMetadata(self, metadata):
127 """Store a copy of the supplied metadata with this calibration.
129 Parameters
130 ----------
131 metadata : `lsst.daf.base.PropertyList`
132 Metadata to associate with the calibration. Will be copied and
133 overwrite existing metadata.
134 """
135 if metadata is not None:
136 self._metadata = copy.copy(metadata)
138 # Ensure that we have the obs type required by calibration ingest
139 self._metadata["OBSTYPE"] = self._OBSTYPE
140 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
141 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
143 def updateMetadata(self, camera=None, detector=None, filterName=None,
144 setCalibId=False, setDate=False,
145 **kwargs):
146 """Update metadata keywords with new values.
148 Parameters
149 ----------
150 camera : `lsst.afw.cameraGeom.Camera`, optional
151 Reference camera to use to set _instrument field.
152 detector : `lsst.afw.cameraGeom.Detector`, optional
153 Reference detector to use to set _detector* fields.
154 filterName : `str`, optional
155 Filter name to assign to this calibration.
156 setCalibId : `bool` optional
157 Construct the _calibId field from other fields.
158 setDate : `bool`, optional
159 Ensure the metadata CALIBDATE fields are set to the current datetime.
160 kwargs : `dict` or `collections.abc.Mapping`, optional
161 Set of key=value pairs to assign to the metadata.
162 """
163 mdOriginal = self.getMetadata()
164 mdSupplemental = dict()
166 if camera:
167 self._instrument = camera.getName()
169 if detector:
170 self._detectorName = detector.getName()
171 self._detectorSerial = detector.getSerial()
172 self._detectorId = detector.getId()
173 if "_" in self._detectorName:
174 (self._raftName, self._slotName) = self._detectorName.split("_")
176 if filterName:
177 # If set via:
178 # exposure.getInfo().getFilter().getName()
179 # then this will hold the abstract filter.
180 self._filter = filterName
182 if setCalibId:
183 values = []
184 values.append(f"instrument={self._instrument}") if self._instrument else None
185 values.append(f"raftName={self._raftName}") if self._raftName else None
186 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
187 values.append(f"detector={self.detector}") if self._detector else None
188 values.append(f"filter={self._filter}") if self._filter else None
189 self._calibId = " ".join(values)
191 if setDate:
192 date = datetime.datetime.now()
193 mdSupplemental['CALIBDATE'] = date.isoformat()
194 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
195 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
197 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
198 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
199 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
200 self._metadata["DETECTOR"] = self._detectorId if self._detectorId else None
201 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
202 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
203 self._metadata["FILTER"] = self._filter if self._filter else None
204 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
206 mdSupplemental.update(kwargs)
207 mdOriginal.update(mdSupplemental)
209 @classmethod
210 def readText(cls, filename):
211 """Read calibration representation from a yaml/ecsv file.
213 Parameters
214 ----------
215 filename : `str`
216 Name of the file containing the calibration definition.
218 Returns
219 -------
220 calib : `~lsst.ip.isr.IsrCalibType`
221 Calibration class.
223 Raises
224 ------
225 RuntimeError :
226 Raised if the filename does not end in ".ecsv" or ".yaml".
227 """
228 if filename.endswith((".ecsv", ".ECSV")):
229 data = Table.read(filename, format='ascii.ecsv')
230 return cls.fromTable([data])
232 elif filename.endswith((".yaml", ".YAML")):
233 with open(filename, 'r') as f:
234 data = yaml.load(f, Loader=yaml.CLoader)
235 return cls.fromDict(data)
236 else:
237 raise RuntimeError(f"Unknown filename extension: {filename}")
239 def writeText(self, filename, format='auto'):
240 """Write the calibration data to a text file.
242 Parameters
243 ----------
244 filename : `str`
245 Name of the file to write.
246 format : `str`
247 Format to write the file as. Supported values are:
248 ``"auto"`` : Determine filetype from filename.
249 ``"yaml"`` : Write as yaml.
250 ``"ecsv"`` : Write as ecsv.
251 Returns
252 -------
253 used : `str`
254 The name of the file used to write the data. This may
255 differ from the input if the format is explicitly chosen.
257 Raises
258 ------
259 RuntimeError :
260 Raised if filename does not end in a known extension, or
261 if all information cannot be written.
263 Notes
264 -----
265 The file is written to YAML/ECSV format and will include any
266 associated metadata.
268 """
269 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
270 outDict = self.toDict()
271 path, ext = os.path.splitext(filename)
272 filename = path + ".yaml"
273 with open(filename, 'w') as f:
274 yaml.dump(outDict, f)
275 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
276 tableList = self.toTable()
277 if len(tableList) > 1:
278 # ECSV doesn't support multiple tables per file, so we
279 # can only write the first table.
280 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
282 table = tableList[0]
283 path, ext = os.path.splitext(filename)
284 filename = path + ".ecsv"
285 table.write(filename, format="ascii.ecsv")
286 else:
287 raise RuntimeError(f"Attempt to write to a file {filename} "
288 "that does not end in '.yaml' or '.ecsv'")
290 return filename
292 @classmethod
293 def readFits(cls, filename):
294 """Read calibration data from a FITS file.
296 Parameters
297 ----------
298 filename : `str`
299 Filename to read data from.
301 Returns
302 -------
303 calib : `lsst.ip.isr.IsrCalib`
304 Calibration contained within the file.
305 """
306 tableList = []
307 tableList.append(Table.read(filename, hdu=1))
308 extNum = 2 # Fits indices start at 1, we've read one already.
309 try:
310 with warnings.catch_warnings("error"):
311 newTable = Table.read(filename, hdu=extNum)
312 tableList.append(newTable)
313 extNum += 1
314 except Exception:
315 pass
317 return cls.fromTable(tableList)
319 def writeFits(self, filename):
320 """Write calibration data to a FITS file.
322 Parameters
323 ----------
324 filename : `str`
325 Filename to write data to.
327 Returns
328 -------
329 used : `str`
330 The name of the file used to write the data.
332 """
333 tableList = self.toTable()
334 with warnings.catch_warnings():
335 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
336 astropyList = [fits.table_to_hdu(table) for table in tableList]
337 astropyList.insert(0, fits.PrimaryHDU())
339 writer = fits.HDUList(astropyList)
340 writer.writeto(filename, overwrite=True)
341 return filename
343 def fromDetector(self, detector):
344 """Modify the calibration parameters to match the supplied detector.
346 Parameters
347 ----------
348 detector : `lsst.afw.cameraGeom.Detector`
349 Detector to use to set parameters from.
351 Raises
352 ------
353 NotImplementedError
354 This needs to be implemented by subclasses for each
355 calibration type.
356 """
357 raise NotImplementedError("Must be implemented by subclass.")
359 @classmethod
360 def fromDict(cls, dictionary):
361 """Construct a calibration from a dictionary of properties.
363 Must be implemented by the specific calibration subclasses.
365 Parameters
366 ----------
367 dictionary : `dict`
368 Dictionary of properties.
370 Returns
371 ------
372 calib : `lsst.ip.isr.CalibType`
373 Constructed calibration.
375 Raises
376 ------
377 NotImplementedError :
378 Raised if not implemented.
379 """
380 raise NotImplementedError("Must be implemented by subclass.")
382 def toDict(self):
383 """Return a dictionary containing the calibration properties.
385 The dictionary should be able to be round-tripped through
386 `fromDict`.
388 Returns
389 -------
390 dictionary : `dict`
391 Dictionary of properties.
393 Raises
394 ------
395 NotImplementedError :
396 Raised if not implemented.
397 """
398 raise NotImplementedError("Must be implemented by subclass.")
400 @classmethod
401 def fromTable(cls, tableList):
402 """Construct a calibration from a dictionary of properties.
404 Must be implemented by the specific calibration subclasses.
406 Parameters
407 ----------
408 tableList : `list` [`lsst.afw.table.Table`]
409 List of tables of properties.
411 Returns
412 ------
413 calib : `lsst.ip.isr.CalibType`
414 Constructed calibration.
416 Raises
417 ------
418 NotImplementedError :
419 Raised if not implemented.
420 """
421 raise NotImplementedError("Must be implemented by subclass.")
423 def toTable(self):
424 """Return a list of tables containing the calibration properties.
426 The table list should be able to be round-tripped through
427 `fromDict`.
429 Returns
430 -------
431 tableList : `list` [`lsst.afw.table.Table`]
432 List of tables of properties.
434 Raises
435 ------
436 NotImplementedError :
437 Raised if not implemented.
438 """
439 raise NotImplementedError("Must be implemented by subclass.")
441 def validate(self, other=None):
442 """Validate that this calibration is defined and can be used.
444 Parameters
445 ----------
446 other : `object`, optional
447 Thing to validate against.
449 Returns
450 -------
451 valid : `bool`
452 Returns true if the calibration is valid and appropriate.
453 """
454 return False
456 def apply(self, target):
457 """Method to apply the calibration to the target object.
459 Parameters
460 ----------
461 target : `object`
462 Thing to validate against.
464 Returns
465 -------
466 valid : `bool`
467 Returns true if the calibration was applied correctly.
469 Raises
470 ------
471 NotImplementedError :
472 Raised if not implemented.
473 """
474 raise NotImplementedError("Must be implemented by subclass.")
477class IsrProvenance(IsrCalib):
478 """Class for the provenance of data used to construct calibration.
480 Provenance is not really a calibration, but we would like to
481 record this when constructing the calibration, and it provides an
482 example of the base calibration class.
484 Parameters
485 ----------
486 instrument : `str`, optional
487 Name of the instrument the data was taken with.
488 calibType : `str`, optional
489 Type of calibration this provenance was generated for.
490 detectorName : `str`, optional
491 Name of the detector this calibration is for.
492 detectorSerial : `str`, optional
493 Identifier for the detector.
495 """
496 _OBSTYPE = 'IsrProvenance'
498 def __init__(self, calibType="unknown",
499 **kwargs):
500 self.calibType = calibType
501 self.dimensions = set()
502 self.dataIdList = list()
504 super().__init__(**kwargs)
506 self.requiredAttributes.update(['calibType', 'dimensions', 'dataIdList'])
508 def __str__(self):
509 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
511 def __eq__(self, other):
512 return super().__eq__(other)
514 def updateMetadata(self, setDate=False, **kwargs):
515 """Update calibration metadata.
517 Parameters
518 ----------
519 setDate : `bool, optional
520 Update the CALIBDATE fields in the metadata to the current
521 time. Defaults to False.
522 kwargs : `dict` or `collections.abc.Mapping`, optional
523 Other keyword parameters to set in the metadata.
524 """
525 kwargs['calibType'] = self.calibType
526 super().updateMetadata(setDate=setDate, **kwargs)
528 def fromDataIds(self, dataIdList):
529 """Update provenance from dataId List.
531 Parameters
532 ----------
533 dataIdList : `list` [`lsst.daf.butler.DataId`]
534 List of dataIds used in generating this calibration.
535 """
536 for dataId in dataIdList:
537 for key in dataId:
538 if key not in self.dimensions:
539 self.dimensions.add(key)
540 self.dataIdList.append(dataId)
542 @classmethod
543 def fromTable(cls, tableList):
544 """Construct provenance from table list.
546 Parameters
547 ----------
548 tableList : `list` [`lsst.afw.table.Table`]
549 List of tables to construct the provenance from.
551 Returns
552 -------
553 provenance : `lsst.ip.isr.IsrProvenance`
554 The provenance defined in the tables.
555 """
556 table = tableList[0]
557 metadata = table.meta
558 inDict = dict()
559 inDict['metadata'] = metadata
560 inDict['detectorName'] = metadata.get('DET_NAME', None)
561 inDict['detectorSerial'] = metadata.get('DET_SER', None)
562 inDict['instrument'] = metadata.get('INSTRUME', None)
563 inDict['calibType'] = metadata['calibType']
564 inDict['dimensions'] = set()
565 inDict['dataIdList'] = list()
567 schema = dict()
568 for colName in table.columns:
569 schema[colName.lower()] = colName
570 inDict['dimensions'].add(colName.lower())
571 inDict['dimensions'] = sorted(inDict['dimensions'])
573 for row in table:
574 entry = dict()
575 for dim in sorted(inDict['dimensions']):
576 entry[dim] = row[schema[dim]]
577 inDict['dataIdList'].append(entry)
579 return cls.fromDict(inDict)
581 @classmethod
582 def fromDict(cls, dictionary):
583 """Construct provenance from a dictionary.
585 Parameters
586 ----------
587 dictionary : `dict`
588 Dictionary of provenance parameters.
590 Returns
591 -------
592 provenance : `lsst.ip.isr.IsrProvenance`
593 The provenance defined in the tables.
594 """
595 calib = cls()
596 calib.updateMetadata(setDate=False, **dictionary['metadata'])
598 # These properties should be in the metadata, but occasionally
599 # are found in the dictionary itself. Check both places,
600 # ending with `None` if neither contains the information.
601 calib._detectorName = dictionary.get('detectorName',
602 dictionary['metadata'].get('DET_NAME', None))
603 calib._detectorSerial = dictionary.get('detectorSerial',
604 dictionary['metadata'].get('DET_SER', None))
605 calib._instrument = dictionary.get('instrument',
606 dictionary['metadata'].get('INSTRUME', None))
607 calib.calibType = dictionary['calibType']
608 calib.dimensions = set(dictionary['dimensions'])
609 calib.dataIdList = dictionary['dataIdList']
611 calib.updateMetadata()
612 return calib
614 def toDict(self):
615 """Return a dictionary containing the provenance information.
617 Returns
618 -------
619 dictionary : `dict`
620 Dictionary of provenance.
621 """
622 self.updateMetadata(setDate=True)
624 outDict = {}
626 metadata = self.getMetadata()
627 outDict['metadata'] = metadata
628 outDict['detectorName'] = self._detectorName
629 outDict['detectorSerial'] = self._detectorSerial
630 outDict['instrument'] = self._instrument
631 outDict['calibType'] = self.calibType
632 outDict['dimensions'] = list(self.dimensions)
633 outDict['dataIdList'] = self.dataIdList
635 return outDict
637 def toTable(self):
638 """Return a list of tables containing the provenance.
640 This seems inefficient and slow, so this may not be the best
641 way to store the data.
643 Returns
644 -------
645 tableList : `list` [`lsst.afw.table.Table`]
646 List of tables containing the provenance information
648 """
649 tableList = []
650 self.updateMetadata(setDate=True)
651 catalog = Table(rows=self.dataIdList,
652 names=self.dimensions)
653 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
654 catalog.meta = filteredMetadata
655 tableList.append(catalog)
656 return tableList