Coverage for python/lsst/ip/isr/calibType.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import copy
23import datetime
24import os.path
25import warnings
26import yaml
27from astropy.table import Table
28from astropy.io import fits
30from lsst.log import Log
31from lsst.daf.base import PropertyList
34__all__ = ["IsrCalib", "IsrProvenance"]
37class IsrCalib(abc.ABC):
38 """Generic calibration type.
40 Subclasses must implement the toDict, fromDict, toTable, fromTable
41 methods that allow the calibration information to be converted
42 from dictionaries and afw tables. This will allow the calibration
43 to be persisted using the base class read/write methods.
45 The validate method is intended to provide a common way to check
46 that the calibration is valid (internally consistent) and
47 appropriate (usable with the intended data). The apply method is
48 intended to allow the calibration to be applied in a consistent
49 manner.
51 Parameters
52 ----------
53 detectorName : `str`, optional
54 Name of the detector this calibration is for.
55 detectorSerial : `str`, optional
56 Identifier for the detector.
57 detector : `lsst.afw.cameraGeom.Detector`, optional
58 Detector to extract metadata from.
59 log : `lsst.log.Log`, optional
60 Log for messages.
62 """
63 _OBSTYPE = 'generic'
64 _SCHEMA = 'NO SCHEMA'
65 _VERSION = 0
67 def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs):
68 self._detectorName = detectorName
69 self._detectorSerial = detectorSerial
70 self.setMetadata(PropertyList())
72 # Define the required attributes for this calibration.
73 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
74 self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata'])
76 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
78 if detector:
79 self.fromDetector(detector)
80 self.updateMetadata(setDate=False)
82 def __str__(self):
83 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
85 def __eq__(self, other):
86 """Calibration equivalence.
88 Subclasses will need to check specific sub-properties. The
89 default is only to check common entries.
90 """
91 if not isinstance(other, self.__class__):
92 return False
94 for attr in self._requiredAttributes:
95 if getattr(self, attr) != getattr(other, attr):
96 return False
98 return True
100 @property
101 def requiredAttributes(self):
102 return self._requiredAttributes
104 @requiredAttributes.setter
105 def requiredAttributes(self, value):
106 self._requiredAttributes = value
108 def getMetadata(self):
110 """Retrieve metadata associated with this calibration.
112 Returns
113 -------
114 meta : `lsst.daf.base.PropertyList`
115 Metadata. The returned `~lsst.daf.base.PropertyList` can be
116 modified by the caller and the changes will be written to
117 external files.
118 """
119 return self._metadata
121 def setMetadata(self, metadata):
122 """Store a copy of the supplied metadata with this calibration.
124 Parameters
125 ----------
126 metadata : `lsst.daf.base.PropertyList`
127 Metadata to associate with the calibration. Will be copied and
128 overwrite existing metadata.
129 """
130 if metadata is not None:
131 self._metadata = copy.copy(metadata)
133 # Ensure that we have the obs type required by calibration ingest
134 self._metadata["OBSTYPE"] = self._OBSTYPE
135 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
136 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
138 def updateMetadata(self, setDate=False, **kwargs):
139 """Update metadata keywords with new values.
141 Parameters
142 ----------
143 setDate : `bool`, optional
144 Ensure the metadata CALIBDATE fields are set to the current datetime.
145 kwargs : `dict` or `collections.abc.Mapping`, optional
146 Set of key=value pairs to assign to the metadata.
147 """
148 mdOriginal = self.getMetadata()
149 mdSupplemental = dict()
151 self._metadata["DETECTOR"] = self._detectorName
152 self._metadata["DETECTOR_SERIAL"] = self._detectorSerial
154 if setDate:
155 date = datetime.datetime.now()
156 mdSupplemental['CALIBDATE'] = date.isoformat()
157 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
158 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
160 mdSupplemental.update(kwargs)
161 mdOriginal.update(mdSupplemental)
163 @classmethod
164 def readText(cls, filename):
165 """Read calibration representation from a yaml/ecsv file.
167 Parameters
168 ----------
169 filename : `str`
170 Name of the file containing the calibration definition.
172 Returns
173 -------
174 calib : `~lsst.ip.isr.IsrCalibType`
175 Calibration class.
177 Raises
178 ------
179 RuntimeError :
180 Raised if the filename does not end in ".ecsv" or ".yaml".
181 """
182 if filename.endswith((".ecsv", ".ECSV")):
183 data = Table.read(filename, format='ascii.ecsv')
184 return cls.fromTable([data])
186 elif filename.endswith((".yaml", ".YAML")):
187 with open(filename, 'r') as f:
188 data = yaml.load(f, Loader=yaml.CLoader)
189 return cls.fromDict(data)
190 else:
191 raise RuntimeError(f"Unknown filename extension: {filename}")
193 def writeText(self, filename, format='auto'):
194 """Write the calibration data to a text file.
196 Parameters
197 ----------
198 filename : `str`
199 Name of the file to write.
200 format : `str`
201 Format to write the file as. Supported values are:
202 ``"auto"`` : Determine filetype from filename.
203 ``"yaml"`` : Write as yaml.
204 ``"ecsv"`` : Write as ecsv.
205 Returns
206 -------
207 used : `str`
208 The name of the file used to write the data. This may
209 differ from the input if the format is explicitly chosen.
211 Raises
212 ------
213 RuntimeError :
214 Raised if filename does not end in a known extension, or
215 if all information cannot be written.
217 Notes
218 -----
219 The file is written to YAML/ECSV format and will include any
220 associated metadata.
222 """
223 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
224 outDict = self.toDict()
225 path, ext = os.path.splitext(filename)
226 filename = path + ".yaml"
227 with open(filename, 'w') as f:
228 yaml.dump(outDict, f)
229 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
230 tableList = self.toTable()
231 if len(tableList) > 1:
232 # ECSV doesn't support multiple tables per file, so we
233 # can only write the first table.
234 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
236 table = tableList[0]
237 path, ext = os.path.splitext(filename)
238 filename = path + ".ecsv"
239 table.write(filename, format="ascii.ecsv")
240 else:
241 raise RuntimeError(f"Attempt to write to a file {filename} "
242 "that does not end in '.yaml' or '.ecsv'")
244 return filename
246 @classmethod
247 def readFits(cls, filename):
248 """Read calibration data from a FITS file.
250 Parameters
251 ----------
252 filename : `str`
253 Filename to read data from.
255 Returns
256 -------
257 calib : `lsst.ip.isr.IsrCalib`
258 Calibration contained within the file.
259 """
260 tableList = []
261 tableList.append(Table.read(filename, hdu=1))
262 extNum = 2 # Fits indices start at 1, we've read one already.
263 try:
264 with warnings.catch_warnings("error"):
265 newTable = Table.read(filename, hdu=extNum)
266 tableList.append(newTable)
267 extNum += 1
268 except Exception:
269 pass
271 return cls.fromTable(tableList)
273 def writeFits(self, filename):
274 """Write calibration data to a FITS file.
276 Parameters
277 ----------
278 filename : `str`
279 Filename to write data to.
281 Returns
282 -------
283 used : `str`
284 The name of the file used to write the data.
286 """
287 tableList = self.toTable()
288 with warnings.catch_warnings():
289 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
290 astropyList = [fits.table_to_hdu(table) for table in tableList]
291 astropyList.insert(0, fits.PrimaryHDU())
293 writer = fits.HDUList(astropyList)
294 writer.writeto(filename, overwrite=True)
295 return filename
297 def fromDetector(self, detector):
298 """Modify the calibration parameters to match the supplied detector.
300 Parameters
301 ----------
302 detector : `lsst.afw.cameraGeom.Detector`
303 Detector to use to set parameters from.
305 Raises
306 ------
307 NotImplementedError
308 This needs to be implemented by subclasses for each
309 calibration type.
310 """
311 raise NotImplementedError("Must be implemented by subclass.")
313 @classmethod
314 def fromDict(cls, dictionary):
315 """Construct a calibration from a dictionary of properties.
317 Must be implemented by the specific calibration subclasses.
319 Parameters
320 ----------
321 dictionary : `dict`
322 Dictionary of properties.
324 Returns
325 ------
326 calib : `lsst.ip.isr.CalibType`
327 Constructed calibration.
329 Raises
330 ------
331 NotImplementedError :
332 Raised if not implemented.
333 """
334 raise NotImplementedError("Must be implemented by subclass.")
336 def toDict(self):
337 """Return a dictionary containing the calibration properties.
339 The dictionary should be able to be round-tripped through
340 `fromDict`.
342 Returns
343 -------
344 dictionary : `dict`
345 Dictionary of properties.
347 Raises
348 ------
349 NotImplementedError :
350 Raised if not implemented.
351 """
352 raise NotImplementedError("Must be implemented by subclass.")
354 @classmethod
355 def fromTable(cls, tableList):
356 """Construct a calibration from a dictionary of properties.
358 Must be implemented by the specific calibration subclasses.
360 Parameters
361 ----------
362 tableList : `list` [`lsst.afw.table.Table`]
363 List of tables of properties.
365 Returns
366 ------
367 calib : `lsst.ip.isr.CalibType`
368 Constructed calibration.
370 Raises
371 ------
372 NotImplementedError :
373 Raised if not implemented.
374 """
375 raise NotImplementedError("Must be implemented by subclass.")
377 def toTable(self):
378 """Return a list of tables containing the calibration properties.
380 The table list should be able to be round-tripped through
381 `fromDict`.
383 Returns
384 -------
385 tableList : `list` [`lsst.afw.table.Table`]
386 List of tables of properties.
388 Raises
389 ------
390 NotImplementedError :
391 Raised if not implemented.
392 """
393 raise NotImplementedError("Must be implemented by subclass.")
395 def validate(self, other=None):
396 """Validate that this calibration is defined and can be used.
398 Parameters
399 ----------
400 other : `object`, optional
401 Thing to validate against.
403 Returns
404 -------
405 valid : `bool`
406 Returns true if the calibration is valid and appropriate.
407 """
408 return False
410 def apply(self, target):
411 """Method to apply the calibration to the target object.
413 Parameters
414 ----------
415 target : `object`
416 Thing to validate against.
418 Returns
419 -------
420 valid : `bool`
421 Returns true if the calibration was applied correctly.
423 Raises
424 ------
425 NotImplementedError :
426 Raised if not implemented.
427 """
428 raise NotImplementedError("Must be implemented by subclass.")
431class IsrProvenance(IsrCalib):
432 """Class for the provenance of data used to construct calibration.
434 Provenance is not really a calibration, but we would like to
435 record this when constructing the calibration, and it provides an
436 example of the base calibration class.
438 Parameters
439 ----------
440 instrument : `str`, optional
441 Name of the instrument the data was taken with.
442 calibType : `str`, optional
443 Type of calibration this provenance was generated for.
444 detectorName : `str`, optional
445 Name of the detector this calibration is for.
446 detectorSerial : `str`, optional
447 Identifier for the detector.
449 """
450 _OBSTYPE = 'IsrProvenance'
452 def __init__(self, instrument="unknown", calibType="unknown",
453 **kwargs):
454 self.instrument = instrument
455 self.calibType = calibType
456 self.dimensions = set()
457 self.dataIdList = list()
459 super().__init__(**kwargs)
461 self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList'])
463 def __str__(self):
464 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
466 def __eq__(self, other):
467 return super().__eq__(other)
469 def updateMetadata(self, setDate=False, **kwargs):
470 """Update calibration metadata.
472 Parameters
473 ----------
474 setDate : `bool, optional
475 Update the CALIBDATE fields in the metadata to the current
476 time. Defaults to False.
477 kwargs : `dict` or `collections.abc.Mapping`, optional
478 Other keyword parameters to set in the metadata.
479 """
480 kwargs["DETECTOR"] = self._detectorName
481 kwargs["DETECTOR_SERIAL"] = self._detectorSerial
483 kwargs['INSTRUME'] = self.instrument
484 kwargs['calibType'] = self.calibType
485 super().updateMetadata(setDate=setDate, **kwargs)
487 def fromDataIds(self, dataIdList):
488 """Update provenance from dataId List.
490 Parameters
491 ----------
492 dataIdList : `list` [`lsst.daf.butler.DataId`]
493 List of dataIds used in generating this calibration.
494 """
495 for dataId in dataIdList:
496 for key in dataId:
497 if key not in self.dimensions:
498 self.dimensions.add(key)
499 self.dataIdList.append(dataId)
501 @classmethod
502 def fromTable(cls, tableList):
503 """Construct provenance from table list.
505 Parameters
506 ----------
507 tableList : `list` [`lsst.afw.table.Table`]
508 List of tables to construct the provenance from.
510 Returns
511 -------
512 provenance : `lsst.ip.isr.IsrProvenance`
513 The provenance defined in the tables.
514 """
515 table = tableList[0]
516 metadata = table.meta
517 inDict = dict()
518 inDict['metadata'] = metadata
519 inDict['detectorName'] = metadata['DETECTOR']
520 inDict['detectorSerial'] = metadata['DETECTOR_SERIAL']
521 inDict['instrument'] = metadata['INSTRUME']
522 inDict['calibType'] = metadata['calibType']
523 inDict['dimensions'] = set()
524 inDict['dataIdList'] = list()
526 schema = dict()
527 for colName in table.columns:
528 schema[colName.lower()] = colName
529 inDict['dimensions'].add(colName.lower())
530 inDict['dimensions'] = sorted(inDict['dimensions'])
532 for row in table:
533 entry = dict()
534 for dim in sorted(inDict['dimensions']):
535 entry[dim] = row[schema[dim]]
536 inDict['dataIdList'].append(entry)
538 return cls.fromDict(inDict)
540 @classmethod
541 def fromDict(cls, dictionary):
542 """Construct provenance from a dictionary.
544 Parameters
545 ----------
546 dictionary : `dict`
547 Dictionary of provenance parameters.
549 Returns
550 -------
551 provenance : `lsst.ip.isr.IsrProvenance`
552 The provenance defined in the tables.
553 """
554 calib = cls()
555 calib.updateMetadata(setDate=False, **dictionary['metadata'])
556 calib._detectorName = dictionary['detectorName']
557 calib._detectorSerial = dictionary['detectorSerial']
558 calib.instrument = dictionary['instrument']
559 calib.calibType = dictionary['calibType']
560 calib.dimensions = set(dictionary['dimensions'])
561 calib.dataIdList = dictionary['dataIdList']
563 calib.updateMetadata()
564 return calib
566 def toDict(self):
567 """Return a dictionary containing the provenance information.
569 Returns
570 -------
571 dictionary : `dict`
572 Dictionary of provenance.
573 """
574 self.updateMetadata(setDate=True)
576 outDict = {}
578 metadata = self.getMetadata()
579 outDict['metadata'] = metadata
580 outDict['detectorName'] = self._detectorName
581 outDict['detectorSerial'] = self._detectorSerial
582 outDict['instrument'] = self.instrument
583 outDict['calibType'] = self.calibType
584 outDict['dimensions'] = list(self.dimensions)
585 outDict['dataIdList'] = self.dataIdList
587 return outDict
589 def toTable(self):
590 """Return a list of tables containing the provenance.
592 This seems inefficient and slow, so this may not be the best
593 way to store the data.
595 Returns
596 -------
597 tableList : `list` [`lsst.afw.table.Table`]
598 List of tables containing the provenance information
600 """
601 tableList = []
602 self.updateMetadata(setDate=True)
603 catalog = Table(rows=self.dataIdList,
604 names=self.dimensions)
605 catalog.meta = self.getMetadata().toDict()
606 tableList.append(catalog)
607 return tableList