Coverage for python/lsst/ip/isr/calibType.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import abc
22import copy
23import datetime
24import os.path
25import warnings
26import yaml
27from astropy.table import Table
29from lsst.log import Log
30from lsst.daf.base import PropertyList
33__all__ = ["IsrCalib", "IsrProvenance"]
36class IsrCalib(abc.ABC):
37 """Generic calibration type.
39 Subclasses must implement the toDict, fromDict, toTable, fromTable
40 methods that allow the calibration information to be converted
41 from dictionaries and afw tables. This will allow the calibration
42 to be persisted using the base class read/write methods.
44 The validate method is intended to provide a common way to check
45 that the calibration is valid (internally consistent) and
46 appropriate (usable with the intended data). The apply method is
47 intended to allow the calibration to be applied in a consistent
48 manner.
50 Parameters
51 ----------
52 detectorName : `str`, optional
53 Name of the detector this calibration is for.
54 detectorSerial : `str`, optional
55 Identifier for the detector.
56 detector : `lsst.afw.cameraGeom.Detector`, optional
57 Detector to extract metadata from.
58 log : `lsst.log.Log`, optional
59 Log for messages.
61 """
62 _OBSTYPE = 'generic'
63 _SCHEMA = 'NO SCHEMA'
64 _VERSION = 0
66 def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs):
67 self._detectorName = detectorName
68 self._detectorSerial = detectorSerial
69 self.setMetadata(PropertyList())
71 # Define the required attributes for this calibration.
72 self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
73 self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata'])
75 self.log = log if log else Log.getLogger(__name__.partition(".")[2])
77 if detector:
78 self.fromDetector(detector)
79 self.updateMetadata(setDate=False)
81 def __str__(self):
82 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
84 def __eq__(self, other):
85 """Calibration equivalence.
87 Subclasses will need to check specific sub-properties. The
88 default is only to check common entries.
89 """
90 if not isinstance(other, self.__class__):
91 return False
93 for attr in self._requiredAttributes:
94 if getattr(self, attr) != getattr(other, attr):
95 return False
97 return True
99 @property
100 def requiredAttributes(self):
101 return self._requiredAttributes
103 @requiredAttributes.setter
104 def requiredAttributes(self, value):
105 self._requiredAttributes = value
107 def getMetadata(self):
109 """Retrieve metadata associated with this calibration.
111 Returns
112 -------
113 meta : `lsst.daf.base.PropertyList`
114 Metadata. The returned `~lsst.daf.base.PropertyList` can be
115 modified by the caller and the changes will be written to
116 external files.
117 """
118 return self._metadata
120 def setMetadata(self, metadata):
121 """Store a copy of the supplied metadata with this calibration.
123 Parameters
124 ----------
125 metadata : `lsst.daf.base.PropertyList`
126 Metadata to associate with the calibration. Will be copied and
127 overwrite existing metadata.
128 """
129 if metadata is not None:
130 self._metadata = copy.copy(metadata)
132 # Ensure that we have the obs type required by calibration ingest
133 self._metadata["OBSTYPE"] = self._OBSTYPE
134 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
135 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
137 def updateMetadata(self, setDate=False, **kwargs):
138 """Update metadata keywords with new values.
140 Parameters
141 ----------
142 setDate : `bool`, optional
143 Ensure the metadata CALIBDATE fields are set to the current datetime.
144 kwargs :
145 Set of key=value pairs to assign to the metadata.
146 """
147 mdOriginal = self.getMetadata()
148 mdSupplemental = dict()
150 self._metadata["DETECTOR"] = self._detectorName
151 self._metadata["DETECTOR_SERIAL"] = self._detectorSerial
153 if setDate:
154 date = datetime.datetime.now()
155 mdSupplemental['CALIBDATE'] = date.isoformat()
156 mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
157 mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
159 mdSupplemental.update(kwargs)
160 mdOriginal.update(mdSupplemental)
162 @classmethod
163 def readText(cls, filename):
164 """Read calibration representation from a yaml/ecsv file.
166 Parameters
167 ----------
168 filename : `str`
169 Name of the file containing the calibration definition.
171 Returns
172 -------
173 calib : `~lsst.ip.isr.IsrCalibType`
174 Calibration class.
176 Raises
177 ------
178 RuntimeError :
179 Raised if the filename does not end in ".ecsv" or ".yaml".
180 """
181 if filename.endswith((".ecsv", ".ECSV")):
182 data = Table.read(filename, format='ascii.ecsv')
183 return cls.fromTable([data])
185 elif filename.endswith((".yaml", ".YAML")):
186 with open(filename, 'r') as f:
187 data = yaml.load(f, Loader=yaml.CLoader)
188 return cls.fromDict(data)
189 else:
190 raise RuntimeError(f"Unknown filename extension: {filename}")
192 def writeText(self, filename, format='auto'):
193 """Write the calibration data to a text file.
195 Parameters
196 ----------
197 filename : `str`
198 Name of the file to write.
199 format : `str`
200 Format to write the file as. Supported values are:
201 ``"auto"`` : Determine filetype from filename.
202 ``"yaml"`` : Write as yaml.
203 ``"ecsv"`` : Write as ecsv.
204 Returns
205 -------
206 used : `str`
207 The name of the file used to write the data. This may
208 differ from the input if the format is explicitly chosen.
210 Raises
211 ------
212 RuntimeError :
213 Raised if filename does not end in a known extension, or
214 if all information cannot be written.
216 Notes
217 -----
218 The file is written to YAML/ECSV format and will include any
219 associated metadata.
221 """
222 if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
223 outDict = self.toDict()
224 path, ext = os.path.splitext(filename)
225 filename = path + ".yaml"
226 with open(filename, 'w') as f:
227 yaml.dump(outDict, f)
228 elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
229 tableList = self.toTable()
230 if len(tableList) > 1:
231 # ECSV doesn't support multiple tables per file, so we
232 # can only write the first table.
233 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
235 table = tableList[0]
236 path, ext = os.path.splitext(filename)
237 filename = path + ".ecsv"
238 table.write(filename, format="ascii.ecsv")
239 else:
240 raise RuntimeError(f"Attempt to write to a file {filename} "
241 "that does not end in '.yaml' or '.ecsv'")
243 return filename
245 @classmethod
246 def readFits(cls, filename):
247 """Read calibration data from a FITS file.
249 Parameters
250 ----------
251 filename : `str`
252 Filename to read data from.
254 Returns
255 -------
256 calib : `lsst.ip.isr.IsrCalib`
257 Calibration contained within the file.
258 """
259 tableList = []
260 tableList.append(Table.read(filename, hdu=1))
261 extNum = 2 # Fits indices start at 1, we've read one already.
262 try:
263 with warnings.catch_warnings("error"):
264 newTable = Table.read(filename, hdu=extNum)
265 tableList.append(newTable)
266 extNum += 1
267 except Exception:
268 pass
270 return cls.fromTable(tableList)
272 def writeFits(self, filename):
273 """Write calibration data to a FITS file.
275 Parameters
276 ----------
277 filename : `str`
278 Filename to write data to.
280 Returns
281 -------
282 used : `str`
283 The name of the file used to write the data.
285 """
286 tableList = self.toTable()
288 with warnings.catch_warnings():
289 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
290 tableList[0].write(filename)
291 for table in tableList[1:]:
292 table.write(filename, append=True)
294 return filename
296 def fromDetector(self, detector):
297 """Modify the calibration parameters to match the supplied detector.
299 Parameters
300 ----------
301 detector : `lsst.afw.cameraGeom.Detector`
302 Detector to use to set parameters from.
304 Raises
305 ------
306 NotImplementedError
307 This needs to be implemented by subclasses for each
308 calibration type.
309 """
310 raise NotImplementedError("Must be implemented by subclass.")
312 @classmethod
313 def fromDict(cls, dictionary):
314 """Construct a calibration from a dictionary of properties.
316 Must be implemented by the specific calibration subclasses.
318 Parameters
319 ----------
320 dictionary : `dict`
321 Dictionary of properties.
323 Returns
324 ------
325 calib : `lsst.ip.isr.CalibType`
326 Constructed calibration.
328 Raises
329 ------
330 NotImplementedError :
331 Raised if not implemented.
332 """
333 raise NotImplementedError("Must be implemented by subclass.")
335 def toDict(self):
336 """Return a dictionary containing the calibration properties.
338 The dictionary should be able to be round-tripped through
339 `fromDict`.
341 Returns
342 -------
343 dictionary : `dict`
344 Dictionary of properties.
346 Raises
347 ------
348 NotImplementedError :
349 Raised if not implemented.
350 """
351 raise NotImplementedError("Must be implemented by subclass.")
353 @classmethod
354 def fromTable(cls, tableList):
355 """Construct a calibration from a dictionary of properties.
357 Must be implemented by the specific calibration subclasses.
359 Parameters
360 ----------
361 tableList : `list` [`lsst.afw.table.Table`]
362 List of tables of properties.
364 Returns
365 ------
366 calib : `lsst.ip.isr.CalibType`
367 Constructed calibration.
369 Raises
370 ------
371 NotImplementedError :
372 Raised if not implemented.
373 """
374 raise NotImplementedError("Must be implemented by subclass.")
376 def toTable(self):
377 """Return a list of tables containing the calibration properties.
379 The table list should be able to be round-tripped through
380 `fromDict`.
382 Returns
383 -------
384 tableList : `list` [`lsst.afw.table.Table`]
385 List of tables of properties.
387 Raises
388 ------
389 NotImplementedError :
390 Raised if not implemented.
391 """
392 raise NotImplementedError("Must be implemented by subclass.")
394 def validate(self, other=None):
395 """Validate that this calibration is defined and can be used.
397 Parameters
398 ----------
399 other : `object`, optional
400 Thing to validate against.
402 Returns
403 -------
404 valid : `bool`
405 Returns true if the calibration is valid and appropriate.
406 """
407 return False
409 def apply(self, target):
410 """Method to apply the calibration to the target object.
412 Parameters
413 ----------
414 target : `object`
415 Thing to validate against.
417 Returns
418 -------
419 valid : `bool`
420 Returns true if the calibration was applied correctly.
422 Raises
423 ------
424 NotImplementedError :
425 Raised if not implemented.
426 """
427 raise NotImplementedError("Must be implemented by subclass.")
430class IsrProvenance(IsrCalib):
431 """Class for the provenance of data used to construct calibration.
433 Provenance is not really a calibration, but we would like to
434 record this when constructing the calibration, and it provides an
435 example of the base calibration class.
437 Parameters
438 ----------
439 instrument : `str`, optional
440 Name of the instrument the data was taken with.
441 calibType : `str`, optional
442 Type of calibration this provenance was generated for.
443 detectorName : `str`, optional
444 Name of the detector this calibration is for.
445 detectorSerial : `str`, optional
446 Identifier for the detector.
448 """
449 _OBSTYPE = 'IsrProvenance'
451 def __init__(self, instrument="unknown", calibType="unknown",
452 **kwargs):
453 self.instrument = instrument
454 self.calibType = calibType
455 self.dimensions = set()
456 self.dataIdList = list()
458 super().__init__(**kwargs)
460 self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList'])
462 def __str__(self):
463 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
465 def __eq__(self, other):
466 return super().__eq__(other)
468 def updateMetadata(self, setDate=False, **kwargs):
469 """Update calibration metadata.
471 Parameters
472 ----------
473 setDate : `bool, optional
474 Update the CALIBDATE fields in the metadata to the current
475 time. Defaults to False.
476 kwargs :
477 Other keyword parameters to set in the metadata.
478 """
479 kwargs["DETECTOR"] = self._detectorName
480 kwargs["DETECTOR_SERIAL"] = self._detectorSerial
482 kwargs['INSTRUME'] = self.instrument
483 kwargs['calibType'] = self.calibType
484 super().updateMetadata(setDate=setDate, **kwargs)
486 def fromDataIds(self, dataIdList):
487 """Update provenance from dataId List.
489 Parameters
490 ----------
491 dataIdList : `list` [`lsst.daf.butler.DataId`]
492 List of dataIds used in generating this calibration.
493 """
494 for dataId in dataIdList:
495 for key in dataId:
496 if key not in self.dimensions:
497 self.dimensions.add(key)
498 self.dataIdList.append(dataId)
500 @classmethod
501 def fromTable(cls, tableList):
502 """Construct provenance from table list.
504 Parameters
505 ----------
506 tableList : `list` [`lsst.afw.table.Table`]
507 List of tables to construct the provenance from.
509 Returns
510 -------
511 provenance : `lsst.ip.isr.IsrProvenance`
512 The provenance defined in the tables.
513 """
514 table = tableList[0]
515 metadata = table.meta
516 inDict = dict()
517 inDict['metadata'] = metadata
518 inDict['detectorName'] = metadata['DETECTOR']
519 inDict['detectorSerial'] = metadata['DETECTOR_SERIAL']
520 inDict['instrument'] = metadata['INSTRUME']
521 inDict['calibType'] = metadata['calibType']
522 inDict['dimensions'] = set()
523 inDict['dataIdList'] = list()
525 schema = dict()
526 for colName in table.columns:
527 schema[colName.lower()] = colName
528 inDict['dimensions'].add(colName.lower())
529 inDict['dimensions'] = sorted(inDict['dimensions'])
531 for row in table:
532 entry = dict()
533 for dim in sorted(inDict['dimensions']):
534 entry[dim] = row[schema[dim]]
535 inDict['dataIdList'].append(entry)
537 return cls.fromDict(inDict)
539 @classmethod
540 def fromDict(cls, dictionary):
541 """Construct provenance from a dictionary.
543 Parameters
544 ----------
545 dictionary : `dict`
546 Dictionary of provenance parameters.
548 Returns
549 -------
550 provenance : `lsst.ip.isr.IsrProvenance`
551 The provenance defined in the tables.
552 """
553 calib = cls()
554 calib.updateMetadata(setDate=False, **dictionary['metadata'])
555 calib._detectorName = dictionary['detectorName']
556 calib._detectorSerial = dictionary['detectorSerial']
557 calib.instrument = dictionary['instrument']
558 calib.calibType = dictionary['calibType']
559 calib.dimensions = set(dictionary['dimensions'])
560 calib.dataIdList = dictionary['dataIdList']
562 calib.updateMetadata()
563 return calib
565 def toDict(self):
566 """Return a dictionary containing the provenance information.
568 Returns
569 -------
570 dictionary : `dict`
571 Dictionary of provenance.
572 """
573 self.updateMetadata(setDate=True)
575 outDict = {}
577 metadata = self.getMetadata()
578 outDict['metadata'] = metadata
579 outDict['detectorName'] = self._detectorName
580 outDict['detectorSerial'] = self._detectorSerial
581 outDict['instrument'] = self.instrument
582 outDict['calibType'] = self.calibType
583 outDict['dimensions'] = list(self.dimensions)
584 outDict['dataIdList'] = self.dataIdList
586 return outDict
588 def toTable(self):
589 """Return a list of tables containing the provenance.
591 This seems inefficient and slow, so this may not be the best
592 way to store the data.
594 Returns
595 -------
596 tableList : `list` [`lsst.afw.table.Table`]
597 List of tables containing the provenance information
599 """
600 tableList = []
601 self.updateMetadata(setDate=True)
602 catalog = Table(rows=self.dataIdList,
603 names=self.dimensions)
604 catalog.meta = self.getMetadata().toDict()
605 tableList.append(catalog)
606 return tableList