lsst.ip.isr  20.0.0-4-ge987224+1
calibType.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 import abc
22 import copy
23 import datetime
24 import os.path
25 import warnings
26 import yaml
27 from astropy.table import Table
28 from astropy.io import fits
29 
30 from lsst.log import Log
31 from lsst.daf.base import PropertyList
32 
33 
34 __all__ = ["IsrCalib", "IsrProvenance"]
35 
36 
37 class IsrCalib(abc.ABC):
38  """Generic calibration type.
39 
40  Subclasses must implement the toDict, fromDict, toTable, fromTable
41  methods that allow the calibration information to be converted
42  from dictionaries and afw tables. This will allow the calibration
43  to be persisted using the base class read/write methods.
44 
45  The validate method is intended to provide a common way to check
46  that the calibration is valid (internally consistent) and
47  appropriate (usable with the intended data). The apply method is
48  intended to allow the calibration to be applied in a consistent
49  manner.
50 
51  Parameters
52  ----------
53  detectorName : `str`, optional
54  Name of the detector this calibration is for.
55  detectorSerial : `str`, optional
56  Identifier for the detector.
57  detector : `lsst.afw.cameraGeom.Detector`, optional
58  Detector to extract metadata from.
59  log : `lsst.log.Log`, optional
60  Log for messages.
61 
62  """
63  _OBSTYPE = 'generic'
64  _SCHEMA = 'NO SCHEMA'
65  _VERSION = 0
66 
67  def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs):
68  self._detectorName = detectorName
69  self._detectorSerial = detectorSerial
71 
72  # Define the required attributes for this calibration.
73  self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
74  self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata'])
75 
76  self.log = log if log else Log.getLogger(__name__.partition(".")[2])
77 
78  if detector:
79  self.fromDetector(detector)
80  self.updateMetadata(setDate=False)
81 
82  def __str__(self):
83  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
84 
85  def __eq__(self, other):
86  """Calibration equivalence.
87 
88  Subclasses will need to check specific sub-properties. The
89  default is only to check common entries.
90  """
91  if not isinstance(other, self.__class__):
92  return False
93 
94  for attr in self._requiredAttributes:
95  if getattr(self, attr) != getattr(other, attr):
96  return False
97 
98  return True
99 
100  @property
102  return self._requiredAttributes
103 
104  @requiredAttributes.setter
105  def requiredAttributes(self, value):
106  self._requiredAttributes = value
107 
108  def getMetadata(self):
109 
110  """Retrieve metadata associated with this calibration.
111 
112  Returns
113  -------
114  meta : `lsst.daf.base.PropertyList`
115  Metadata. The returned `~lsst.daf.base.PropertyList` can be
116  modified by the caller and the changes will be written to
117  external files.
118  """
119  return self._metadata
120 
121  def setMetadata(self, metadata):
122  """Store a copy of the supplied metadata with this calibration.
123 
124  Parameters
125  ----------
126  metadata : `lsst.daf.base.PropertyList`
127  Metadata to associate with the calibration. Will be copied and
128  overwrite existing metadata.
129  """
130  if metadata is not None:
131  self._metadata = copy.copy(metadata)
132 
133  # Ensure that we have the obs type required by calibration ingest
134  self._metadata["OBSTYPE"] = self._OBSTYPE
135  self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
136  self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
137 
138  def updateMetadata(self, setDate=False, **kwargs):
139  """Update metadata keywords with new values.
140 
141  Parameters
142  ----------
143  setDate : `bool`, optional
144  Ensure the metadata CALIBDATE fields are set to the current datetime.
145  kwargs : `dict` or `collections.abc.Mapping`, optional
146  Set of key=value pairs to assign to the metadata.
147  """
148  mdOriginal = self.getMetadata()
149  mdSupplemental = dict()
150 
151  self._metadata["DETECTOR"] = self._detectorName
152  self._metadata["DETECTOR_SERIAL"] = self._detectorSerial
153 
154  if setDate:
155  date = datetime.datetime.now()
156  mdSupplemental['CALIBDATE'] = date.isoformat()
157  mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
158  mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
159 
160  mdSupplemental.update(kwargs)
161  mdOriginal.update(mdSupplemental)
162 
163  @classmethod
164  def readText(cls, filename):
165  """Read calibration representation from a yaml/ecsv file.
166 
167  Parameters
168  ----------
169  filename : `str`
170  Name of the file containing the calibration definition.
171 
172  Returns
173  -------
174  calib : `~lsst.ip.isr.IsrCalibType`
175  Calibration class.
176 
177  Raises
178  ------
179  RuntimeError :
180  Raised if the filename does not end in ".ecsv" or ".yaml".
181  """
182  if filename.endswith((".ecsv", ".ECSV")):
183  data = Table.read(filename, format='ascii.ecsv')
184  return cls.fromTable([data])
185 
186  elif filename.endswith((".yaml", ".YAML")):
187  with open(filename, 'r') as f:
188  data = yaml.load(f, Loader=yaml.CLoader)
189  return cls.fromDict(data)
190  else:
191  raise RuntimeError(f"Unknown filename extension: {filename}")
192 
193  def writeText(self, filename, format='auto'):
194  """Write the calibration data to a text file.
195 
196  Parameters
197  ----------
198  filename : `str`
199  Name of the file to write.
200  format : `str`
201  Format to write the file as. Supported values are:
202  ``"auto"`` : Determine filetype from filename.
203  ``"yaml"`` : Write as yaml.
204  ``"ecsv"`` : Write as ecsv.
205  Returns
206  -------
207  used : `str`
208  The name of the file used to write the data. This may
209  differ from the input if the format is explicitly chosen.
210 
211  Raises
212  ------
213  RuntimeError :
214  Raised if filename does not end in a known extension, or
215  if all information cannot be written.
216 
217  Notes
218  -----
219  The file is written to YAML/ECSV format and will include any
220  associated metadata.
221 
222  """
223  if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
224  outDict = self.toDict()
225  path, ext = os.path.splitext(filename)
226  filename = path + ".yaml"
227  with open(filename, 'w') as f:
228  yaml.dump(outDict, f)
229  elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
230  tableList = self.toTable()
231  if len(tableList) > 1:
232  # ECSV doesn't support multiple tables per file, so we
233  # can only write the first table.
234  raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
235 
236  table = tableList[0]
237  path, ext = os.path.splitext(filename)
238  filename = path + ".ecsv"
239  table.write(filename, format="ascii.ecsv")
240  else:
241  raise RuntimeError(f"Attempt to write to a file {filename} "
242  "that does not end in '.yaml' or '.ecsv'")
243 
244  return filename
245 
246  @classmethod
247  def readFits(cls, filename):
248  """Read calibration data from a FITS file.
249 
250  Parameters
251  ----------
252  filename : `str`
253  Filename to read data from.
254 
255  Returns
256  -------
257  calib : `lsst.ip.isr.IsrCalib`
258  Calibration contained within the file.
259  """
260  tableList = []
261  tableList.append(Table.read(filename, hdu=1))
262  extNum = 2 # Fits indices start at 1, we've read one already.
263  try:
264  with warnings.catch_warnings("error"):
265  newTable = Table.read(filename, hdu=extNum)
266  tableList.append(newTable)
267  extNum += 1
268  except Exception:
269  pass
270 
271  return cls.fromTable(tableList)
272 
273  def writeFits(self, filename):
274  """Write calibration data to a FITS file.
275 
276  Parameters
277  ----------
278  filename : `str`
279  Filename to write data to.
280 
281  Returns
282  -------
283  used : `str`
284  The name of the file used to write the data.
285 
286  """
287  tableList = self.toTable()
288  with warnings.catch_warnings():
289  warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
290  astropyList = [fits.table_to_hdu(table) for table in tableList]
291  astropyList.insert(0, fits.PrimaryHDU())
292 
293  writer = fits.HDUList(astropyList)
294  writer.writeto(filename, overwrite=True)
295  return filename
296 
297  def fromDetector(self, detector):
298  """Modify the calibration parameters to match the supplied detector.
299 
300  Parameters
301  ----------
302  detector : `lsst.afw.cameraGeom.Detector`
303  Detector to use to set parameters from.
304 
305  Raises
306  ------
307  NotImplementedError
308  This needs to be implemented by subclasses for each
309  calibration type.
310  """
311  raise NotImplementedError("Must be implemented by subclass.")
312 
313  @classmethod
314  def fromDict(cls, dictionary):
315  """Construct a calibration from a dictionary of properties.
316 
317  Must be implemented by the specific calibration subclasses.
318 
319  Parameters
320  ----------
321  dictionary : `dict`
322  Dictionary of properties.
323 
324  Returns
325  ------
326  calib : `lsst.ip.isr.CalibType`
327  Constructed calibration.
328 
329  Raises
330  ------
331  NotImplementedError :
332  Raised if not implemented.
333  """
334  raise NotImplementedError("Must be implemented by subclass.")
335 
336  def toDict(self):
337  """Return a dictionary containing the calibration properties.
338 
339  The dictionary should be able to be round-tripped through
340  `fromDict`.
341 
342  Returns
343  -------
344  dictionary : `dict`
345  Dictionary of properties.
346 
347  Raises
348  ------
349  NotImplementedError :
350  Raised if not implemented.
351  """
352  raise NotImplementedError("Must be implemented by subclass.")
353 
354  @classmethod
355  def fromTable(cls, tableList):
356  """Construct a calibration from a dictionary of properties.
357 
358  Must be implemented by the specific calibration subclasses.
359 
360  Parameters
361  ----------
362  tableList : `list` [`lsst.afw.table.Table`]
363  List of tables of properties.
364 
365  Returns
366  ------
367  calib : `lsst.ip.isr.CalibType`
368  Constructed calibration.
369 
370  Raises
371  ------
372  NotImplementedError :
373  Raised if not implemented.
374  """
375  raise NotImplementedError("Must be implemented by subclass.")
376 
377  def toTable(self):
378  """Return a list of tables containing the calibration properties.
379 
380  The table list should be able to be round-tripped through
381  `fromDict`.
382 
383  Returns
384  -------
385  tableList : `list` [`lsst.afw.table.Table`]
386  List of tables of properties.
387 
388  Raises
389  ------
390  NotImplementedError :
391  Raised if not implemented.
392  """
393  raise NotImplementedError("Must be implemented by subclass.")
394 
395  def validate(self, other=None):
396  """Validate that this calibration is defined and can be used.
397 
398  Parameters
399  ----------
400  other : `object`, optional
401  Thing to validate against.
402 
403  Returns
404  -------
405  valid : `bool`
406  Returns true if the calibration is valid and appropriate.
407  """
408  return False
409 
410  def apply(self, target):
411  """Method to apply the calibration to the target object.
412 
413  Parameters
414  ----------
415  target : `object`
416  Thing to validate against.
417 
418  Returns
419  -------
420  valid : `bool`
421  Returns true if the calibration was applied correctly.
422 
423  Raises
424  ------
425  NotImplementedError :
426  Raised if not implemented.
427  """
428  raise NotImplementedError("Must be implemented by subclass.")
429 
430 
432  """Class for the provenance of data used to construct calibration.
433 
434  Provenance is not really a calibration, but we would like to
435  record this when constructing the calibration, and it provides an
436  example of the base calibration class.
437 
438  Parameters
439  ----------
440  instrument : `str`, optional
441  Name of the instrument the data was taken with.
442  calibType : `str`, optional
443  Type of calibration this provenance was generated for.
444  detectorName : `str`, optional
445  Name of the detector this calibration is for.
446  detectorSerial : `str`, optional
447  Identifier for the detector.
448 
449  """
450  _OBSTYPE = 'IsrProvenance'
451 
452  def __init__(self, instrument="unknown", calibType="unknown",
453  **kwargs):
454  self.instrument = instrument
455  self.calibType = calibType
456  self.dimensions = set()
457  self.dataIdList = list()
458 
459  super().__init__(**kwargs)
460 
461  self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList'])
462 
463  def __str__(self):
464  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
465 
466  def __eq__(self, other):
467  return super().__eq__(other)
468 
469  def updateMetadata(self, setDate=False, **kwargs):
470  """Update calibration metadata.
471 
472  Parameters
473  ----------
474  setDate : `bool, optional
475  Update the CALIBDATE fields in the metadata to the current
476  time. Defaults to False.
477  kwargs : `dict` or `collections.abc.Mapping`, optional
478  Other keyword parameters to set in the metadata.
479  """
480  kwargs["DETECTOR"] = self._detectorName
481  kwargs["DETECTOR_SERIAL"] = self._detectorSerial
482 
483  kwargs['INSTRUME'] = self.instrument
484  kwargs['calibType'] = self.calibType
485  super().updateMetadata(setDate=setDate, **kwargs)
486 
487  def fromDataIds(self, dataIdList):
488  """Update provenance from dataId List.
489 
490  Parameters
491  ----------
492  dataIdList : `list` [`lsst.daf.butler.DataId`]
493  List of dataIds used in generating this calibration.
494  """
495  for dataId in dataIdList:
496  for key in dataId:
497  if key not in self.dimensions:
498  self.dimensions.add(key)
499  self.dataIdList.append(dataId)
500 
501  @classmethod
502  def fromTable(cls, tableList):
503  """Construct provenance from table list.
504 
505  Parameters
506  ----------
507  tableList : `list` [`lsst.afw.table.Table`]
508  List of tables to construct the provenance from.
509 
510  Returns
511  -------
512  provenance : `lsst.ip.isr.IsrProvenance`
513  The provenance defined in the tables.
514  """
515  table = tableList[0]
516  metadata = table.meta
517  inDict = dict()
518  inDict['metadata'] = metadata
519  inDict['detectorName'] = metadata['DETECTOR']
520  inDict['detectorSerial'] = metadata['DETECTOR_SERIAL']
521  inDict['instrument'] = metadata['INSTRUME']
522  inDict['calibType'] = metadata['calibType']
523  inDict['dimensions'] = set()
524  inDict['dataIdList'] = list()
525 
526  schema = dict()
527  for colName in table.columns:
528  schema[colName.lower()] = colName
529  inDict['dimensions'].add(colName.lower())
530  inDict['dimensions'] = sorted(inDict['dimensions'])
531 
532  for row in table:
533  entry = dict()
534  for dim in sorted(inDict['dimensions']):
535  entry[dim] = row[schema[dim]]
536  inDict['dataIdList'].append(entry)
537 
538  return cls.fromDict(inDict)
539 
540  @classmethod
541  def fromDict(cls, dictionary):
542  """Construct provenance from a dictionary.
543 
544  Parameters
545  ----------
546  dictionary : `dict`
547  Dictionary of provenance parameters.
548 
549  Returns
550  -------
551  provenance : `lsst.ip.isr.IsrProvenance`
552  The provenance defined in the tables.
553  """
554  calib = cls()
555  calib.updateMetadata(setDate=False, **dictionary['metadata'])
556  calib._detectorName = dictionary['detectorName']
557  calib._detectorSerial = dictionary['detectorSerial']
558  calib.instrument = dictionary['instrument']
559  calib.calibType = dictionary['calibType']
560  calib.dimensions = set(dictionary['dimensions'])
561  calib.dataIdList = dictionary['dataIdList']
562 
563  calib.updateMetadata()
564  return calib
565 
566  def toDict(self):
567  """Return a dictionary containing the provenance information.
568 
569  Returns
570  -------
571  dictionary : `dict`
572  Dictionary of provenance.
573  """
574  self.updateMetadata(setDate=True)
575 
576  outDict = {}
577 
578  metadata = self.getMetadata()
579  outDict['metadata'] = metadata
580  outDict['detectorName'] = self._detectorName
581  outDict['detectorSerial'] = self._detectorSerial
582  outDict['instrument'] = self.instrument
583  outDict['calibType'] = self.calibType
584  outDict['dimensions'] = list(self.dimensions)
585  outDict['dataIdList'] = self.dataIdList
586 
587  return outDict
588 
589  def toTable(self):
590  """Return a list of tables containing the provenance.
591 
592  This seems inefficient and slow, so this may not be the best
593  way to store the data.
594 
595  Returns
596  -------
597  tableList : `list` [`lsst.afw.table.Table`]
598  List of tables containing the provenance information
599 
600  """
601  tableList = []
602  self.updateMetadata(setDate=True)
603  catalog = Table(rows=self.dataIdList,
604  names=self.dimensions)
605  catalog.meta = self.getMetadata().toDict()
606  tableList.append(catalog)
607  return tableList
lsst::ip::isr.calibType.IsrCalib.toTable
def toTable(self)
Definition: calibType.py:377
lsst::ip::isr.calibType.IsrProvenance.updateMetadata
def updateMetadata(self, setDate=False, **kwargs)
Definition: calibType.py:469
lsst::ip::isr.calibType.IsrCalib.apply
def apply(self, target)
Definition: calibType.py:410
lsst::ip::isr.calibType.IsrCalib.fromDetector
def fromDetector(self, detector)
Definition: calibType.py:297
lsst::ip::isr.calibType.IsrCalib.writeText
def writeText(self, filename, format='auto')
Definition: calibType.py:193
lsst::ip::isr.calibType.IsrProvenance.dataIdList
dataIdList
Definition: calibType.py:456
lsst::ip::isr.calibType.IsrCalib.__eq__
def __eq__(self, other)
Definition: calibType.py:85
lsst::ip::isr.calibType.IsrProvenance.toTable
def toTable(self)
Definition: calibType.py:589
lsst::ip::isr.calibType.IsrProvenance.fromDict
def fromDict(cls, dictionary)
Definition: calibType.py:541
lsst::ip::isr.calibType.IsrCalib.__init__
def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs)
Definition: calibType.py:67
lsst::ip::isr.calibType.IsrCalib.updateMetadata
def updateMetadata(self, setDate=False, **kwargs)
Definition: calibType.py:138
lsst::ip::isr.calibType.IsrProvenance.dimensions
dimensions
Definition: calibType.py:455
lsst::ip::isr.calibType.IsrCalib.readText
def readText(cls, filename)
Definition: calibType.py:164
lsst::ip::isr.calibType.IsrCalib.validate
def validate(self, other=None)
Definition: calibType.py:395
lsst::daf::base::PropertyList
lsst::ip::isr.calibType.IsrCalib.fromTable
def fromTable(cls, tableList)
Definition: calibType.py:355
lsst::ip::isr.calibType.IsrCalib._SCHEMA
string _SCHEMA
Definition: calibType.py:64
lsst::ip::isr.calibType.IsrCalib._VERSION
int _VERSION
Definition: calibType.py:65
lsst::ip::isr.calibType.IsrCalib.log
log
Definition: calibType.py:76
lsst::ip::isr.calibType.IsrCalib._requiredAttributes
_requiredAttributes
Definition: calibType.py:106
lsst::ip::isr.calibType.IsrProvenance.__init__
def __init__(self, instrument="unknown", calibType="unknown", **kwargs)
Definition: calibType.py:452
lsst::ip::isr.calibType.IsrProvenance
Definition: calibType.py:431
lsst::ip::isr.calibType.IsrProvenance.__str__
def __str__(self)
Definition: calibType.py:463
lsst::ip::isr.calibType.IsrCalib._metadata
_metadata
Definition: calibType.py:131
lsst::ip::isr.calibType.IsrCalib._detectorSerial
_detectorSerial
Definition: calibType.py:69
lsst::ip::isr.calibType.IsrProvenance.__eq__
def __eq__(self, other)
Definition: calibType.py:466
lsst::ip::isr.calibType.IsrProvenance.toDict
def toDict(self)
Definition: calibType.py:566
lsst::ip::isr.calibType.IsrCalib.requiredAttributes
requiredAttributes
Definition: calibType.py:73
lsst::ip::isr.calibType.IsrCalib.writeFits
def writeFits(self, filename)
Definition: calibType.py:273
lsst::ip::isr.calibType.IsrProvenance.calibType
calibType
Definition: calibType.py:454
lsst::ip::isr.calibType.IsrProvenance.instrument
instrument
Definition: calibType.py:453
lsst::ip::isr.calibType.IsrProvenance.fromDataIds
def fromDataIds(self, dataIdList)
Definition: calibType.py:487
lsst::ip::isr.calibType.IsrCalib.getMetadata
def getMetadata(self)
Definition: calibType.py:108
lsst::ip::isr.calibType.IsrCalib.readFits
def readFits(cls, filename)
Definition: calibType.py:247
lsst::daf::base
lsst::ip::isr.calibType.IsrCalib.fromDict
def fromDict(cls, dictionary)
Definition: calibType.py:314
lsst::ip::isr.calibType.IsrCalib
Definition: calibType.py:37
lsst::ip::isr.calibType.IsrCalib.setMetadata
def setMetadata(self, metadata)
Definition: calibType.py:121
lsst::ip::isr.calibType.IsrProvenance.fromTable
def fromTable(cls, tableList)
Definition: calibType.py:502
lsst::ip::isr.calibType.IsrCalib._detectorName
_detectorName
Definition: calibType.py:68
lsst::log
lsst::ip::isr.calibType.IsrCalib.__str__
def __str__(self)
Definition: calibType.py:82
lsst::ip::isr.calibType.IsrCalib._OBSTYPE
string _OBSTYPE
Definition: calibType.py:63
lsst::ip::isr.calibType.IsrCalib.toDict
def toDict(self)
Definition: calibType.py:336