lsst.ip.isr  19.0.0-25-gb330496+2
calibType.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 import abc
22 import copy
23 import datetime
24 import os.path
25 import warnings
26 import yaml
27 from astropy.table import Table
28 
29 from lsst.log import Log
30 from lsst.daf.base import PropertyList
31 
32 
33 __all__ = ["IsrCalib", "IsrProvenance"]
34 
35 
36 class IsrCalib(abc.ABC):
37  """Generic calibration type.
38 
39  Subclasses must implement the toDict, fromDict, toTable, fromTable
40  methods that allow the calibration information to be converted
41  from dictionaries and afw tables. This will allow the calibration
42  to be persisted using the base class read/write methods.
43 
44  The validate method is intended to provide a common way to check
45  that the calibration is valid (internally consistent) and
46  appropriate (usable with the intended data). The apply method is
47  intended to allow the calibration to be applied in a consistent
48  manner.
49 
50  Parameters
51  ----------
52  detectorName : `str`, optional
53  Name of the detector this calibration is for.
54  detectorSerial : `str`, optional
55  Identifier for the detector.
56  detector : `lsst.afw.cameraGeom.Detector`, optional
57  Detector to extract metadata from.
58  log : `lsst.log.Log`, optional
59  Log for messages.
60 
61  """
62  _OBSTYPE = 'generic'
63  _SCHEMA = 'NO SCHEMA'
64  _VERSION = 0
65 
66  def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs):
67  self._detectorName = detectorName
68  self._detectorSerial = detectorSerial
70 
71  # Define the required attributes for this calibration.
72  self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
73  self.requiredAttributes.update(['_detectorName', '_detectorSerial', '_metadata'])
74 
75  self.log = log if log else Log.getLogger(__name__.partition(".")[2])
76 
77  if detector:
78  self.fromDetector(detector)
79  self.updateMetadata(setDate=False)
80 
81  def __str__(self):
82  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
83 
84  def __eq__(self, other):
85  """Calibration equivalence.
86 
87  Subclasses will need to check specific sub-properties. The
88  default is only to check common entries.
89  """
90  if not isinstance(other, self.__class__):
91  return False
92 
93  for attr in self._requiredAttributes:
94  if getattr(self, attr) != getattr(other, attr):
95  return False
96 
97  return True
98 
99  @property
101  return self._requiredAttributes
102 
103  @requiredAttributes.setter
104  def requiredAttributes(self, value):
105  self._requiredAttributes = value
106 
107  def getMetadata(self):
108 
109  """Retrieve metadata associated with this calibration.
110 
111  Returns
112  -------
113  meta : `lsst.daf.base.PropertyList`
114  Metadata. The returned `~lsst.daf.base.PropertyList` can be
115  modified by the caller and the changes will be written to
116  external files.
117  """
118  return self._metadata
119 
120  def setMetadata(self, metadata):
121  """Store a copy of the supplied metadata with this calibration.
122 
123  Parameters
124  ----------
125  metadata : `lsst.daf.base.PropertyList`
126  Metadata to associate with the calibration. Will be copied and
127  overwrite existing metadata.
128  """
129  if metadata is not None:
130  self._metadata = copy.copy(metadata)
131 
132  # Ensure that we have the obs type required by calibration ingest
133  self._metadata["OBSTYPE"] = self._OBSTYPE
134  self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
135  self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
136 
137  def updateMetadata(self, setDate=False, **kwargs):
138  """Update metadata keywords with new values.
139 
140  Parameters
141  ----------
142  setDate : `bool`, optional
143  Ensure the metadata CALIBDATE fields are set to the current datetime.
144  kwargs :
145  Set of key=value pairs to assign to the metadata.
146  """
147  mdOriginal = self.getMetadata()
148  mdSupplemental = dict()
149 
150  self._metadata["DETECTOR"] = self._detectorName
151  self._metadata["DETECTOR_SERIAL"] = self._detectorSerial
152 
153  if setDate:
154  date = datetime.datetime.now()
155  mdSupplemental['CALIBDATE'] = date.isoformat()
156  mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
157  mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
158 
159  mdSupplemental.update(kwargs)
160  mdOriginal.update(mdSupplemental)
161 
162  @classmethod
163  def readText(cls, filename):
164  """Read calibration representation from a yaml/ecsv file.
165 
166  Parameters
167  ----------
168  filename : `str`
169  Name of the file containing the calibration definition.
170 
171  Returns
172  -------
173  calib : `~lsst.ip.isr.IsrCalibType`
174  Calibration class.
175 
176  Raises
177  ------
178  RuntimeError :
179  Raised if the filename does not end in ".ecsv" or ".yaml".
180  """
181  if filename.endswith((".ecsv", ".ECSV")):
182  data = Table.read(filename, format='ascii.ecsv')
183  return cls.fromTable([data])
184 
185  elif filename.endswith((".yaml", ".YAML")):
186  with open(filename, 'r') as f:
187  data = yaml.load(f, Loader=yaml.CLoader)
188  return cls.fromDict(data)
189  else:
190  raise RuntimeError(f"Unknown filename extension: {filename}")
191 
192  def writeText(self, filename, format='auto'):
193  """Write the calibration data to a text file.
194 
195  Parameters
196  ----------
197  filename : `str`
198  Name of the file to write.
199  format : `str`
200  Format to write the file as. Supported values are:
201  ``"auto"`` : Determine filetype from filename.
202  ``"yaml"`` : Write as yaml.
203  ``"ecsv"`` : Write as ecsv.
204  Returns
205  -------
206  used : `str`
207  The name of the file used to write the data. This may
208  differ from the input if the format is explicitly chosen.
209 
210  Raises
211  ------
212  RuntimeError :
213  Raised if filename does not end in a known extension, or
214  if all information cannot be written.
215 
216  Notes
217  -----
218  The file is written to YAML/ECSV format and will include any
219  associated metadata.
220 
221  """
222  if format == 'yaml' or (format == 'auto' and filename.lower().endswith((".yaml", ".YAML"))):
223  outDict = self.toDict()
224  path, ext = os.path.splitext(filename)
225  filename = path + ".yaml"
226  with open(filename, 'w') as f:
227  yaml.dump(outDict, f)
228  elif format == 'ecsv' or (format == 'auto' and filename.lower().endswith((".ecsv", ".ECSV"))):
229  tableList = self.toTable()
230  if len(tableList) > 1:
231  # ECSV doesn't support multiple tables per file, so we
232  # can only write the first table.
233  raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
234 
235  table = tableList[0]
236  path, ext = os.path.splitext(filename)
237  filename = path + ".ecsv"
238  table.write(filename, format="ascii.ecsv")
239  else:
240  raise RuntimeError(f"Attempt to write to a file {filename} "
241  "that does not end in '.yaml' or '.ecsv'")
242 
243  return filename
244 
245  @classmethod
246  def readFits(cls, filename):
247  """Read calibration data from a FITS file.
248 
249  Parameters
250  ----------
251  filename : `str`
252  Filename to read data from.
253 
254  Returns
255  -------
256  calib : `lsst.ip.isr.IsrCalib`
257  Calibration contained within the file.
258  """
259  tableList = []
260  tableList.append(Table.read(filename, hdu=1))
261  extNum = 2 # Fits indices start at 1, we've read one already.
262  try:
263  with warnings.catch_warnings("error"):
264  newTable = Table.read(filename, hdu=extNum)
265  tableList.append(newTable)
266  extNum += 1
267  except Exception:
268  pass
269 
270  return cls.fromTable(tableList)
271 
272  def writeFits(self, filename):
273  """Write calibration data to a FITS file.
274 
275  Parameters
276  ----------
277  filename : `str`
278  Filename to write data to.
279 
280  Returns
281  -------
282  used : `str`
283  The name of the file used to write the data.
284 
285  """
286  tableList = self.toTable()
287 
288  with warnings.catch_warnings():
289  warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
290  tableList[0].write(filename)
291  for table in tableList[1:]:
292  table.write(filename, append=True)
293 
294  return filename
295 
296  def fromDetector(self, detector):
297  """Modify the calibration parameters to match the supplied detector.
298 
299  Parameters
300  ----------
301  detector : `lsst.afw.cameraGeom.Detector`
302  Detector to use to set parameters from.
303 
304  Raises
305  ------
306  NotImplementedError
307  This needs to be implemented by subclasses for each
308  calibration type.
309  """
310  raise NotImplementedError("Must be implemented by subclass.")
311 
312  @classmethod
313  def fromDict(cls, dictionary):
314  """Construct a calibration from a dictionary of properties.
315 
316  Must be implemented by the specific calibration subclasses.
317 
318  Parameters
319  ----------
320  dictionary : `dict`
321  Dictionary of properties.
322 
323  Returns
324  ------
325  calib : `lsst.ip.isr.CalibType`
326  Constructed calibration.
327 
328  Raises
329  ------
330  NotImplementedError :
331  Raised if not implemented.
332  """
333  raise NotImplementedError("Must be implemented by subclass.")
334 
335  def toDict(self):
336  """Return a dictionary containing the calibration properties.
337 
338  The dictionary should be able to be round-tripped through
339  `fromDict`.
340 
341  Returns
342  -------
343  dictionary : `dict`
344  Dictionary of properties.
345 
346  Raises
347  ------
348  NotImplementedError :
349  Raised if not implemented.
350  """
351  raise NotImplementedError("Must be implemented by subclass.")
352 
353  @classmethod
354  def fromTable(cls, tableList):
355  """Construct a calibration from a dictionary of properties.
356 
357  Must be implemented by the specific calibration subclasses.
358 
359  Parameters
360  ----------
361  tableList : `list` [`lsst.afw.table.Table`]
362  List of tables of properties.
363 
364  Returns
365  ------
366  calib : `lsst.ip.isr.CalibType`
367  Constructed calibration.
368 
369  Raises
370  ------
371  NotImplementedError :
372  Raised if not implemented.
373  """
374  raise NotImplementedError("Must be implemented by subclass.")
375 
376  def toTable(self):
377  """Return a list of tables containing the calibration properties.
378 
379  The table list should be able to be round-tripped through
380  `fromDict`.
381 
382  Returns
383  -------
384  tableList : `list` [`lsst.afw.table.Table`]
385  List of tables of properties.
386 
387  Raises
388  ------
389  NotImplementedError :
390  Raised if not implemented.
391  """
392  raise NotImplementedError("Must be implemented by subclass.")
393 
394  def validate(self, other=None):
395  """Validate that this calibration is defined and can be used.
396 
397  Parameters
398  ----------
399  other : `object`, optional
400  Thing to validate against.
401 
402  Returns
403  -------
404  valid : `bool`
405  Returns true if the calibration is valid and appropriate.
406  """
407  return False
408 
409  def apply(self, target):
410  """Method to apply the calibration to the target object.
411 
412  Parameters
413  ----------
414  target : `object`
415  Thing to validate against.
416 
417  Returns
418  -------
419  valid : `bool`
420  Returns true if the calibration was applied correctly.
421 
422  Raises
423  ------
424  NotImplementedError :
425  Raised if not implemented.
426  """
427  raise NotImplementedError("Must be implemented by subclass.")
428 
429 
431  """Class for the provenance of data used to construct calibration.
432 
433  Provenance is not really a calibration, but we would like to
434  record this when constructing the calibration, and it provides an
435  example of the base calibration class.
436 
437  Parameters
438  ----------
439  instrument : `str`, optional
440  Name of the instrument the data was taken with.
441  calibType : `str`, optional
442  Type of calibration this provenance was generated for.
443  detectorName : `str`, optional
444  Name of the detector this calibration is for.
445  detectorSerial : `str`, optional
446  Identifier for the detector.
447 
448  """
449  _OBSTYPE = 'IsrProvenance'
450 
451  def __init__(self, instrument="unknown", calibType="unknown",
452  **kwargs):
453  self.instrument = instrument
454  self.calibType = calibType
455  self.dimensions = set()
456  self.dataIdList = list()
457 
458  super().__init__(**kwargs)
459 
460  self.requiredAttributes.update(['instrument', 'calibType', 'dimensions', 'dataIdList'])
461 
462  def __str__(self):
463  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
464 
465  def __eq__(self, other):
466  return super().__eq__(other)
467 
468  def updateMetadata(self, setDate=False, **kwargs):
469  """Update calibration metadata.
470 
471  Parameters
472  ----------
473  setDate : `bool, optional
474  Update the CALIBDATE fields in the metadata to the current
475  time. Defaults to False.
476  kwargs :
477  Other keyword parameters to set in the metadata.
478  """
479  kwargs["DETECTOR"] = self._detectorName
480  kwargs["DETECTOR_SERIAL"] = self._detectorSerial
481 
482  kwargs['INSTRUME'] = self.instrument
483  kwargs['calibType'] = self.calibType
484  super().updateMetadata(setDate=setDate, **kwargs)
485 
486  def fromDataIds(self, dataIdList):
487  """Update provenance from dataId List.
488 
489  Parameters
490  ----------
491  dataIdList : `list` [`lsst.daf.butler.DataId`]
492  List of dataIds used in generating this calibration.
493  """
494  for dataId in dataIdList:
495  for key in dataId:
496  if key not in self.dimensions:
497  self.dimensions.add(key)
498  self.dataIdList.append(dataId)
499 
500  @classmethod
501  def fromTable(cls, tableList):
502  """Construct provenance from table list.
503 
504  Parameters
505  ----------
506  tableList : `list` [`lsst.afw.table.Table`]
507  List of tables to construct the provenance from.
508 
509  Returns
510  -------
511  provenance : `lsst.ip.isr.IsrProvenance`
512  The provenance defined in the tables.
513  """
514  table = tableList[0]
515  metadata = table.meta
516  inDict = dict()
517  inDict['metadata'] = metadata
518  inDict['detectorName'] = metadata['DETECTOR']
519  inDict['detectorSerial'] = metadata['DETECTOR_SERIAL']
520  inDict['instrument'] = metadata['INSTRUME']
521  inDict['calibType'] = metadata['calibType']
522  inDict['dimensions'] = set()
523  inDict['dataIdList'] = list()
524 
525  schema = dict()
526  for colName in table.columns:
527  schema[colName.lower()] = colName
528  inDict['dimensions'].add(colName.lower())
529  inDict['dimensions'] = sorted(inDict['dimensions'])
530 
531  for row in table:
532  entry = dict()
533  for dim in sorted(inDict['dimensions']):
534  entry[dim] = row[schema[dim]]
535  inDict['dataIdList'].append(entry)
536 
537  return cls.fromDict(inDict)
538 
539  @classmethod
540  def fromDict(cls, dictionary):
541  """Construct provenance from a dictionary.
542 
543  Parameters
544  ----------
545  dictionary : `dict`
546  Dictionary of provenance parameters.
547 
548  Returns
549  -------
550  provenance : `lsst.ip.isr.IsrProvenance`
551  The provenance defined in the tables.
552  """
553  calib = cls()
554  calib.updateMetadata(setDate=False, **dictionary['metadata'])
555  calib._detectorName = dictionary['detectorName']
556  calib._detectorSerial = dictionary['detectorSerial']
557  calib.instrument = dictionary['instrument']
558  calib.calibType = dictionary['calibType']
559  calib.dimensions = set(dictionary['dimensions'])
560  calib.dataIdList = dictionary['dataIdList']
561 
562  calib.updateMetadata()
563  return calib
564 
565  def toDict(self):
566  """Return a dictionary containing the provenance information.
567 
568  Returns
569  -------
570  dictionary : `dict`
571  Dictionary of provenance.
572  """
573  self.updateMetadata(setDate=True)
574 
575  outDict = {}
576 
577  metadata = self.getMetadata()
578  outDict['metadata'] = metadata
579  outDict['detectorName'] = self._detectorName
580  outDict['detectorSerial'] = self._detectorSerial
581  outDict['instrument'] = self.instrument
582  outDict['calibType'] = self.calibType
583  outDict['dimensions'] = list(self.dimensions)
584  outDict['dataIdList'] = self.dataIdList
585 
586  return outDict
587 
588  def toTable(self):
589  """Return a list of tables containing the provenance.
590 
591  This seems inefficient and slow, so this may not be the best
592  way to store the data.
593 
594  Returns
595  -------
596  tableList : `list` [`lsst.afw.table.Table`]
597  List of tables containing the provenance information
598 
599  """
600  tableList = []
601  self.updateMetadata(setDate=True)
602  catalog = Table(rows=self.dataIdList,
603  names=self.dimensions)
604  catalog.meta = self.getMetadata().toDict()
605  tableList.append(catalog)
606  return tableList
lsst::ip::isr.calibType.IsrCalib.toTable
def toTable(self)
Definition: calibType.py:376
lsst::ip::isr.calibType.IsrProvenance.updateMetadata
def updateMetadata(self, setDate=False, **kwargs)
Definition: calibType.py:468
lsst::ip::isr.calibType.IsrCalib.apply
def apply(self, target)
Definition: calibType.py:409
lsst::ip::isr.calibType.IsrCalib.fromDetector
def fromDetector(self, detector)
Definition: calibType.py:296
lsst::ip::isr.calibType.IsrCalib.writeText
def writeText(self, filename, format='auto')
Definition: calibType.py:192
lsst::ip::isr.calibType.IsrProvenance.dataIdList
dataIdList
Definition: calibType.py:455
lsst::ip::isr.calibType.IsrCalib.__eq__
def __eq__(self, other)
Definition: calibType.py:84
lsst::ip::isr.calibType.IsrProvenance.toTable
def toTable(self)
Definition: calibType.py:588
lsst::ip::isr.calibType.IsrProvenance.fromDict
def fromDict(cls, dictionary)
Definition: calibType.py:540
lsst::ip::isr.calibType.IsrCalib.__init__
def __init__(self, detectorName=None, detectorSerial=None, detector=None, log=None, **kwargs)
Definition: calibType.py:66
lsst::ip::isr.calibType.IsrCalib.updateMetadata
def updateMetadata(self, setDate=False, **kwargs)
Definition: calibType.py:137
lsst::ip::isr.calibType.IsrProvenance.dimensions
dimensions
Definition: calibType.py:454
lsst::ip::isr.calibType.IsrCalib.readText
def readText(cls, filename)
Definition: calibType.py:163
lsst::ip::isr.calibType.IsrCalib.validate
def validate(self, other=None)
Definition: calibType.py:394
lsst::daf::base::PropertyList
lsst::ip::isr.calibType.IsrCalib.fromTable
def fromTable(cls, tableList)
Definition: calibType.py:354
lsst::ip::isr.calibType.IsrCalib._SCHEMA
string _SCHEMA
Definition: calibType.py:63
lsst::ip::isr.calibType.IsrCalib._VERSION
int _VERSION
Definition: calibType.py:64
lsst::ip::isr.calibType.IsrCalib.log
log
Definition: calibType.py:75
lsst::ip::isr.calibType.IsrCalib._requiredAttributes
_requiredAttributes
Definition: calibType.py:105
lsst::ip::isr.calibType.IsrProvenance.__init__
def __init__(self, instrument="unknown", calibType="unknown", **kwargs)
Definition: calibType.py:451
lsst::ip::isr.calibType.IsrProvenance
Definition: calibType.py:430
lsst::ip::isr.calibType.IsrProvenance.__str__
def __str__(self)
Definition: calibType.py:462
lsst::ip::isr.calibType.IsrCalib._metadata
_metadata
Definition: calibType.py:130
lsst::ip::isr.calibType.IsrCalib._detectorSerial
_detectorSerial
Definition: calibType.py:68
lsst::ip::isr.calibType.IsrProvenance.__eq__
def __eq__(self, other)
Definition: calibType.py:465
lsst::ip::isr.calibType.IsrProvenance.toDict
def toDict(self)
Definition: calibType.py:565
lsst::ip::isr.calibType.IsrCalib.requiredAttributes
requiredAttributes
Definition: calibType.py:72
lsst::ip::isr.calibType.IsrCalib.writeFits
def writeFits(self, filename)
Definition: calibType.py:272
lsst::ip::isr.calibType.IsrProvenance.calibType
calibType
Definition: calibType.py:453
lsst::ip::isr.calibType.IsrProvenance.instrument
instrument
Definition: calibType.py:452
lsst::ip::isr.calibType.IsrProvenance.fromDataIds
def fromDataIds(self, dataIdList)
Definition: calibType.py:486
lsst::ip::isr.calibType.IsrCalib.getMetadata
def getMetadata(self)
Definition: calibType.py:107
lsst::ip::isr.calibType.IsrCalib.readFits
def readFits(cls, filename)
Definition: calibType.py:246
lsst::daf::base
lsst::ip::isr.calibType.IsrCalib.fromDict
def fromDict(cls, dictionary)
Definition: calibType.py:313
lsst::ip::isr.calibType.IsrCalib
Definition: calibType.py:36
lsst::ip::isr.calibType.IsrCalib.setMetadata
def setMetadata(self, metadata)
Definition: calibType.py:120
lsst::ip::isr.calibType.IsrProvenance.fromTable
def fromTable(cls, tableList)
Definition: calibType.py:501
lsst::ip::isr.calibType.IsrCalib._detectorName
_detectorName
Definition: calibType.py:67
lsst::log
lsst::ip::isr.calibType.IsrCalib.__str__
def __str__(self)
Definition: calibType.py:81
lsst::ip::isr.calibType.IsrCalib._OBSTYPE
string _OBSTYPE
Definition: calibType.py:62
lsst::ip::isr.calibType.IsrCalib.toDict
def toDict(self)
Definition: calibType.py:335