lsst.cp.pipe  20.0.0-15-g4cba61e+2185b9856c
cpCertify.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import datetime
22 from astropy.time import Time
23 
24 import lsst.pex.config as pexConfig
25 import lsst.pipe.base as pipeBase
26 from lsst.daf.butler import DatasetType
27 
28 
29 class CertifyCalibration(pipeBase.Task):
30  """Create a way to bless existing calibration products.
31 
32  The inputs are assumed to have been constructed via cp_pipe, and
33  already exist in the butler.
34 
35  Parameters
36  ----------
37  butler : `lsst.daf.butler.Butler`
38  Butler repository to use.
39  inputCollection : `str`
40  Data collection to pull calibrations from.
41  outputCollection : `str`
42  Data collection to store final calibrations.
43  **kwargs :
44  Additional arguments forwarded to `lsst.pipe.base.Task.__init__`.
45  """
46  _DefaultName = 'CertifyCalibration'
47  ConfigClass = pexConfig.Config
48 
49  def __init__(self, *, butler, inputCollection, outputCollection,
50  **kwargs):
51  super().__init__(**kwargs)
52  self.butler = butler
53  self.registry = self.butler.registry
54  self.inputCollection = inputCollection
55  self.outputCollection = outputCollection
56 
57  self.calibrationLabel = None
58  self.instrument = None
59 
60  def findInputs(self, datasetTypeName, inputDatasetTypeName=None):
61  """Find and prepare inputs for blessing.
62 
63  Parameters
64  ----------
65  datasetTypeName : `str`
66  Dataset that will be blessed.
67  inputDatasetTypeName : `str`, optional
68  Dataset name for the input datasets. Default to
69  datasetTypeName + "Proposal".
70 
71  Raises
72  ------
73  RuntimeError
74  Raised if no input datasets found or if the calibration
75  label exists and is not empty.
76  """
77  if inputDatasetTypeName is None:
78  inputDatasetTypeName = datasetTypeName + "Proposal"
79 
80  self.inputValues = list(self.registry.queryDatasets(inputDatasetTypeName,
81  collections=[self.inputCollection],
82  deduplicate=True))
83  # THIS IS INELEGANT AT BEST => fixed by passing deduplicate=True above.
84  # self.inputValues = list(filter(lambda vv: self.inputCollection in vv.run, self.inputValues))
85 
86  if len(self.inputValues) == 0:
87  raise RuntimeError(f"No inputs found for dataset {inputDatasetTypeName} "
88  f"in {self.inputCollection}")
89 
90  # Construct calibration label and choose instrument to use.
91  self.calibrationLabel = f"{datasetTypeName}/{self.inputCollection}"
92  self.instrument = self.inputValues[0].dataId['instrument']
93 
94  # Prepare combination of new data ids and object data:
95  self.newDataIds = [value.dataId for value in self.inputValues]
96 
97  self.objects = [self.butler.get(value) for value in self.inputValues]
98 
99  def registerCalibrations(self, datasetTypeName):
100  """Add blessed inputs to the output collection.
101 
102  Parameters
103  ----------
104  datasetTypeName : `str`
105  Dataset type these calibrations will be registered for.
106  """
107  # Find/make the run we will use for the output
108  self.registry.registerRun(self.outputCollection)
109  self.butler.run = self.outputCollection
110  self.butler.collection = None
111 
112  try:
113  self.registerDatasetType(datasetTypeName, self.newDataIds[0])
114  except Exception as e:
115  print(f"Could not registerDatasetType {datasetTypeName}. Failure {e}?")
116 
117  with self.butler.transaction():
118  for newId, data in zip(self.newDataIds, self.objects):
119  self.butler.put(data, datasetTypeName, dataId=newId,
120  calibration_label=self.calibrationLabel,
121  producer=None)
122 
123  def registerDatasetType(self, datasetTypeName, dataId):
124  """Ensure registry can handle this dataset type.
125 
126  Parameters
127  ----------
128  datasetTypeName : `str`
129  Name of the dataset that will be registered.
130  dataId : `lsst.daf.butler.dataId`
131  Data ID providing the list of dimensions for the new
132  datasetType.
133  """
134  storageClassMap = {'crosstalk': 'CrosstalkCalib'}
135  storageClass = storageClassMap.get(datasetTypeName, 'ExposureF')
136 
137  dimensionArray = set(list(dataId.keys()) + ["calibration_label"])
138  datasetType = DatasetType(datasetTypeName,
139  dimensionArray,
140  storageClass,
141  universe=self.butler.registry.dimensions)
142  self.butler.registry.registerDatasetType(datasetType)
143 
144  def addCalibrationLabel(self, name=None, instrument=None,
145  beginDate="1970-01-01", endDate="2038-12-31"):
146 
147  """Method to allow tasks to add calibration_label for master calibrations.
148 
149  Parameters
150  ----------
151  name : `str`, optional
152  A unique string for the calibration_label key.
153  instrument : `str`, optional
154  Instrument this calibration is for.
155  beginDate : `str`, optional
156  An ISO 8601 date string for the beginning of the valid date range.
157  endDate : `str`, optional
158  An ISO 8601 date string for the end of the valid date range.
159 
160  Raises
161  ------
162  RuntimeError :
163  Raised if the instrument or calibration_label name are not set.
164  """
165  if name is None:
166  name = self.calibrationLabel
167  if instrument is None:
168  instrument = self.instrument
169  if name is None and instrument is None:
170  raise RuntimeError("Instrument and calibration_label name not set.")
171 
172  try:
173  existingValues = self.registry.queryDataIds(['calibration_label'],
174  instrument=self.instrument,
175  calibration_label=name)
176  existingValues = [a for a in existingValues]
177  print(f"Found {len(existingValues)} Entries for {self.calibrationLabel}")
178  except LookupError:
179  self.butler.registry.insertDimensionData(
180  "calibration_label",
181  {
182  "name": name,
183  "instrument": instrument,
184  "datetime_begin": Time(datetime.datetime.fromisoformat(beginDate), scale='utc'),
185  "datetime_end": Time(datetime.datetime.fromisoformat(endDate), scale='utc'),
186  }
187  )
lsst.cp.pipe.cpCertify.CertifyCalibration.butler
butler
Definition: cpCertify.py:51
lsst.cp.pipe.cpCertify.CertifyCalibration.registerCalibrations
def registerCalibrations(self, datasetTypeName)
Definition: cpCertify.py:99
lsst.cp.pipe.cpCertify.CertifyCalibration.instrument
instrument
Definition: cpCertify.py:57
lsst.cp.pipe.cpCertify.CertifyCalibration.inputValues
inputValues
Definition: cpCertify.py:80
lsst.cp.pipe.cpCertify.CertifyCalibration.registerDatasetType
def registerDatasetType(self, datasetTypeName, dataId)
Definition: cpCertify.py:123
lsst.cp.pipe.cpCertify.CertifyCalibration.__init__
def __init__(self, *butler, inputCollection, outputCollection, **kwargs)
Definition: cpCertify.py:49
lsst.cp.pipe.cpCertify.CertifyCalibration.addCalibrationLabel
def addCalibrationLabel(self, name=None, instrument=None, beginDate="1970-01-01", endDate="2038-12-31")
Definition: cpCertify.py:144
lsst.cp.pipe.cpCertify.CertifyCalibration.inputCollection
inputCollection
Definition: cpCertify.py:53
lsst.cp.pipe.cpCertify.CertifyCalibration.newDataIds
newDataIds
Definition: cpCertify.py:95
lsst.cp.pipe.cpCertify.CertifyCalibration.findInputs
def findInputs(self, datasetTypeName, inputDatasetTypeName=None)
Definition: cpCertify.py:60
lsst::pex::config
lsst.cp.pipe.cpCertify.CertifyCalibration.registry
registry
Definition: cpCertify.py:52
lsst.cp.pipe.cpCertify.CertifyCalibration
Definition: cpCertify.py:29
lsst.cp.pipe.cpCertify.CertifyCalibration.objects
objects
Definition: cpCertify.py:97
lsst.cp.pipe.cpCertify.CertifyCalibration.outputCollection
outputCollection
Definition: cpCertify.py:54
lsst.cp.pipe.cpCertify.CertifyCalibration.calibrationLabel
calibrationLabel
Definition: cpCertify.py:56
lsst::pipe::base