lsst.cp.pipe  20.0.0-2-g058f3a7
cpCertify.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import datetime
22 from astropy.time import Time
23 
24 import lsst.pex.config as pexConfig
25 import lsst.pipe.base as pipeBase
26 from lsst.daf.butler import DatasetType
27 
28 
29 class BlessCalibration(pipeBase.Task):
30  """Create a way to bless existing calibration products.
31 
32  The inputs are assumed to have been constructed via cp_pipe, and
33  already exist in the butler.
34 
35  Parameters
36  ----------
37  butler : `lsst.daf.butler.Butler`
38  Butler repository to use.
39  inputCollection : `str`
40  Data collection to pull calibrations from.
41  outputCollection : `str`
42  Data collection to store final calibrations.
43  **kwargs :
44  Additional arguments forwarded to `lsst.pipe.base.Task.__init__`.
45  """
46  _DefaultName = 'BlessCalibration'
47  ConfigClass = pexConfig.Config
48 
49  def __init__(self, *, butler, inputCollection, outputCollection,
50  **kwargs):
51  super().__init__(**kwargs)
52  self.butler = butler
53  self.registry = self.butler.registry
54  self.inputCollection = inputCollection
55  self.outputCollection = outputCollection
56 
57  self.calibrationLabel = None
58  self.instrument = None
59 
60  def findInputs(self, datasetTypeName, inputDatasetTypeName=None):
61  """Find and prepare inputs for blessing.
62 
63  Parameters
64  ----------
65  datasetTypeName : `str`
66  Dataset that will be blessed.
67  inputDatasetTypeName : `str`, optional
68  Dataset name for the input datasets. Default to
69  datasetTypeName + "Proposal".
70 
71  Raises
72  ------
73  RuntimeError
74  Raised if no input datasets found or if the calibration
75  label exists and is not empty.
76  """
77  if inputDatasetTypeName is None:
78  inputDatasetTypeName = datasetTypeName + "Proposal"
79 
80  self.inputValues = list(self.registry.queryDatasets(inputDatasetTypeName,
81  collections=[self.inputCollection],
82  deduplicate=True))
83  # THIS IS INELEGANT AT BEST => fixed by passing deduplicate=True above.
84  # self.inputValues = list(filter(lambda vv: self.inputCollection in vv.run, self.inputValues))
85 
86  if len(self.inputValues) == 0:
87  raise RuntimeError(f"No inputs found for dataset {inputDatasetTypeName} "
88  f"in {self.inputCollection}")
89 
90  # Construct calibration label and choose instrument to use.
91  self.calibrationLabel = f"{datasetTypeName}/{self.inputCollection}"
92  self.instrument = self.inputValues[0].dataId['instrument']
93 
94  # Prepare combination of new data ids and object data:
95  self.newDataIds = [value.dataId for value in self.inputValues]
96 
97  self.objects = [self.butler.get(value) for value in self.inputValues]
98 
99  def registerCalibrations(self, datasetTypeName):
100  """Add blessed inputs to the output collection.
101 
102  Parameters
103  ----------
104  datasetTypeName : `str`
105  Dataset type these calibrations will be registered for.
106  """
107  # Find/make the run we will use for the output
108  self.registry.registerRun(self.outputCollection)
109  self.butler.run = self.outputCollection
110  self.butler.collection = None
111 
112  try:
113  self.registerDatasetType(datasetTypeName, self.newDataIds[0])
114  except Exception as e:
115  print(f"Could not registerDatasetType {datasetTypeName}. Failure {e}?")
116 
117  with self.butler.transaction():
118  for newId, data in zip(self.newDataIds, self.objects):
119  data = self.convertStorageClass(data, datasetTypeName)
120 
121  self.butler.put(data, datasetTypeName, dataId=newId,
122  calibration_label=self.calibrationLabel,
123  producer=None)
124 
125  def convertStorageClass(self, data, datasetTypeName):
126  """Switch from an exposure to the image type expected.
127 
128  Parameters
129  ----------
130  data : `lsst.afw.image.Exposure`
131  Input exposure data to convert.
132  datasetTypeName : `str`
133  Dataset type that will be registered
134 
135  Returns
136  -------
137  data : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
138  Converted image data to register.
139  """
140  if datasetTypeName in ('bias', 'dark'):
141  data = data.getImage()
142  # elif datasetTypeName in ('flat', ):
143  # data = data.getMaskedImage()
144 
145  return data
146 
147  def registerDatasetType(self, datasetTypeName, dataId):
148  """Ensure registry can handle this dataset type.
149 
150  Parameters
151  ----------
152  datasetTypeName : `str`
153  Name of the dataset that will be registered.
154  dataId : `lsst.daf.butler.dataId`
155  Data ID providing the list of dimensions for the new
156  datasetType.
157  """
158  storageClassMap = {'bias': 'ImageF',
159  'dark': 'ImageF',
160  'flat': 'ExposureF',
161  }
162  storageClass = storageClassMap.get(datasetTypeName, 'ExposureF')
163 
164  dimensionArray = set(list(dataId.keys()) + ["calibration_label"])
165  datasetType = DatasetType(datasetTypeName,
166  dimensionArray,
167  storageClass,
168  universe=self.butler.registry.dimensions)
169  self.butler.registry.registerDatasetType(datasetType)
170 
171  def addCalibrationLabel(self, name=None, instrument=None,
172  beginDate="1970-01-01", endDate="2038-12-31"):
173 
174  """Method to allow tasks to add calibration_label for master calibrations.
175 
176  Parameters
177  ----------
178  name : `str`, optional
179  A unique string for the calibration_label key.
180  instrument : `str`, optional
181  Instrument this calibration is for.
182  beginDate : `str`, optional
183  An ISO 8601 date string for the beginning of the valid date range.
184  endDate : `str`, optional
185  An ISO 8601 date string for the end of the valid date range.
186 
187  Raises
188  ------
189  RuntimeError :
190  Raised if the instrument or calibration_label name are not set.
191  """
192  if name is None:
193  name = self.calibrationLabel
194  if instrument is None:
195  instrument = self.instrument
196  if name is None and instrument is None:
197  raise RuntimeError("Instrument and calibration_label name not set.")
198 
199  try:
200  existingValues = self.registry.queryDimensions(['calibration_label'],
201  instrument=self.instrument,
202  calibration_label=name)
203  existingValues = [a for a in existingValues]
204  print(f"Found {len(existingValues)} Entries for {self.calibrationLabel}")
205  except LookupError:
206  self.butler.registry.insertDimensionData(
207  "calibration_label",
208  {
209  "name": name,
210  "instrument": instrument,
211  "datetime_begin": Time(datetime.datetime.fromisoformat(beginDate), scale='utc'),
212  "datetime_end": Time(datetime.datetime.fromisoformat(endDate), scale='utc'),
213  }
214  )
lsst.cp.pipe.cpCertify.BlessCalibration.__init__
def __init__(self, *butler, inputCollection, outputCollection, **kwargs)
Definition: cpCertify.py:49
lsst.cp.pipe.cpCertify.BlessCalibration.inputCollection
inputCollection
Definition: cpCertify.py:53
lsst.cp.pipe.cpCertify.BlessCalibration.objects
objects
Definition: cpCertify.py:97
lsst.cp.pipe.cpCertify.BlessCalibration.inputValues
inputValues
Definition: cpCertify.py:80
lsst.cp.pipe.cpCertify.BlessCalibration.addCalibrationLabel
def addCalibrationLabel(self, name=None, instrument=None, beginDate="1970-01-01", endDate="2038-12-31")
Definition: cpCertify.py:171
lsst.cp.pipe.cpCertify.BlessCalibration.outputCollection
outputCollection
Definition: cpCertify.py:54
lsst.cp.pipe.cpCertify.BlessCalibration.registerCalibrations
def registerCalibrations(self, datasetTypeName)
Definition: cpCertify.py:99
lsst.cp.pipe.cpCertify.BlessCalibration.registry
registry
Definition: cpCertify.py:52
lsst.cp.pipe.cpCertify.BlessCalibration.instrument
instrument
Definition: cpCertify.py:57
lsst.cp.pipe.cpCertify.BlessCalibration
Definition: cpCertify.py:29
lsst.cp.pipe.cpCertify.BlessCalibration.calibrationLabel
calibrationLabel
Definition: cpCertify.py:56
lsst.cp.pipe.cpCertify.BlessCalibration.registerDatasetType
def registerDatasetType(self, datasetTypeName, dataId)
Definition: cpCertify.py:147
lsst.cp.pipe.cpCertify.BlessCalibration.butler
butler
Definition: cpCertify.py:51
lsst.cp.pipe.cpCertify.BlessCalibration.newDataIds
newDataIds
Definition: cpCertify.py:95
lsst.cp.pipe.cpCertify.BlessCalibration.findInputs
def findInputs(self, datasetTypeName, inputDatasetTypeName=None)
Definition: cpCertify.py:60
lsst::pipe::base
lsst.cp.pipe.cpCertify.BlessCalibration.convertStorageClass
def convertStorageClass(self, data, datasetTypeName)
Definition: cpCertify.py:125