Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import datetime 

22from astropy.time import Time 

23 

24import lsst.pex.config as pexConfig 

25import lsst.pipe.base as pipeBase 

26from lsst.daf.butler import DatasetType 

27 

28 

29class CertifyCalibration(pipeBase.Task): 

30 """Create a way to bless existing calibration products. 

31 

32 The inputs are assumed to have been constructed via cp_pipe, and 

33 already exist in the butler. 

34 

35 Parameters 

36 ---------- 

37 butler : `lsst.daf.butler.Butler` 

38 Butler repository to use. 

39 inputCollection : `str` 

40 Data collection to pull calibrations from. 

41 outputCollection : `str` 

42 Data collection to store final calibrations. 

43 **kwargs : 

44 Additional arguments forwarded to `lsst.pipe.base.Task.__init__`. 

45 """ 

46 _DefaultName = 'CertifyCalibration' 

47 ConfigClass = pexConfig.Config 

48 

49 def __init__(self, *, butler, inputCollection, outputCollection, 

50 **kwargs): 

51 super().__init__(**kwargs) 

52 self.butler = butler 

53 self.registry = self.butler.registry 

54 self.inputCollection = inputCollection 

55 self.outputCollection = outputCollection 

56 

57 self.calibrationLabel = None 

58 self.instrument = None 

59 

60 def findInputs(self, datasetTypeName, inputDatasetTypeName=None): 

61 """Find and prepare inputs for blessing. 

62 

63 Parameters 

64 ---------- 

65 datasetTypeName : `str` 

66 Dataset that will be blessed. 

67 inputDatasetTypeName : `str`, optional 

68 Dataset name for the input datasets. Default to 

69 datasetTypeName + "Proposal". 

70 

71 Raises 

72 ------ 

73 RuntimeError 

74 Raised if no input datasets found or if the calibration 

75 label exists and is not empty. 

76 """ 

77 if inputDatasetTypeName is None: 

78 inputDatasetTypeName = datasetTypeName + "Proposal" 

79 

80 self.inputValues = list(self.registry.queryDatasets(inputDatasetTypeName, 

81 collections=[self.inputCollection], 

82 deduplicate=True)) 

83 # THIS IS INELEGANT AT BEST => fixed by passing deduplicate=True above. 

84 # self.inputValues = list(filter(lambda vv: self.inputCollection in vv.run, self.inputValues)) 

85 

86 if len(self.inputValues) == 0: 

87 raise RuntimeError(f"No inputs found for dataset {inputDatasetTypeName} " 

88 f"in {self.inputCollection}") 

89 

90 # Construct calibration label and choose instrument to use. 

91 self.calibrationLabel = f"{datasetTypeName}/{self.inputCollection}" 

92 self.instrument = self.inputValues[0].dataId['instrument'] 

93 

94 # Prepare combination of new data ids and object data: 

95 self.newDataIds = [value.dataId for value in self.inputValues] 

96 

97 self.objects = [self.butler.get(value) for value in self.inputValues] 

98 

99 def registerCalibrations(self, datasetTypeName): 

100 """Add blessed inputs to the output collection. 

101 

102 Parameters 

103 ---------- 

104 datasetTypeName : `str` 

105 Dataset type these calibrations will be registered for. 

106 """ 

107 # Find/make the run we will use for the output 

108 self.registry.registerRun(self.outputCollection) 

109 self.butler.run = self.outputCollection 

110 self.butler.collection = None 

111 

112 try: 

113 self.registerDatasetType(datasetTypeName, self.newDataIds[0]) 

114 except Exception as e: 

115 print(f"Could not registerDatasetType {datasetTypeName}. Failure {e}?") 

116 

117 with self.butler.transaction(): 

118 for newId, data in zip(self.newDataIds, self.objects): 

119 self.butler.put(data, datasetTypeName, dataId=newId, 

120 calibration_label=self.calibrationLabel, 

121 producer=None) 

122 

123 def registerDatasetType(self, datasetTypeName, dataId): 

124 """Ensure registry can handle this dataset type. 

125 

126 Parameters 

127 ---------- 

128 datasetTypeName : `str` 

129 Name of the dataset that will be registered. 

130 dataId : `lsst.daf.butler.dataId` 

131 Data ID providing the list of dimensions for the new 

132 datasetType. 

133 """ 

134 storageClassMap = {'crosstalk': 'CrosstalkCalib'} 

135 storageClass = storageClassMap.get(datasetTypeName, 'ExposureF') 

136 

137 dimensionArray = set(list(dataId.keys()) + ["calibration_label"]) 

138 datasetType = DatasetType(datasetTypeName, 

139 dimensionArray, 

140 storageClass, 

141 universe=self.butler.registry.dimensions) 

142 self.butler.registry.registerDatasetType(datasetType) 

143 

144 def addCalibrationLabel(self, name=None, instrument=None, 

145 beginDate="1970-01-01", endDate="2038-12-31"): 

146 

147 """Method to allow tasks to add calibration_label for master calibrations. 

148 

149 Parameters 

150 ---------- 

151 name : `str`, optional 

152 A unique string for the calibration_label key. 

153 instrument : `str`, optional 

154 Instrument this calibration is for. 

155 beginDate : `str`, optional 

156 An ISO 8601 date string for the beginning of the valid date range. 

157 endDate : `str`, optional 

158 An ISO 8601 date string for the end of the valid date range. 

159 

160 Raises 

161 ------ 

162 RuntimeError : 

163 Raised if the instrument or calibration_label name are not set. 

164 """ 

165 if name is None: 

166 name = self.calibrationLabel 

167 if instrument is None: 

168 instrument = self.instrument 

169 if name is None and instrument is None: 

170 raise RuntimeError("Instrument and calibration_label name not set.") 

171 

172 try: 

173 existingValues = self.registry.queryDataIds(['calibration_label'], 

174 instrument=self.instrument, 

175 calibration_label=name) 

176 existingValues = [a for a in existingValues] 

177 print(f"Found {len(existingValues)} Entries for {self.calibrationLabel}") 

178 except LookupError: 

179 self.butler.registry.insertDimensionData( 

180 "calibration_label", 

181 { 

182 "name": name, 

183 "instrument": instrument, 

184 "datetime_begin": Time(datetime.datetime.fromisoformat(beginDate), scale='utc'), 

185 "datetime_end": Time(datetime.datetime.fromisoformat(endDate), scale='utc'), 

186 } 

187 )