Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("LsstCamInstrument", "ImsimInstrument", "PhosimInstrument", "Ts8Instrument", 

23 "LatissInstrument", "Ts3Instrument", "UcdCamInstrument", "LsstComCamInstrument") 

24 

25import os.path 

26from dateutil import parser 

27 

28import lsst.obs.base.yamlCamera as yamlCamera 

29from lsst.utils import getPackageDir 

30from lsst.obs.base.instrument import Instrument, addUnboundedCalibrationLabel 

31from lsst.daf.butler import DatasetType, DataCoordinate 

32from lsst.pipe.tasks.read_curated_calibs import read_all 

33from ..filters import LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS 

34 

35from ..translators import LsstLatissTranslator, LsstCamTranslator, \ 

36 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \ 

37 PhosimTranslator, LsstTS8Translator, ImsimTranslator 

38 

39PACKAGE_DIR = getPackageDir("obs_lsst") 

40 

41 

42class LsstCamInstrument(Instrument): 

43 """Gen3 Butler specialization for the LSST Main Camera. 

44 

45 Parameters 

46 ---------- 

47 camera : `lsst.cameraGeom.Camera` 

48 Camera object from which to extract detector information. 

49 filters : `list` of `FilterDefinition` 

50 An ordered list of filters to define the set of PhysicalFilters 

51 associated with this instrument in the registry. 

52 

53 While both the camera geometry and the set of filters associated with a 

54 camera are expected to change with time in general, their Butler Registry 

55 representations defined by an Instrument do not. Instead: 

56 

57 - We only extract names, IDs, and purposes from the detectors in the 

58 camera, which should be static information that actually reflects 

59 detector "slots" rather than the physical sensors themselves. Because 

60 the distinction between physical sensors and slots is unimportant in 

61 the vast majority of Butler use cases, we just use "detector" even 

62 though the concept really maps better to "detector slot". Ideally in 

63 the future this distinction between static and time-dependent 

64 information would be encoded in cameraGeom itself (e.g. by making the 

65 time-dependent Detector class inherit from a related class that only 

66 carries static content). 

67 

68 - The Butler Registry is expected to contain physical_filter entries for 

69 all filters an instrument has ever had, because we really only care 

70 about which filters were used for particular observations, not which 

71 filters were *available* at some point in the past. And changes in 

72 individual filters over time will be captured as changes in their 

73 TransmissionCurve datasets, not changes in the registry content (which 

74 is really just a label). While at present Instrument and Registry 

75 do not provide a way to add new physical_filters, they will in the 

76 future. 

77 """ 

78 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

79 instrument = "lsstCam" 

80 policyName = "lsstCam" 

81 _camera = None 

82 _cameraCachedClass = None 

83 translatorClass = LsstCamTranslator 

84 obsDataPackageDir = getPackageDir("obs_lsst_data") 

85 

86 @property 

87 def configPaths(self): 

88 return [os.path.join(PACKAGE_DIR, "config"), 

89 os.path.join(PACKAGE_DIR, "config", self.policyName)] 

90 

91 @classmethod 

92 def getName(cls): 

93 # Docstring inherited from Instrument.getName 

94 return cls.instrument 

95 

96 @classmethod 

97 def getCamera(cls): 

98 # Constructing a YAML camera takes a long time so cache the result 

99 # We have to be careful to ensure we cache at the subclass level 

100 # since LsstCam base class will look like a cache to the subclasses 

101 if cls._camera is None or cls._cameraCachedClass != cls: 

102 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") 

103 cls._camera = yamlCamera.makeCamera(cameraYamlFile) 

104 cls._cameraCachedClass = cls 

105 return cls._camera 

106 

107 def getRawFormatter(self, dataId): 

108 # Docstring inherited from Instrument.getRawFormatter 

109 # local import to prevent circular dependency 

110 from .rawFormatter import LsstCamRawFormatter 

111 return LsstCamRawFormatter 

112 

113 def register(self, registry): 

114 # Docstring inherited from Instrument.register 

115 # The maximum values below make Gen3's ObservationDataIdPacker produce 

116 # outputs that match Gen2's ccdExposureId. 

117 obsMax = self.translatorClass.max_detector_exposure_id() 

118 registry.insertDimensionData("instrument", 

119 {"name": self.getName(), 

120 "detector_max": self.translatorClass.DETECTOR_MAX, 

121 "visit_max": obsMax, 

122 "exposure_max": obsMax}) 

123 

124 records = [self.extractDetectorRecord(detector) for detector in self.getCamera()] 

125 registry.insertDimensionData("detector", *records) 

126 

127 self._registerFilters(registry) 

128 

129 def extractDetectorRecord(self, camGeomDetector): 

130 """Create a Gen3 Detector entry dict from a cameraGeom.Detector. 

131 """ 

132 # All of the LSST instruments have detector names like R??_S??; we'll 

133 # split them up here, and instruments with only one raft can override 

134 # to change the group to something else if desired. 

135 # Long-term, we should get these fields into cameraGeom separately 

136 # so there's no need to specialize at this stage. 

137 # They are separate in ObservationInfo 

138 group, name = camGeomDetector.getName().split("_") 

139 

140 # getType() returns a pybind11-wrapped enum, which unfortunately 

141 # has no way to extract the name of just the value (it's always 

142 # prefixed by the enum type name). 

143 purpose = str(camGeomDetector.getType()).split(".")[-1] 

144 

145 return dict( 

146 instrument=self.getName(), 

147 id=camGeomDetector.getId(), 

148 full_name=camGeomDetector.getName(), 

149 name_in_raft=name, 

150 purpose=purpose, 

151 raft=group, 

152 ) 

153 

154 def writeCuratedCalibrations(self, butler): 

155 """Write human-curated calibration Datasets to the given Butler with 

156 the appropriate validity ranges. 

157 

158 This is a temporary API that should go away once obs_ packages have 

159 a standardized approach to this problem. 

160 """ 

161 

162 # Write cameraGeom.Camera, with an infinite validity range. 

163 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

164 universe=butler.registry.dimensions) 

165 butler.registry.registerDatasetType(datasetType) 

166 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

167 camera = self.getCamera() 

168 butler.put(camera, datasetType, unboundedDataId) 

169 

170 # Write calibrations from obs_lsst_data 

171 

172 curatedCalibrations = { 

173 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

174 "storageClass": "Defects"}, 

175 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

176 "storageClass": "QECurve"}, 

177 } 

178 

179 for typeName, definition in curatedCalibrations.items(): 

180 # We need to define the dataset types. 

181 datasetType = DatasetType(typeName, definition["dimensions"], 

182 definition["storageClass"], 

183 universe=butler.registry.dimensions) 

184 butler.registry.registerDatasetType(datasetType) 

185 self._writeCuratedCalibrationDataset(butler, datasetType) 

186 

187 def _writeCuratedCalibrationDataset(self, butler, datasetType): 

188 """Write a standardized curated calibration dataset from an obs data 

189 package. 

190 

191 Parameters 

192 ---------- 

193 butler : `lsst.daf.butler.Butler` 

194 Gen3 butler in which to put the calibrations. 

195 datasetType : `lsst.daf.butler.DatasetType` 

196 Dataset type to be put. 

197 

198 Notes 

199 ----- 

200 This method scans the location defined in the ``obsDataPackageDir`` 

201 class attribute for curated calibrations corresponding to the 

202 supplied dataset type. The directory name in the data package much 

203 match the name of the dataset type. They are assumed to use the 

204 standard layout and can be read by 

205 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

206 metadata. 

207 """ 

208 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

209 datasetType.name) 

210 

211 if not os.path.exists(calibPath): 

212 return 

213 

214 camera = self.getCamera() 

215 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

216 endOfTime = '20380119T031407' 

217 dimensionRecords = [] 

218 datasetRecords = [] 

219 for det in calibsDict: 

220 times = sorted([k for k in calibsDict[det]]) 

221 calibs = [calibsDict[det][time] for time in times] 

222 times = times + [parser.parse(endOfTime), ] 

223 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

224 md = calib.getMetadata() 

225 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

226 dataId = DataCoordinate.standardize( 

227 universe=butler.registry.dimensions, 

228 instrument=self.getName(), 

229 calibration_label=calibrationLabel, 

230 detector=md["DETECTOR"], 

231 ) 

232 datasetRecords.append((calib, dataId)) 

233 dimensionRecords.append({ 

234 "instrument": self.getName(), 

235 "name": calibrationLabel, 

236 "datetime_begin": beginTime, 

237 "datetime_end": endTime, 

238 }) 

239 

240 # Second loop actually does the inserts and filesystem writes. 

241 with butler.transaction(): 

242 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

243 # TODO: vectorize these puts, once butler APIs for that become 

244 # available. 

245 for calib, dataId in datasetRecords: 

246 butler.put(calib, datasetType, dataId) 

247 

248 

249class LsstComCamInstrument(LsstCamInstrument): 

250 """Gen3 Butler specialization for ComCam data. 

251 """ 

252 

253 instrument = "LSST-ComCam" 

254 policyName = "comCam" 

255 translatorClass = LsstComCamTranslator 

256 

257 def getRawFormatter(self, dataId): 

258 # local import to prevent circular dependency 

259 from .rawFormatter import LsstComCamRawFormatter 

260 return LsstComCamRawFormatter 

261 

262 

263class ImsimInstrument(LsstCamInstrument): 

264 """Gen3 Butler specialization for ImSim simulations. 

265 """ 

266 

267 instrument = "LSST-ImSim" 

268 policyName = "imsim" 

269 translatorClass = ImsimTranslator 

270 

271 def getRawFormatter(self, dataId): 

272 # local import to prevent circular dependency 

273 from .rawFormatter import ImsimRawFormatter 

274 return ImsimRawFormatter 

275 

276 

277class PhosimInstrument(LsstCamInstrument): 

278 """Gen3 Butler specialization for Phosim simulations. 

279 """ 

280 

281 instrument = "PhoSim" 

282 policyName = "phosim" 

283 translatorClass = PhosimTranslator 

284 

285 def getRawFormatter(self, dataId): 

286 # local import to prevent circular dependency 

287 from .rawFormatter import PhosimRawFormatter 

288 return PhosimRawFormatter 

289 

290 

291class Ts8Instrument(LsstCamInstrument): 

292 """Gen3 Butler specialization for raft test stand data. 

293 """ 

294 

295 instrument = "LSST-TS8" 

296 policyName = "ts8" 

297 translatorClass = LsstTS8Translator 

298 

299 def getRawFormatter(self, dataId): 

300 # local import to prevent circular dependency 

301 from .rawFormatter import Ts8RawFormatter 

302 return Ts8RawFormatter 

303 

304 

305class UcdCamInstrument(LsstCamInstrument): 

306 """Gen3 Butler specialization for UCDCam test stand data. 

307 """ 

308 

309 instrument = "LSST-UCDCam" 

310 policyName = "ucd" 

311 translatorClass = LsstUCDCamTranslator 

312 

313 def getRawFormatter(self, dataId): 

314 # local import to prevent circular dependency 

315 from .rawFormatter import UcdCamRawFormatter 

316 return UcdCamRawFormatter 

317 

318 

319class Ts3Instrument(LsstCamInstrument): 

320 """Gen3 Butler specialization for TS3 test stand data. 

321 """ 

322 

323 instrument = "LSST-TS3" 

324 policyName = "ts3" 

325 translatorClass = LsstTS3Translator 

326 

327 def getRawFormatter(self, dataId): 

328 # local import to prevent circular dependency 

329 from .rawFormatter import Ts3RawFormatter 

330 return Ts3RawFormatter 

331 

332 

333class LatissInstrument(LsstCamInstrument): 

334 """Gen3 Butler specialization for AuxTel LATISS data. 

335 """ 

336 filterDefinitions = LATISS_FILTER_DEFINITIONS 

337 instrument = "LATISS" 

338 policyName = "latiss" 

339 translatorClass = LsstLatissTranslator 

340 

341 def extractDetectorRecord(self, camGeomDetector): 

342 # Override to remove group (raft) name, because LATISS only has one 

343 # detector. 

344 record = super().extractDetectorRecord(camGeomDetector) 

345 record["raft"] = None 

346 record["name_in_raft"] = record["full_name"] 

347 return record 

348 

349 def getRawFormatter(self, dataId): 

350 # local import to prevent circular dependency 

351 from .rawFormatter import LatissRawFormatter 

352 return LatissRawFormatter