Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("LsstCam", "LsstImSim", "LsstPhoSim", "LsstTS8", 

23 "Latiss", "LsstTS3", "LsstUCDCam", "LsstComCam") 

24 

25import os.path 

26 

27import astropy.time 

28 

29import lsst.obs.base.yamlCamera as yamlCamera 

30from lsst.utils import getPackageDir 

31from lsst.obs.base.instrument import Instrument, addUnboundedCalibrationLabel 

32from lsst.daf.butler import DatasetType, DataCoordinate, TIMESPAN_MAX 

33from lsst.pipe.tasks.read_curated_calibs import read_all 

34from .filters import LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS 

35 

36from .translators import LatissTranslator, LsstCamTranslator, \ 

37 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \ 

38 LsstPhoSimTranslator, LsstTS8Translator, LsstImSimTranslator 

39 

40PACKAGE_DIR = getPackageDir("obs_lsst") 

41 

42 

43class LsstCam(Instrument): 

44 """Gen3 Butler specialization for the LSST Main Camera. 

45 

46 Parameters 

47 ---------- 

48 camera : `lsst.cameraGeom.Camera` 

49 Camera object from which to extract detector information. 

50 filters : `list` of `FilterDefinition` 

51 An ordered list of filters to define the set of PhysicalFilters 

52 associated with this instrument in the registry. 

53 

54 While both the camera geometry and the set of filters associated with a 

55 camera are expected to change with time in general, their Butler Registry 

56 representations defined by an Instrument do not. Instead: 

57 

58 - We only extract names, IDs, and purposes from the detectors in the 

59 camera, which should be static information that actually reflects 

60 detector "slots" rather than the physical sensors themselves. Because 

61 the distinction between physical sensors and slots is unimportant in 

62 the vast majority of Butler use cases, we just use "detector" even 

63 though the concept really maps better to "detector slot". Ideally in 

64 the future this distinction between static and time-dependent 

65 information would be encoded in cameraGeom itself (e.g. by making the 

66 time-dependent Detector class inherit from a related class that only 

67 carries static content). 

68 

69 - The Butler Registry is expected to contain physical_filter entries for 

70 all filters an instrument has ever had, because we really only care 

71 about which filters were used for particular observations, not which 

72 filters were *available* at some point in the past. And changes in 

73 individual filters over time will be captured as changes in their 

74 TransmissionCurve datasets, not changes in the registry content (which 

75 is really just a label). While at present Instrument and Registry 

76 do not provide a way to add new physical_filters, they will in the 

77 future. 

78 """ 

79 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

80 instrument = "LSSTCam" 

81 policyName = "lsstCam" 

82 _camera = None 

83 _cameraCachedClass = None 

84 translatorClass = LsstCamTranslator 

85 obsDataPackageDir = getPackageDir("obs_lsst_data") 

86 

87 @property 

88 def configPaths(self): 

89 return [os.path.join(PACKAGE_DIR, "config"), 

90 os.path.join(PACKAGE_DIR, "config", self.policyName)] 

91 

92 @classmethod 

93 def getName(cls): 

94 # Docstring inherited from Instrument.getName 

95 return cls.instrument 

96 

97 @classmethod 

98 def getCamera(cls): 

99 # Constructing a YAML camera takes a long time so cache the result 

100 # We have to be careful to ensure we cache at the subclass level 

101 # since LsstCam base class will look like a cache to the subclasses 

102 if cls._camera is None or cls._cameraCachedClass != cls: 

103 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") 

104 cls._camera = yamlCamera.makeCamera(cameraYamlFile) 

105 cls._cameraCachedClass = cls 

106 return cls._camera 

107 

108 def getRawFormatter(self, dataId): 

109 # Docstring inherited from Instrument.getRawFormatter 

110 # local import to prevent circular dependency 

111 from .rawFormatter import LsstCamRawFormatter 

112 return LsstCamRawFormatter 

113 

114 def register(self, registry): 

115 # Docstring inherited from Instrument.register 

116 # The maximum values below make Gen3's ObservationDataIdPacker produce 

117 # outputs that match Gen2's ccdExposureId. 

118 obsMax = self.translatorClass.max_detector_exposure_id() 

119 registry.insertDimensionData("instrument", 

120 {"name": self.getName(), 

121 "detector_max": self.translatorClass.DETECTOR_MAX, 

122 "visit_max": obsMax, 

123 "exposure_max": obsMax}) 

124 

125 records = [self.extractDetectorRecord(detector) for detector in self.getCamera()] 

126 registry.insertDimensionData("detector", *records) 

127 

128 self._registerFilters(registry) 

129 

130 def extractDetectorRecord(self, camGeomDetector): 

131 """Create a Gen3 Detector entry dict from a cameraGeom.Detector. 

132 """ 

133 # All of the LSST instruments have detector names like R??_S??; we'll 

134 # split them up here, and instruments with only one raft can override 

135 # to change the group to something else if desired. 

136 # Long-term, we should get these fields into cameraGeom separately 

137 # so there's no need to specialize at this stage. 

138 # They are separate in ObservationInfo 

139 group, name = camGeomDetector.getName().split("_") 

140 

141 # getType() returns a pybind11-wrapped enum, which unfortunately 

142 # has no way to extract the name of just the value (it's always 

143 # prefixed by the enum type name). 

144 purpose = str(camGeomDetector.getType()).split(".")[-1] 

145 

146 return dict( 

147 instrument=self.getName(), 

148 id=camGeomDetector.getId(), 

149 full_name=camGeomDetector.getName(), 

150 name_in_raft=name, 

151 purpose=purpose, 

152 raft=group, 

153 ) 

154 

155 def writeCuratedCalibrations(self, butler): 

156 """Write human-curated calibration Datasets to the given Butler with 

157 the appropriate validity ranges. 

158 

159 This is a temporary API that should go away once obs_ packages have 

160 a standardized approach to this problem. 

161 """ 

162 

163 # Write cameraGeom.Camera, with an infinite validity range. 

164 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

165 universe=butler.registry.dimensions) 

166 butler.registry.registerDatasetType(datasetType) 

167 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

168 camera = self.getCamera() 

169 butler.put(camera, datasetType, unboundedDataId) 

170 

171 # Write calibrations from obs_lsst_data 

172 

173 curatedCalibrations = { 

174 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

175 "storageClass": "Defects"}, 

176 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

177 "storageClass": "QECurve"}, 

178 } 

179 

180 for typeName, definition in curatedCalibrations.items(): 

181 # We need to define the dataset types. 

182 datasetType = DatasetType(typeName, definition["dimensions"], 

183 definition["storageClass"], 

184 universe=butler.registry.dimensions) 

185 butler.registry.registerDatasetType(datasetType) 

186 self._writeCuratedCalibrationDataset(butler, datasetType) 

187 

188 def _writeCuratedCalibrationDataset(self, butler, datasetType): 

189 """Write a standardized curated calibration dataset from an obs data 

190 package. 

191 

192 Parameters 

193 ---------- 

194 butler : `lsst.daf.butler.Butler` 

195 Gen3 butler in which to put the calibrations. 

196 datasetType : `lsst.daf.butler.DatasetType` 

197 Dataset type to be put. 

198 

199 Notes 

200 ----- 

201 This method scans the location defined in the ``obsDataPackageDir`` 

202 class attribute for curated calibrations corresponding to the 

203 supplied dataset type. The directory name in the data package much 

204 match the name of the dataset type. They are assumed to use the 

205 standard layout and can be read by 

206 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

207 metadata. 

208 """ 

209 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

210 datasetType.name) 

211 

212 if not os.path.exists(calibPath): 

213 return 

214 

215 camera = self.getCamera() 

216 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

217 endOfTime = TIMESPAN_MAX 

218 dimensionRecords = [] 

219 datasetRecords = [] 

220 for det in calibsDict: 

221 times = sorted([k for k in calibsDict[det]]) 

222 calibs = [calibsDict[det][time] for time in times] 

223 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

224 times += [endOfTime] 

225 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

226 md = calib.getMetadata() 

227 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

228 dataId = DataCoordinate.standardize( 

229 universe=butler.registry.dimensions, 

230 instrument=self.getName(), 

231 calibration_label=calibrationLabel, 

232 detector=md["DETECTOR"], 

233 ) 

234 datasetRecords.append((calib, dataId)) 

235 dimensionRecords.append({ 

236 "instrument": self.getName(), 

237 "name": calibrationLabel, 

238 "datetime_begin": beginTime, 

239 "datetime_end": endTime, 

240 }) 

241 

242 # Second loop actually does the inserts and filesystem writes. 

243 with butler.transaction(): 

244 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

245 # TODO: vectorize these puts, once butler APIs for that become 

246 # available. 

247 for calib, dataId in datasetRecords: 

248 butler.put(calib, datasetType, dataId) 

249 

250 

251class LsstComCam(LsstCam): 

252 """Gen3 Butler specialization for ComCam data. 

253 """ 

254 

255 instrument = "LSSTComCam" 

256 policyName = "comCam" 

257 translatorClass = LsstComCamTranslator 

258 

259 def getRawFormatter(self, dataId): 

260 # local import to prevent circular dependency 

261 from .rawFormatter import LsstComCamRawFormatter 

262 return LsstComCamRawFormatter 

263 

264 

265class LsstImSim(LsstCam): 

266 """Gen3 Butler specialization for ImSim simulations. 

267 """ 

268 

269 instrument = "LSST-ImSim" 

270 policyName = "imsim" 

271 translatorClass = LsstImSimTranslator 

272 

273 def getRawFormatter(self, dataId): 

274 # local import to prevent circular dependency 

275 from .rawFormatter import LsstImSimRawFormatter 

276 return LsstImSimRawFormatter 

277 

278 

279class LsstPhoSim(LsstCam): 

280 """Gen3 Butler specialization for Phosim simulations. 

281 """ 

282 

283 instrument = "LSST-PhoSim" 

284 policyName = "phosim" 

285 translatorClass = LsstPhoSimTranslator 

286 

287 def getRawFormatter(self, dataId): 

288 # local import to prevent circular dependency 

289 from .rawFormatter import LsstPhoSimRawFormatter 

290 return LsstPhoSimRawFormatter 

291 

292 

293class LsstTS8(LsstCam): 

294 """Gen3 Butler specialization for raft test stand data. 

295 """ 

296 

297 instrument = "LSST-TS8" 

298 policyName = "ts8" 

299 translatorClass = LsstTS8Translator 

300 

301 def getRawFormatter(self, dataId): 

302 # local import to prevent circular dependency 

303 from .rawFormatter import LsstTS8RawFormatter 

304 return LsstTS8RawFormatter 

305 

306 

307class LsstUCDCam(LsstCam): 

308 """Gen3 Butler specialization for UCDCam test stand data. 

309 """ 

310 

311 instrument = "LSST-UCDCam" 

312 policyName = "ucd" 

313 translatorClass = LsstUCDCamTranslator 

314 

315 def getRawFormatter(self, dataId): 

316 # local import to prevent circular dependency 

317 from .rawFormatter import LsstUCDCamRawFormatter 

318 return LsstUCDCamRawFormatter 

319 

320 

321class LsstTS3(LsstCam): 

322 """Gen3 Butler specialization for TS3 test stand data. 

323 """ 

324 

325 instrument = "LSST-TS3" 

326 policyName = "ts3" 

327 translatorClass = LsstTS3Translator 

328 

329 def getRawFormatter(self, dataId): 

330 # local import to prevent circular dependency 

331 from .rawFormatter import LsstTS3RawFormatter 

332 return LsstTS3RawFormatter 

333 

334 

335class Latiss(LsstCam): 

336 """Gen3 Butler specialization for AuxTel LATISS data. 

337 """ 

338 filterDefinitions = LATISS_FILTER_DEFINITIONS 

339 instrument = "LATISS" 

340 policyName = "latiss" 

341 translatorClass = LatissTranslator 

342 

343 def extractDetectorRecord(self, camGeomDetector): 

344 # Override to remove group (raft) name, because LATISS only has one 

345 # detector. 

346 record = super().extractDetectorRecord(camGeomDetector) 

347 record["raft"] = None 

348 record["name_in_raft"] = record["full_name"] 

349 return record 

350 

351 def getRawFormatter(self, dataId): 

352 # local import to prevent circular dependency 

353 from .rawFormatter import LatissRawFormatter 

354 return LatissRawFormatter