Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("LsstCamInstrument", "ImsimInstrument", "PhosimInstrument", "Ts8Instrument", 

23 "LatissInstrument", "Ts3Instrument", "UcdCamInstrument", "LsstComCamInstrument") 

24 

25import os.path 

26from dateutil import parser 

27 

28import lsst.obs.base.yamlCamera as yamlCamera 

29from lsst.utils import getPackageDir 

30from lsst.obs.base.instrument import Instrument, addUnboundedCalibrationLabel 

31from lsst.daf.butler import DatasetType 

32from lsst.pipe.tasks.read_curated_calibs import read_all 

33from ..filters import LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS 

34 

35from ..translators import LsstLatissTranslator, LsstCamTranslator, \ 

36 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \ 

37 PhosimTranslator, LsstTS8Translator, ImsimTranslator 

38 

39PACKAGE_DIR = getPackageDir("obs_lsst") 

40 

41 

42class LsstCamInstrument(Instrument): 

43 """Gen3 Butler specialization for the LSST Main Camera. 

44 

45 Parameters 

46 ---------- 

47 camera : `lsst.cameraGeom.Camera` 

48 Camera object from which to extract detector information. 

49 filters : `list` of `FilterDefinition` 

50 An ordered list of filters to define the set of PhysicalFilters 

51 associated with this instrument in the registry. 

52 

53 While both the camera geometry and the set of filters associated with a 

54 camera are expected to change with time in general, their Butler Registry 

55 representations defined by an Instrument do not. Instead: 

56 

57 - We only extract names, IDs, and purposes from the detectors in the 

58 camera, which should be static information that actually reflects 

59 detector "slots" rather than the physical sensors themselves. Because 

60 the distinction between physical sensors and slots is unimportant in 

61 the vast majority of Butler use cases, we just use "detector" even 

62 though the concept really maps better to "detector slot". Ideally in 

63 the future this distinction between static and time-dependent 

64 information would be encoded in cameraGeom itself (e.g. by making the 

65 time-dependent Detector class inherit from a related class that only 

66 carries static content). 

67 

68 - The Butler Registry is expected to contain physical_filter entries for 

69 all filters an instrument has ever had, because we really only care 

70 about which filters were used for particular observations, not which 

71 filters were *available* at some point in the past. And changes in 

72 individual filters over time will be captured as changes in their 

73 TransmissionCurve datasets, not changes in the registry content (which 

74 is really just a label). While at present Instrument and Registry 

75 do not provide a way to add new physical_filters, they will in the 

76 future. 

77 """ 

78 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

79 instrument = "lsstCam" 

80 policyName = "lsstCam" 

81 _camera = None 

82 _cameraCachedClass = None 

83 translatorClass = LsstCamTranslator 

84 

85 @property 

86 def configPaths(self): 

87 return [os.path.join(PACKAGE_DIR, "config"), 

88 os.path.join(PACKAGE_DIR, "config", self.policyName)] 

89 

90 @classmethod 

91 def getName(cls): 

92 # Docstring inherited from Instrument.getName 

93 return cls.instrument 

94 

95 @classmethod 

96 def getCamera(cls): 

97 # Constructing a YAML camera takes a long time so cache the result 

98 # We have to be careful to ensure we cache at the subclass level 

99 # since LsstCam base class will look like a cache to the subclasses 

100 if cls._camera is None or cls._cameraCachedClass != cls: 

101 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") 

102 cls._camera = yamlCamera.makeCamera(cameraYamlFile) 

103 cls._cameraCachedClass = cls 

104 return cls._camera 

105 

106 def getRawFormatter(self, dataId): 

107 # Docstring inherited from Instrument.getRawFormatter 

108 # local import to prevent circular dependency 

109 from .rawFormatter import LsstCamRawFormatter 

110 return LsstCamRawFormatter 

111 

112 def register(self, registry): 

113 # Docstring inherited from Instrument.register 

114 # The maximum values below make Gen3's ObservationDataIdPacker produce 

115 # outputs that match Gen2's ccdExposureId. 

116 obsMax = self.translatorClass.max_detector_exposure_id() 

117 registry.insertDimensionData("instrument", 

118 {"name": self.getName(), 

119 "detector_max": self.translatorClass.DETECTOR_MAX, 

120 "visit_max": obsMax, 

121 "exposure_max": obsMax}) 

122 

123 records = [self.extractDetectorRecord(detector) for detector in self.getCamera()] 

124 registry.insertDimensionData("detector", *records) 

125 

126 self._registerFilters(registry) 

127 

128 def extractDetectorRecord(self, camGeomDetector): 

129 """Create a Gen3 Detector entry dict from a cameraGeom.Detector. 

130 """ 

131 # All of the LSST instruments have detector names like R??_S??; we'll 

132 # split them up here, and instruments with only one raft can override 

133 # to change the group to something else if desired. 

134 # Long-term, we should get these fields into cameraGeom separately 

135 # so there's no need to specialize at this stage. 

136 # They are separate in ObservationInfo 

137 group, name = camGeomDetector.getName().split("_") 

138 

139 # getType() returns a pybind11-wrapped enum, which unfortunately 

140 # has no way to extract the name of just the value (it's always 

141 # prefixed by the enum type name). 

142 purpose = str(camGeomDetector.getType()).split(".")[-1] 

143 

144 return dict( 

145 instrument=self.getName(), 

146 id=camGeomDetector.getId(), 

147 full_name=camGeomDetector.getName(), 

148 name_in_raft=name, 

149 purpose=purpose, 

150 raft=group, 

151 ) 

152 

153 def writeCuratedCalibrations(self, butler): 

154 """Write human-curated calibration Datasets to the given Butler with 

155 the appropriate validity ranges. 

156 

157 This is a temporary API that should go away once obs_ packages have 

158 a standardized approach to this problem. 

159 """ 

160 

161 # Write cameraGeom.Camera, with an infinite validity range. 

162 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

163 universe=butler.registry.dimensions) 

164 butler.registry.registerDatasetType(datasetType) 

165 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

166 camera = self.getCamera() 

167 butler.put(camera, datasetType, unboundedDataId) 

168 

169 # Write defects with validity ranges taken from 

170 # obs_lsst_data/{name}/defects (along with the defects themselves). 

171 datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "Defects", 

172 universe=butler.registry.dimensions) 

173 butler.registry.registerDatasetType(datasetType) 

174 defectPath = os.path.join(getPackageDir("obs_lsst"), self.policyName, "defects") 

175 

176 if os.path.exists(defectPath): 

177 camera = self.getCamera() 

178 defectsDict = read_all(defectPath, camera)[0] # second return is calib type 

179 endOfTime = '20380119T031407' 

180 with butler.transaction(): 

181 for det in defectsDict: 

182 detector = camera[det] 

183 times = sorted([k for k in defectsDict[det]]) 

184 defects = [defectsDict[det][time] for time in times] 

185 times = times + [parser.parse(endOfTime), ] 

186 for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]): 

187 md = defect.getMetadata() 

188 calibrationLabelName = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}" 

189 butler.registry.insertDimensionData( 

190 "calibration_label", 

191 { 

192 "instrument": self.getName(), 

193 "name": calibrationLabelName, 

194 "datetime_begin": beginTime, 

195 "datetime_end": endTime, 

196 } 

197 ) 

198 butler.put(defect, datasetType, instrument=self.getName(), 

199 calibration_label=calibrationLabelName, detector=detector.getId()) 

200 

201 

202class LsstComCamInstrument(LsstCamInstrument): 

203 """Gen3 Butler specialization for ComCam data. 

204 """ 

205 

206 instrument = "LSST-ComCam" 

207 policyName = "comCam" 

208 translatorClass = LsstComCamTranslator 

209 

210 def getRawFormatter(self, dataId): 

211 # local import to prevent circular dependency 

212 from .rawFormatter import LsstComCamRawFormatter 

213 return LsstComCamRawFormatter 

214 

215 

216class ImsimInstrument(LsstCamInstrument): 

217 """Gen3 Butler specialization for ImSim simulations. 

218 """ 

219 

220 instrument = "LSST-ImSim" 

221 policyName = "imsim" 

222 translatorClass = ImsimTranslator 

223 

224 def getRawFormatter(self, dataId): 

225 # local import to prevent circular dependency 

226 from .rawFormatter import ImsimRawFormatter 

227 return ImsimRawFormatter 

228 

229 

230class PhosimInstrument(LsstCamInstrument): 

231 """Gen3 Butler specialization for Phosim simulations. 

232 """ 

233 

234 instrument = "LSST-PhoSim" 

235 policyName = "phosim" 

236 translatorClass = PhosimTranslator 

237 

238 def getRawFormatter(self, dataId): 

239 # local import to prevent circular dependency 

240 from .rawFormatter import PhosimRawFormatter 

241 return PhosimRawFormatter 

242 

243 

244class Ts8Instrument(LsstCamInstrument): 

245 """Gen3 Butler specialization for raft test stand data. 

246 """ 

247 

248 instrument = "LSST-TS8" 

249 policyName = "ts8" 

250 translatorClass = LsstTS8Translator 

251 

252 def getRawFormatter(self, dataId): 

253 # local import to prevent circular dependency 

254 from .rawFormatter import Ts8RawFormatter 

255 return Ts8RawFormatter 

256 

257 

258class UcdCamInstrument(LsstCamInstrument): 

259 """Gen3 Butler specialization for UCDCam test stand data. 

260 """ 

261 

262 instrument = "UCDCam" 

263 policyName = "ucd" 

264 translatorClass = LsstUCDCamTranslator 

265 

266 def getRawFormatter(self, dataId): 

267 # local import to prevent circular dependency 

268 from .rawFormatter import UcdCamRawFormatter 

269 return UcdCamRawFormatter 

270 

271 

272class Ts3Instrument(LsstCamInstrument): 

273 """Gen3 Butler specialization for TS3 test stand data. 

274 """ 

275 

276 instrument = "LSST-TS3" 

277 policyName = "ts3" 

278 translatorClass = LsstTS3Translator 

279 

280 def getRawFormatter(self, dataId): 

281 # local import to prevent circular dependency 

282 from .rawFormatter import Ts3RawFormatter 

283 return Ts3RawFormatter 

284 

285 

286class LatissInstrument(LsstCamInstrument): 

287 """Gen3 Butler specialization for AuxTel LATISS data. 

288 """ 

289 filterDefinitions = LATISS_FILTER_DEFINITIONS 

290 instrument = "LATISS" 

291 policyName = "latiss" 

292 translatorClass = LsstLatissTranslator 

293 

294 def extractDetectorRecord(self, camGeomDetector): 

295 # Override to remove group (raft) name, because LATISS only has one 

296 # detector. 

297 record = super().extractDetectorRecord(camGeomDetector) 

298 record["raft"] = None 

299 record["name_in_raft"] = record["full_name"] 

300 return record 

301 

302 def getRawFormatter(self, dataId): 

303 # local import to prevent circular dependency 

304 from .rawFormatter import LatissRawFormatter 

305 return LatissRawFormatter