Coverage for python/lsst/obs/lsst/_instrument.py: 56%

158 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-28 04:13 -0700

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("LsstCam", "LsstCamImSim", "LsstCamPhoSim", "LsstTS8", 

23 "Latiss", "LsstTS3", "LsstUCDCam", "LsstComCam", "LsstComCamSim") 

24 

25import datetime 

26import hashlib 

27import os.path 

28 

29import lsst.obs.base.yamlCamera as yamlCamera 

30from lsst.utils.introspection import get_full_type_name 

31from lsst.utils import getPackageDir 

32from lsst.obs.base import Instrument, VisitSystem 

33from .filters import (LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS, 

34 LSSTCAM_IMSIM_FILTER_DEFINITIONS, TS3_FILTER_DEFINITIONS, 

35 TS8_FILTER_DEFINITIONS, COMCAM_FILTER_DEFINITIONS, 

36 GENERIC_FILTER_DEFINITIONS, UCD_FILTER_DEFINITIONS, 

37 ) 

38 

39from .translators import LatissTranslator, LsstCamTranslator, \ 

40 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \ 

41 LsstCamPhoSimTranslator, LsstTS8Translator, LsstCamImSimTranslator, \ 

42 LsstComCamSimTranslator 

43 

44from .translators.lsst import GROUP_RE, TZERO_DATETIME 

45 

46PACKAGE_DIR = getPackageDir("obs_lsst") 

47 

48 

49class LsstCam(Instrument): 

50 """Gen3 Butler specialization for the LSST Main Camera. 

51 

52 Parameters 

53 ---------- 

54 camera : `lsst.cameraGeom.Camera` 

55 Camera object from which to extract detector information. 

56 filters : `list` of `FilterDefinition` 

57 An ordered list of filters to define the set of PhysicalFilters 

58 associated with this instrument in the registry. 

59 

60 While both the camera geometry and the set of filters associated with a 

61 camera are expected to change with time in general, their Butler Registry 

62 representations defined by an Instrument do not. Instead: 

63 

64 - We only extract names, IDs, and purposes from the detectors in the 

65 camera, which should be static information that actually reflects 

66 detector "slots" rather than the physical sensors themselves. Because 

67 the distinction between physical sensors and slots is unimportant in 

68 the vast majority of Butler use cases, we just use "detector" even 

69 though the concept really maps better to "detector slot". Ideally in 

70 the future this distinction between static and time-dependent 

71 information would be encoded in cameraGeom itself (e.g. by making the 

72 time-dependent Detector class inherit from a related class that only 

73 carries static content). 

74 

75 - The Butler Registry is expected to contain physical_filter entries for 

76 all filters an instrument has ever had, because we really only care 

77 about which filters were used for particular observations, not which 

78 filters were *available* at some point in the past. And changes in 

79 individual filters over time will be captured as changes in their 

80 TransmissionCurve datasets, not changes in the registry content (which 

81 is really just a label). While at present Instrument and Registry 

82 do not provide a way to add new physical_filters, they will in the 

83 future. 

84 """ 

85 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

86 instrument = "LSSTCam" 

87 policyName = "lsstCam" 

88 translatorClass = LsstCamTranslator 

89 obsDataPackage = "obs_lsst_data" 

90 visitSystem = VisitSystem.BY_SEQ_START_END 

91 

92 @property 

93 def configPaths(self): 

94 return [os.path.join(PACKAGE_DIR, "config"), 

95 os.path.join(PACKAGE_DIR, "config", self.policyName)] 

96 

97 @classmethod 

98 def getName(cls): 

99 # Docstring inherited from Instrument.getName 

100 return cls.instrument 

101 

102 @classmethod 

103 def getCamera(cls): 

104 # Constructing a YAML camera takes a long time but we rely on 

105 # yamlCamera to cache for us. 

106 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") 

107 camera = yamlCamera.makeCamera(cameraYamlFile) 

108 if camera.getName() != cls.getName(): 

109 raise RuntimeError(f"Expected to read camera geometry for {cls.instrument}" 

110 f" but instead got geometry for {camera.getName()}") 

111 return camera 

112 

113 def _make_default_dimension_packer( 

114 self, 

115 config_attr, 

116 data_id, 

117 is_exposure=None, 

118 default="rubin", 

119 ): 

120 # Docstring inherited from Instrument._make_default_dimension_packer. 

121 # Only difference is the change to default above. 

122 return super()._make_default_dimension_packer( 

123 config_attr, 

124 data_id, 

125 is_exposure=is_exposure, 

126 default=default, 

127 ) 

128 

129 def getRawFormatter(self, dataId): 

130 # Docstring inherited from Instrument.getRawFormatter 

131 # local import to prevent circular dependency 

132 from .rawFormatter import LsstCamRawFormatter 

133 return LsstCamRawFormatter 

134 

135 def register(self, registry, update=False): 

136 # Docstring inherited from Instrument.register 

137 # The maximum values below make Gen3's ObservationDataIdPacker produce 

138 # outputs that match Gen2's ccdExposureId. 

139 obsMax = self.translatorClass.max_exposure_id() 

140 # Make sure this is at least 1 to avoid non-uniqueness issues (e.g. 

141 # for data ids that also get used in indexing). 

142 detectorMax = max(self.translatorClass.DETECTOR_MAX, 1) 

143 with registry.transaction(): 

144 registry.syncDimensionData( 

145 "instrument", 

146 { 

147 "name": self.getName(), 

148 "detector_max": detectorMax, 

149 "visit_max": obsMax, 

150 "exposure_max": obsMax, 

151 "class_name": get_full_type_name(self), 

152 "visit_system": None if self.visitSystem is None else self.visitSystem.value, 

153 }, 

154 update=update 

155 ) 

156 for detector in self.getCamera(): 

157 registry.syncDimensionData("detector", self.extractDetectorRecord(detector), update=update) 

158 

159 self._registerFilters(registry, update=update) 

160 

161 def extractDetectorRecord(self, camGeomDetector): 

162 """Create a Gen3 Detector entry dict from a cameraGeom.Detector. 

163 """ 

164 # All of the LSST instruments have detector names like R??_S??; we'll 

165 # split them up here, and instruments with only one raft can override 

166 # to change the group to something else if desired. 

167 # Long-term, we should get these fields into cameraGeom separately 

168 # so there's no need to specialize at this stage. 

169 # They are separate in ObservationInfo 

170 group, name = camGeomDetector.getName().split("_") 

171 

172 # getType() returns a pybind11-wrapped enum, which unfortunately 

173 # has no way to extract the name of just the value (it's always 

174 # prefixed by the enum type name). 

175 purpose = str(camGeomDetector.getType()).split(".")[-1] 

176 

177 return dict( 

178 instrument=self.getName(), 

179 id=camGeomDetector.getId(), 

180 full_name=camGeomDetector.getName(), 

181 name_in_raft=name, 

182 purpose=purpose, 

183 raft=group, 

184 ) 

185 

186 @classmethod 

187 def group_name_to_group_id(cls, group_name: str) -> int: 

188 """Translate the exposure group name to an integer. 

189 

190 Parameters 

191 ---------- 

192 group_name : `str` 

193 The name of the exposure group. 

194 

195 Returns 

196 ------- 

197 id : `int` 

198 The exposure group name in integer form. This integer might be 

199 used as an ID to uniquely identify the group in contexts where 

200 a string can not be used. 

201 

202 Notes 

203 ----- 

204 If given a group name that can be directly cast to an integer it 

205 returns the integer. If the group name looks like an ISO date the ID 

206 returned is seconds since an arbitrary recent epoch. Otherwise 

207 the group name is hashed and the first 14 digits of the hash is 

208 returned along with the length of the group name. 

209 """ 

210 # If the group is an int we return it 

211 try: 

212 group_id = int(group_name) 

213 return group_id 

214 except ValueError: 

215 pass 

216 

217 # A Group is defined as ISO date with an extension 

218 # The integer must be the same for a given group so we can never 

219 # use datetime_begin. 

220 # Nominally a GROUPID looks like "ISODATE+N" where the +N is 

221 # optional. This can be converted to seconds since epoch with 

222 # N being zero-padded to 4 digits and appended (defaulting to 0). 

223 # For early data lacking that form we hash the group and return 

224 # the int. 

225 matches_date = GROUP_RE.match(group_name) 

226 if matches_date: 

227 iso_str = matches_date.group(1) 

228 fraction = matches_date.group(2) 

229 n = matches_date.group(3) 

230 if n is not None: 

231 n = int(n) 

232 else: 

233 n = 0 

234 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S") 

235 

236 tdelta = iso - TZERO_DATETIME 

237 epoch = int(tdelta.total_seconds()) 

238 

239 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N 

240 group_id = int(f"{epoch}{fraction}{n:04d}") 

241 else: 

242 # Non-standard string so convert to numbers 

243 # using a hash function. Use the first N hex digits 

244 group_bytes = group_name.encode("us-ascii") 

245 hasher = hashlib.blake2b(group_bytes) 

246 # Need to be big enough it does not possibly clash with the 

247 # date-based version above 

248 digest = hasher.hexdigest()[:14] 

249 group_id = int(digest, base=16) 

250 

251 # To help with hash collision, append the string length 

252 group_id = int(f"{group_id}{len(group_name):02d}") 

253 

254 return group_id 

255 

256 

257class LsstComCam(LsstCam): 

258 """Gen3 Butler specialization for ComCam data. 

259 """ 

260 

261 filterDefinitions = COMCAM_FILTER_DEFINITIONS 

262 instrument = "LSSTComCam" 

263 policyName = "comCam" 

264 translatorClass = LsstComCamTranslator 

265 

266 def getRawFormatter(self, dataId): 

267 # local import to prevent circular dependency 

268 from .rawFormatter import LsstComCamRawFormatter 

269 return LsstComCamRawFormatter 

270 

271 

272class LsstComCamSim(LsstCam): 

273 """Gen3 Butler specialization for ComCamSim data. 

274 """ 

275 

276 filterDefinitions = COMCAM_FILTER_DEFINITIONS 

277 instrument = "LSSTComCamSim" 

278 policyName = "comCamSim" 

279 translatorClass = LsstComCamSimTranslator 

280 

281 def getRawFormatter(self, dataId): 

282 # local import to prevent circular dependency 

283 from .rawFormatter import LsstComCamSimRawFormatter 

284 return LsstComCamSimRawFormatter 

285 

286 

287class LsstCamImSim(LsstCam): 

288 """Gen3 Butler specialization for ImSim simulations. 

289 """ 

290 

291 instrument = "LSSTCam-imSim" 

292 policyName = "imsim" 

293 translatorClass = LsstCamImSimTranslator 

294 filterDefinitions = LSSTCAM_IMSIM_FILTER_DEFINITIONS 

295 visitSystem = VisitSystem.ONE_TO_ONE 

296 

297 def getRawFormatter(self, dataId): 

298 # local import to prevent circular dependency 

299 from .rawFormatter import LsstCamImSimRawFormatter 

300 return LsstCamImSimRawFormatter 

301 

302 def _make_default_dimension_packer( 

303 self, 

304 config_attr, 

305 data_id, 

306 is_exposure=None, 

307 default="observation", 

308 ): 

309 # Docstring inherited from Instrument._make_default_dimension_packer. 

310 # Only difference is the change to default above (which reverts back 

311 # the default in lsst.pipe.base.Instrument). 

312 return super()._make_default_dimension_packer( 

313 config_attr, 

314 data_id, 

315 is_exposure=is_exposure, 

316 default=default, 

317 ) 

318 

319 

320class LsstCamPhoSim(LsstCam): 

321 """Gen3 Butler specialization for Phosim simulations. 

322 """ 

323 

324 instrument = "LSSTCam-PhoSim" 

325 policyName = "phosim" 

326 translatorClass = LsstCamPhoSimTranslator 

327 filterDefinitions = GENERIC_FILTER_DEFINITIONS 

328 visitSystem = VisitSystem.ONE_TO_ONE 

329 

330 def getRawFormatter(self, dataId): 

331 # local import to prevent circular dependency 

332 from .rawFormatter import LsstCamPhoSimRawFormatter 

333 return LsstCamPhoSimRawFormatter 

334 

335 def _make_default_dimension_packer( 

336 self, 

337 config_attr, 

338 data_id, 

339 is_exposure=None, 

340 default="observation", 

341 ): 

342 # Docstring inherited from Instrument._make_default_dimension_packer. 

343 # Only difference is the change to default above (which reverts back 

344 # the default in lsst.pipe.base.Instrument). 

345 return super()._make_default_dimension_packer( 

346 config_attr, 

347 data_id, 

348 is_exposure=is_exposure, 

349 default=default, 

350 ) 

351 

352 

353class LsstTS8(LsstCam): 

354 """Gen3 Butler specialization for raft test stand data. 

355 """ 

356 

357 filterDefinitions = TS8_FILTER_DEFINITIONS 

358 instrument = "LSST-TS8" 

359 policyName = "ts8" 

360 translatorClass = LsstTS8Translator 

361 visitSystem = VisitSystem.ONE_TO_ONE 

362 

363 def getRawFormatter(self, dataId): 

364 # local import to prevent circular dependency 

365 from .rawFormatter import LsstTS8RawFormatter 

366 return LsstTS8RawFormatter 

367 

368 def _make_default_dimension_packer( 

369 self, 

370 config_attr, 

371 data_id, 

372 is_exposure=None, 

373 default="observation", 

374 ): 

375 # Docstring inherited from Instrument._make_default_dimension_packer. 

376 # Only difference is the change to default above (which reverts back 

377 # the default in lsst.pipe.base.Instrument). 

378 return super()._make_default_dimension_packer( 

379 config_attr, 

380 data_id, 

381 is_exposure=is_exposure, 

382 default=default, 

383 ) 

384 

385 

386class LsstUCDCam(LsstCam): 

387 """Gen3 Butler specialization for UCDCam test stand data. 

388 """ 

389 filterDefinitions = UCD_FILTER_DEFINITIONS 

390 instrument = "LSST-UCDCam" 

391 policyName = "ucd" 

392 translatorClass = LsstUCDCamTranslator 

393 visitSystem = VisitSystem.ONE_TO_ONE 

394 

395 def getRawFormatter(self, dataId): 

396 # local import to prevent circular dependency 

397 from .rawFormatter import LsstUCDCamRawFormatter 

398 return LsstUCDCamRawFormatter 

399 

400 def _make_default_dimension_packer( 

401 self, 

402 config_attr, 

403 data_id, 

404 is_exposure=None, 

405 default="observation", 

406 ): 

407 # Docstring inherited from Instrument._make_default_dimension_packer. 

408 # Only difference is the change to default above (which reverts back 

409 # the default in lsst.pipe.base.Instrument). 

410 return super()._make_default_dimension_packer( 

411 config_attr, 

412 data_id, 

413 is_exposure=is_exposure, 

414 default=default, 

415 ) 

416 

417 

418class LsstTS3(LsstCam): 

419 """Gen3 Butler specialization for TS3 test stand data. 

420 """ 

421 

422 filterDefinitions = TS3_FILTER_DEFINITIONS 

423 instrument = "LSST-TS3" 

424 policyName = "ts3" 

425 translatorClass = LsstTS3Translator 

426 visitSystem = VisitSystem.ONE_TO_ONE 

427 

428 def getRawFormatter(self, dataId): 

429 # local import to prevent circular dependency 

430 from .rawFormatter import LsstTS3RawFormatter 

431 return LsstTS3RawFormatter 

432 

433 def _make_default_dimension_packer( 

434 self, 

435 config_attr, 

436 data_id, 

437 is_exposure=None, 

438 default="observation", 

439 ): 

440 # Docstring inherited from Instrument._make_default_dimension_packer. 

441 # Only difference is the change to default above (which reverts back 

442 # the default in lsst.pipe.base.Instrument). 

443 return super()._make_default_dimension_packer( 

444 config_attr, 

445 data_id, 

446 is_exposure=is_exposure, 

447 default=default, 

448 ) 

449 

450 

451class Latiss(LsstCam): 

452 """Gen3 Butler specialization for AuxTel LATISS data. 

453 """ 

454 filterDefinitions = LATISS_FILTER_DEFINITIONS 

455 instrument = "LATISS" 

456 policyName = "latiss" 

457 translatorClass = LatissTranslator 

458 

459 def extractDetectorRecord(self, camGeomDetector): 

460 # Override to remove group (raft) name, because LATISS only has one 

461 # detector. 

462 record = super().extractDetectorRecord(camGeomDetector) 

463 record["raft"] = None 

464 record["name_in_raft"] = record["full_name"] 

465 return record 

466 

467 def getRawFormatter(self, dataId): 

468 # local import to prevent circular dependency 

469 from .rawFormatter import LatissRawFormatter 

470 return LatissRawFormatter