Coverage for python/lsst/obs/lsst/_instrument.py: 57%

166 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-02 04:44 -0700

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("LsstCam", "LsstCamImSim", "LsstCamPhoSim", "LsstTS8", 

23 "Latiss", "LsstTS3", "LsstUCDCam", "LsstComCam", "LsstComCamSim", 

24 "LsstCamSim") 

25 

26import datetime 

27import hashlib 

28import os.path 

29 

30import lsst.obs.base.yamlCamera as yamlCamera 

31from lsst.utils.introspection import get_full_type_name 

32from lsst.utils import getPackageDir 

33from lsst.obs.base import Instrument, VisitSystem 

34from .filters import (LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS, 

35 LSSTCAM_IMSIM_FILTER_DEFINITIONS, TS3_FILTER_DEFINITIONS, 

36 TS8_FILTER_DEFINITIONS, COMCAM_FILTER_DEFINITIONS, 

37 GENERIC_FILTER_DEFINITIONS, UCD_FILTER_DEFINITIONS, 

38 ) 

39 

40from .translators import LatissTranslator, LsstCamTranslator, \ 

41 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \ 

42 LsstCamPhoSimTranslator, LsstTS8Translator, LsstCamImSimTranslator, \ 

43 LsstComCamSimTranslator, LsstCamSimTranslator 

44 

45from .translators.lsst import GROUP_RE, TZERO_DATETIME 

46 

47PACKAGE_DIR = getPackageDir("obs_lsst") 

48 

49 

50class LsstCam(Instrument): 

51 """Gen3 Butler specialization for the LSST Main Camera. 

52 

53 Parameters 

54 ---------- 

55 camera : `lsst.cameraGeom.Camera` 

56 Camera object from which to extract detector information. 

57 filters : `list` of `FilterDefinition` 

58 An ordered list of filters to define the set of PhysicalFilters 

59 associated with this instrument in the registry. 

60 

61 While both the camera geometry and the set of filters associated with a 

62 camera are expected to change with time in general, their Butler Registry 

63 representations defined by an Instrument do not. Instead: 

64 

65 - We only extract names, IDs, and purposes from the detectors in the 

66 camera, which should be static information that actually reflects 

67 detector "slots" rather than the physical sensors themselves. Because 

68 the distinction between physical sensors and slots is unimportant in 

69 the vast majority of Butler use cases, we just use "detector" even 

70 though the concept really maps better to "detector slot". Ideally in 

71 the future this distinction between static and time-dependent 

72 information would be encoded in cameraGeom itself (e.g. by making the 

73 time-dependent Detector class inherit from a related class that only 

74 carries static content). 

75 

76 - The Butler Registry is expected to contain physical_filter entries for 

77 all filters an instrument has ever had, because we really only care 

78 about which filters were used for particular observations, not which 

79 filters were *available* at some point in the past. And changes in 

80 individual filters over time will be captured as changes in their 

81 TransmissionCurve datasets, not changes in the registry content (which 

82 is really just a label). While at present Instrument and Registry 

83 do not provide a way to add new physical_filters, they will in the 

84 future. 

85 """ 

86 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

87 instrument = "LSSTCam" 

88 policyName = "lsstCam" 

89 translatorClass = LsstCamTranslator 

90 obsDataPackage = "obs_lsst_data" 

91 visitSystem = VisitSystem.BY_SEQ_START_END 

92 

93 @property 

94 def configPaths(self): 

95 return [os.path.join(PACKAGE_DIR, "config"), 

96 os.path.join(PACKAGE_DIR, "config", self.policyName)] 

97 

98 @classmethod 

99 def getName(cls): 

100 # Docstring inherited from Instrument.getName 

101 return cls.instrument 

102 

103 @classmethod 

104 def getCamera(cls): 

105 # Constructing a YAML camera takes a long time but we rely on 

106 # yamlCamera to cache for us. 

107 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") 

108 camera = yamlCamera.makeCamera(cameraYamlFile) 

109 if camera.getName() != cls.getName(): 

110 raise RuntimeError(f"Expected to read camera geometry for {cls.instrument}" 

111 f" but instead got geometry for {camera.getName()}") 

112 return camera 

113 

114 def _make_default_dimension_packer( 

115 self, 

116 config_attr, 

117 data_id, 

118 is_exposure=None, 

119 default="rubin", 

120 ): 

121 # Docstring inherited from Instrument._make_default_dimension_packer. 

122 # Only difference is the change to default above. 

123 return super()._make_default_dimension_packer( 

124 config_attr, 

125 data_id, 

126 is_exposure=is_exposure, 

127 default=default, 

128 ) 

129 

130 def getRawFormatter(self, dataId): 

131 # Docstring inherited from Instrument.getRawFormatter 

132 # local import to prevent circular dependency 

133 from .rawFormatter import LsstCamRawFormatter 

134 return LsstCamRawFormatter 

135 

136 def register(self, registry, update=False): 

137 # Docstring inherited from Instrument.register 

138 # The maximum values below make Gen3's ObservationDataIdPacker produce 

139 # outputs that match Gen2's ccdExposureId. 

140 obsMax = self.translatorClass.max_exposure_id() 

141 # Make sure this is at least 1 to avoid non-uniqueness issues (e.g. 

142 # for data ids that also get used in indexing). 

143 detectorMax = max(self.translatorClass.DETECTOR_MAX, 1) 

144 with registry.transaction(): 

145 registry.syncDimensionData( 

146 "instrument", 

147 { 

148 "name": self.getName(), 

149 "detector_max": detectorMax, 

150 "visit_max": obsMax, 

151 "exposure_max": obsMax, 

152 "class_name": get_full_type_name(self), 

153 "visit_system": None if self.visitSystem is None else self.visitSystem.value, 

154 }, 

155 update=update 

156 ) 

157 for detector in self.getCamera(): 

158 registry.syncDimensionData("detector", self.extractDetectorRecord(detector), update=update) 

159 

160 self._registerFilters(registry, update=update) 

161 

162 def extractDetectorRecord(self, camGeomDetector): 

163 """Create a Gen3 Detector entry dict from a cameraGeom.Detector. 

164 """ 

165 # All of the LSST instruments have detector names like R??_S??; we'll 

166 # split them up here, and instruments with only one raft can override 

167 # to change the group to something else if desired. 

168 # Long-term, we should get these fields into cameraGeom separately 

169 # so there's no need to specialize at this stage. 

170 # They are separate in ObservationInfo 

171 group, name = camGeomDetector.getName().split("_") 

172 

173 # getType() returns a pybind11-wrapped enum, which unfortunately 

174 # has no way to extract the name of just the value (it's always 

175 # prefixed by the enum type name). 

176 purpose = str(camGeomDetector.getType()).split(".")[-1] 

177 

178 return dict( 

179 instrument=self.getName(), 

180 id=camGeomDetector.getId(), 

181 full_name=camGeomDetector.getName(), 

182 name_in_raft=name, 

183 purpose=purpose, 

184 raft=group, 

185 ) 

186 

187 @classmethod 

188 def group_name_to_group_id(cls, group_name: str) -> int: 

189 """Translate the exposure group name to an integer. 

190 

191 Parameters 

192 ---------- 

193 group_name : `str` 

194 The name of the exposure group. 

195 

196 Returns 

197 ------- 

198 id : `int` 

199 The exposure group name in integer form. This integer might be 

200 used as an ID to uniquely identify the group in contexts where 

201 a string can not be used. 

202 

203 Notes 

204 ----- 

205 If given a group name that can be directly cast to an integer it 

206 returns the integer. If the group name looks like an ISO date the ID 

207 returned is seconds since an arbitrary recent epoch. Otherwise 

208 the group name is hashed and the first 14 digits of the hash is 

209 returned along with the length of the group name. 

210 """ 

211 # If the group is an int we return it 

212 try: 

213 group_id = int(group_name) 

214 return group_id 

215 except ValueError: 

216 pass 

217 

218 # A Group is defined as ISO date with an extension 

219 # The integer must be the same for a given group so we can never 

220 # use datetime_begin. 

221 # Nominally a GROUPID looks like "ISODATE+N" where the +N is 

222 # optional. This can be converted to seconds since epoch with 

223 # N being zero-padded to 4 digits and appended (defaulting to 0). 

224 # For early data lacking that form we hash the group and return 

225 # the int. 

226 matches_date = GROUP_RE.match(group_name) 

227 if matches_date: 

228 iso_str = matches_date.group(1) 

229 fraction = matches_date.group(2) 

230 n = matches_date.group(3) 

231 if n is not None: 

232 n = int(n) 

233 else: 

234 n = 0 

235 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S") 

236 

237 tdelta = iso - TZERO_DATETIME 

238 epoch = int(tdelta.total_seconds()) 

239 

240 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N 

241 group_id = int(f"{epoch}{fraction}{n:04d}") 

242 else: 

243 # Non-standard string so convert to numbers 

244 # using a hash function. Use the first N hex digits 

245 group_bytes = group_name.encode("us-ascii") 

246 hasher = hashlib.blake2b(group_bytes) 

247 # Need to be big enough it does not possibly clash with the 

248 # date-based version above 

249 digest = hasher.hexdigest()[:14] 

250 group_id = int(digest, base=16) 

251 

252 # To help with hash collision, append the string length 

253 group_id = int(f"{group_id}{len(group_name):02d}") 

254 

255 return group_id 

256 

257 

258class LsstComCam(LsstCam): 

259 """Gen3 Butler specialization for ComCam data. 

260 """ 

261 

262 filterDefinitions = COMCAM_FILTER_DEFINITIONS 

263 instrument = "LSSTComCam" 

264 policyName = "comCam" 

265 translatorClass = LsstComCamTranslator 

266 

267 def getRawFormatter(self, dataId): 

268 # local import to prevent circular dependency 

269 from .rawFormatter import LsstComCamRawFormatter 

270 return LsstComCamRawFormatter 

271 

272 

273class LsstComCamSim(LsstCam): 

274 """Gen3 Butler specialization for ComCamSim data. 

275 """ 

276 

277 filterDefinitions = COMCAM_FILTER_DEFINITIONS 

278 instrument = "LSSTComCamSim" 

279 policyName = "comCamSim" 

280 translatorClass = LsstComCamSimTranslator 

281 

282 def getRawFormatter(self, dataId): 

283 # local import to prevent circular dependency 

284 from .rawFormatter import LsstComCamSimRawFormatter 

285 return LsstComCamSimRawFormatter 

286 

287 

288class LsstCamSim(LsstCam): 

289 """Gen3 Butler specialization for LSSTCamSim data. 

290 """ 

291 

292 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

293 instrument = "LSSTCamSim" 

294 policyName = "lsstCamSim" 

295 translatorClass = LsstCamSimTranslator 

296 

297 def getRawFormatter(self, dataId): 

298 # local import to prevent circular dependency 

299 from .rawFormatter import LsstCamSimRawFormatter 

300 return LsstCamSimRawFormatter 

301 

302 

303class LsstCamImSim(LsstCam): 

304 """Gen3 Butler specialization for ImSim simulations. 

305 """ 

306 

307 instrument = "LSSTCam-imSim" 

308 policyName = "imsim" 

309 translatorClass = LsstCamImSimTranslator 

310 filterDefinitions = LSSTCAM_IMSIM_FILTER_DEFINITIONS 

311 visitSystem = VisitSystem.ONE_TO_ONE 

312 

313 def getRawFormatter(self, dataId): 

314 # local import to prevent circular dependency 

315 from .rawFormatter import LsstCamImSimRawFormatter 

316 return LsstCamImSimRawFormatter 

317 

318 def _make_default_dimension_packer( 

319 self, 

320 config_attr, 

321 data_id, 

322 is_exposure=None, 

323 default="observation", 

324 ): 

325 # Docstring inherited from Instrument._make_default_dimension_packer. 

326 # Only difference is the change to default above (which reverts back 

327 # the default in lsst.pipe.base.Instrument). 

328 return super()._make_default_dimension_packer( 

329 config_attr, 

330 data_id, 

331 is_exposure=is_exposure, 

332 default=default, 

333 ) 

334 

335 

336class LsstCamPhoSim(LsstCam): 

337 """Gen3 Butler specialization for Phosim simulations. 

338 """ 

339 

340 instrument = "LSSTCam-PhoSim" 

341 policyName = "phosim" 

342 translatorClass = LsstCamPhoSimTranslator 

343 filterDefinitions = GENERIC_FILTER_DEFINITIONS 

344 visitSystem = VisitSystem.ONE_TO_ONE 

345 

346 def getRawFormatter(self, dataId): 

347 # local import to prevent circular dependency 

348 from .rawFormatter import LsstCamPhoSimRawFormatter 

349 return LsstCamPhoSimRawFormatter 

350 

351 def _make_default_dimension_packer( 

352 self, 

353 config_attr, 

354 data_id, 

355 is_exposure=None, 

356 default="observation", 

357 ): 

358 # Docstring inherited from Instrument._make_default_dimension_packer. 

359 # Only difference is the change to default above (which reverts back 

360 # the default in lsst.pipe.base.Instrument). 

361 return super()._make_default_dimension_packer( 

362 config_attr, 

363 data_id, 

364 is_exposure=is_exposure, 

365 default=default, 

366 ) 

367 

368 

369class LsstTS8(LsstCam): 

370 """Gen3 Butler specialization for raft test stand data. 

371 """ 

372 

373 filterDefinitions = TS8_FILTER_DEFINITIONS 

374 instrument = "LSST-TS8" 

375 policyName = "ts8" 

376 translatorClass = LsstTS8Translator 

377 visitSystem = VisitSystem.ONE_TO_ONE 

378 

379 def getRawFormatter(self, dataId): 

380 # local import to prevent circular dependency 

381 from .rawFormatter import LsstTS8RawFormatter 

382 return LsstTS8RawFormatter 

383 

384 def _make_default_dimension_packer( 

385 self, 

386 config_attr, 

387 data_id, 

388 is_exposure=None, 

389 default="observation", 

390 ): 

391 # Docstring inherited from Instrument._make_default_dimension_packer. 

392 # Only difference is the change to default above (which reverts back 

393 # the default in lsst.pipe.base.Instrument). 

394 return super()._make_default_dimension_packer( 

395 config_attr, 

396 data_id, 

397 is_exposure=is_exposure, 

398 default=default, 

399 ) 

400 

401 

402class LsstUCDCam(LsstCam): 

403 """Gen3 Butler specialization for UCDCam test stand data. 

404 """ 

405 filterDefinitions = UCD_FILTER_DEFINITIONS 

406 instrument = "LSST-UCDCam" 

407 policyName = "ucd" 

408 translatorClass = LsstUCDCamTranslator 

409 visitSystem = VisitSystem.ONE_TO_ONE 

410 

411 def getRawFormatter(self, dataId): 

412 # local import to prevent circular dependency 

413 from .rawFormatter import LsstUCDCamRawFormatter 

414 return LsstUCDCamRawFormatter 

415 

416 def _make_default_dimension_packer( 

417 self, 

418 config_attr, 

419 data_id, 

420 is_exposure=None, 

421 default="observation", 

422 ): 

423 # Docstring inherited from Instrument._make_default_dimension_packer. 

424 # Only difference is the change to default above (which reverts back 

425 # the default in lsst.pipe.base.Instrument). 

426 return super()._make_default_dimension_packer( 

427 config_attr, 

428 data_id, 

429 is_exposure=is_exposure, 

430 default=default, 

431 ) 

432 

433 

434class LsstTS3(LsstCam): 

435 """Gen3 Butler specialization for TS3 test stand data. 

436 """ 

437 

438 filterDefinitions = TS3_FILTER_DEFINITIONS 

439 instrument = "LSST-TS3" 

440 policyName = "ts3" 

441 translatorClass = LsstTS3Translator 

442 visitSystem = VisitSystem.ONE_TO_ONE 

443 

444 def getRawFormatter(self, dataId): 

445 # local import to prevent circular dependency 

446 from .rawFormatter import LsstTS3RawFormatter 

447 return LsstTS3RawFormatter 

448 

449 def _make_default_dimension_packer( 

450 self, 

451 config_attr, 

452 data_id, 

453 is_exposure=None, 

454 default="observation", 

455 ): 

456 # Docstring inherited from Instrument._make_default_dimension_packer. 

457 # Only difference is the change to default above (which reverts back 

458 # the default in lsst.pipe.base.Instrument). 

459 return super()._make_default_dimension_packer( 

460 config_attr, 

461 data_id, 

462 is_exposure=is_exposure, 

463 default=default, 

464 ) 

465 

466 

467class Latiss(LsstCam): 

468 """Gen3 Butler specialization for AuxTel LATISS data. 

469 """ 

470 filterDefinitions = LATISS_FILTER_DEFINITIONS 

471 instrument = "LATISS" 

472 policyName = "latiss" 

473 translatorClass = LatissTranslator 

474 

475 def extractDetectorRecord(self, camGeomDetector): 

476 # Override to remove group (raft) name, because LATISS only has one 

477 # detector. 

478 record = super().extractDetectorRecord(camGeomDetector) 

479 record["raft"] = None 

480 record["name_in_raft"] = record["full_name"] 

481 return record 

482 

483 def getRawFormatter(self, dataId): 

484 # local import to prevent circular dependency 

485 from .rawFormatter import LatissRawFormatter 

486 return LatissRawFormatter