Coverage for python/lsst/obs/lsst/_packer.py: 25%

132 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 17:06 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("RubinDimensionPacker",) 

25 

26import datetime 

27import math 

28 

29from lsst.daf.butler import DataCoordinate, DimensionPacker 

30from lsst.pex.config import Config, Field, DictField 

31from lsst.pipe.base import observation_packer_registry 

32from .translators.lsst import CONTROLLERS, EXPOSURE_ID_MAXDIGITS, LsstBaseTranslator 

33 

34 

35def convert_day_obs_to_ordinal(day_obs: int) -> int: 

36 """Convert a YYYYMMDD decimal-digit integer date to a day ordinal. 

37 

38 Parameters 

39 ---------- 

40 day_obs : `int` 

41 A YYYYMMDD decimal-digit integer. 

42 

43 Returns 

44 ------- 

45 day_ordinal : `int` 

46 An integer that counts days directly, with absolute offset 

47 unspecified. 

48 """ 

49 year_month, day = divmod(day_obs, 100) 

50 year, month = divmod(year_month, 100) 

51 return datetime.date(year, month, day).toordinal() 

52 

53 

54def convert_ordinal_to_day_obs(day_ordinal: int) -> int: 

55 """Convert a day ordinal to a YYYYMMDD decimal-digit integer date. 

56 

57 Parameters 

58 ---------- 

59 day_ordinal : `int` 

60 An integer that counts days directly, with absolute offset 

61 unspecified. 

62 

63 Returns 

64 ------- 

65 day_obs : `int` 

66 A YYYYMMDD decimal-digit integer. 

67 """ 

68 date = datetime.date.fromordinal(day_ordinal) 

69 return (date.year * 100 + date.month) * 100 + date.day 

70 

71 

72def _is_positive(x: int) -> bool: 

73 """Callable that tests whether an integer is positive, for use as a 

74 config ``check`` argument.""" 

75 return x > 0 

76 

77 

78class RubinDimensionPackerConfig(Config): 

79 

80 controllers = DictField( 

81 "Mapping from controller code to integer.", 

82 keytype=str, 

83 itemtype=int 

84 # Default is set from the CONTROLLERS constant in translators/lsst.py 

85 # below. 

86 ) 

87 

88 n_controllers = Field( 

89 "Reserved number of controller codes. May be larger than `len(controllers)`.", 

90 dtype=int, 

91 check=_is_positive, 

92 default=8, 

93 # Default by rounding 5 (current set of codes) up to the nearest power 

94 # of 2. 

95 ) 

96 

97 n_visit_definitions = Field( 

98 "Reserved number of visit definitions a single exposure may belong to.", 

99 dtype=int, 

100 check=_is_positive, 

101 default=2, 

102 # We need one bit for one-to-one visits that contain only the first 

103 # exposure in a sequence that was originally observed as a multi-snap 

104 # sequence. 

105 ) 

106 

107 n_days = Field( 

108 "Reserved number of distinct valid-date day_obs values, starting from `day_obs_begin`.", 

109 dtype=int, 

110 check=_is_positive, 

111 default=16384, 

112 # Default of 16384 is about 45 years, which with day_obs_begin is 

113 # roughly consistent (and a bit bigger than) the bounds permitted by 

114 # the translator. 

115 ) 

116 

117 n_seq_nums = Field( 

118 "Reserved number of seq_num values, starting from 0.", 

119 dtype=int, 

120 check=_is_positive, 

121 default=32768, 

122 # Default is one exposure every 2.63s for a full day, which is really 

123 # close to the hardware limit of one every 2.3s, and far from what 

124 # anyone would actually do in practice. 

125 ) 

126 

127 n_detectors = Field( 

128 "Reserved number of detectors, starting from 0.", 

129 dtype=int, 

130 check=_is_positive, 

131 default=256 

132 # Default is the number of actual detectors (201, including corner 

133 # rafts) rounded up to a power of 2. 

134 ) 

135 

136 day_obs_begin = Field( 

137 "Inclusive lower bound on day_obs.", 

138 dtype=int, 

139 default=20100101 

140 # Default is just a nice round date that (with n_days) puts the end 

141 # point just after the 2050 bound in the translators. 

142 ) 

143 

144 def setDefaults(self): 

145 super().setDefaults() 

146 self.controllers = {c: i for i, c in enumerate(CONTROLLERS)} 

147 

148 def validate(self): 

149 super().validate() 

150 for c, i in self.controllers.items(): 

151 if i >= self.n_controllers: 

152 raise ValueError( 

153 f"Controller code {c!r} has index {i}, which is out of bounds " 

154 f"for n_controllers={self.n_controllers}." 

155 ) 

156 

157 

158class RubinDimensionPacker(DimensionPacker): 

159 """A data ID packer that converts Rubin visit+detector and 

160 exposure+detector data IDs to integers. 

161 

162 Parameters 

163 ---------- 

164 data_id : `lsst.daf.butler.DataCoordinate` 

165 Data ID identifying at least the instrument dimension. Does not need 

166 to have dimension records attached. 

167 config : `RubinDimensionPackerConfig` 

168 Configuration for this dimension packer. 

169 is_exposure : `bool`, optional 

170 If `False`, construct a packer for visit+detector data IDs. If `True`, 

171 construct a packer for exposure+detector data IDs. If `None`, this is 

172 determined based on whether ``visit`` or ``exposure`` is present in 

173 ``data_id``, with ``visit`` checked first and hence used if both are 

174 present. 

175 

176 Notes 

177 ----- 

178 The packing used by this class is considered stable and part of its public 

179 interface so it can be reimplemented in contexts where delegation to this 

180 code is impractical (e.g. SQL user-defined functions):: 

181 

182 packed = \ 

183 detector + config.n_detectors * ( 

184 seq_num + config.n_seq_nums * ( 

185 convert_day_obs_to_ordinal(day_obs) 

186 - convert_day_obs_to_ordinal(config.day_obs_begin) 

187 + config.n_days * ( 

188 config.controllers[controllers] 

189 config.n_controllers * is_one_to_one_reinterpretation 

190 ) 

191 ) 

192 ) 

193 

194 See `RubinDimensionPackerConfig` and `pack_decomposition` for definitions 

195 of the above variables. 

196 """ 

197 

198 ConfigClass = RubinDimensionPackerConfig 

199 

200 def __init__( 

201 self, 

202 data_id: DataCoordinate, 

203 *, 

204 config: RubinDimensionPackerConfig | None, 

205 is_exposure: bool | None = None, 

206 ): 

207 if config is None: 

208 config = RubinDimensionPackerConfig() 

209 fixed = data_id.subset(data_id.universe.extract(["instrument"])) 

210 if is_exposure is None and data_id is not None: 

211 if "visit" in data_id.graph.names: 

212 is_exposure = False 

213 elif "exposure" in data_id.graph.names: 

214 is_exposure = True 

215 else: 

216 raise ValueError( 

217 "'is_exposure' was not provided and 'data_id' has no visit or exposure value." 

218 ) 

219 if is_exposure: 

220 dimensions = fixed.universe.extract(["instrument", "exposure", "detector"]) 

221 else: 

222 dimensions = fixed.universe.extract(["instrument", "visit", "detector"]) 

223 super().__init__(fixed, dimensions) 

224 self.config = config 

225 self.is_exposure = is_exposure 

226 self._max_bits = ( 

227 math.prod( 

228 [ 

229 self.config.n_visit_definitions, 

230 self.config.n_controllers, 

231 self.config.n_days, 

232 self.config.n_seq_nums, 

233 self.config.n_detectors, 

234 ] 

235 ) 

236 - 1 

237 ).bit_length() 

238 

239 @property 

240 def maxBits(self) -> int: 

241 # Docstring inherited from DimensionPacker.maxBits 

242 return self._max_bits 

243 

244 def _pack(self, dataId: DataCoordinate) -> int: 

245 # Docstring inherited from DimensionPacker._pack 

246 is_one_to_one_reinterpretation = False 

247 if not self.is_exposure: 

248 # Using a leading "9" as the indicator of a 

249 # one_to_one_reinterpretation visit is _slightly_ distasteful, as 

250 # it'd be better to delegate that to something in obs_base closer 

251 # to what puts the "9" there in the first place, but that class 

252 # doesn't have its own public interface where we could put such 

253 # things, and we don't have much choice but to assume which of the 

254 # visit system definitions we're using anyway. Good news is that 

255 # this is all very strictly RFC-controlled stable stuff that is 

256 # not going to change out from under us without warning. 

257 nine_if_special, exposure_id = divmod( 

258 dataId["visit"], 10**EXPOSURE_ID_MAXDIGITS 

259 ) 

260 if nine_if_special == 9: 

261 is_one_to_one_reinterpretation = True 

262 elif nine_if_special != 0: 

263 raise ValueError(f"Could not parse visit in {dataId}.") 

264 else: 

265 exposure_id = dataId["exposure"] 

266 # We unpack the exposure ID (which may really be [the remnant of] a 

267 # visit ID) instead of extracting these values from dimension records 

268 # because we really don't want to demand that the given data ID have 

269 # records attached. 

270 return self.pack_id_pair( 

271 exposure_id, 

272 dataId["detector"], 

273 is_one_to_one_reinterpretation, 

274 config=self.config, 

275 ) 

276 

277 def unpack(self, packedId: int) -> DataCoordinate: 

278 # Docstring inherited from DimensionPacker.unpack 

279 ( 

280 exposure_id, 

281 detector, 

282 is_one_to_one_reinterpretation, 

283 ) = self.unpack_id_pair(packedId, config=self.config) 

284 if self.is_exposure: 

285 if is_one_to_one_reinterpretation: 

286 raise ValueError( 

287 f"Packed data ID {packedId} may correspond to a valid visit ID, " 

288 "but not a valid exposure ID." 

289 ) 

290 return DataCoordinate.standardize( 

291 self.fixed, exposure=exposure_id, detector=detector 

292 ) 

293 else: 

294 if is_one_to_one_reinterpretation: 

295 visit_id = int(f"9{exposure_id}") 

296 else: 

297 visit_id = exposure_id 

298 return DataCoordinate.standardize( 

299 self.fixed, visit=visit_id, detector=detector 

300 ) 

301 

302 @staticmethod 

303 def pack_id_pair( 

304 exposure_id: int, 

305 detector: int, 

306 is_one_to_one_reinterpretation: bool = False, 

307 config: RubinDimensionPackerConfig | None = None, 

308 ) -> int: 

309 """Pack data ID values passed as arguments. 

310 

311 Parameters 

312 ---------- 

313 exposure_id : `int` 

314 Integer that uniquely identifies an exposure. 

315 detector : `int` 

316 Integer that uniquely identifies a detector. 

317 is_one_to_one_reinterpretation : `bool`, optional 

318 If `True`, instead of packing the given ``exposure_id``, pack a 

319 visit ID that represents the alternate interpretation of that 

320 exposure (which must be the first snap in a multi-snap sequence) as 

321 a standalone visit. 

322 

323 Returns 

324 ------- 

325 packed_id : `int` 

326 Integer that reversibly combines all of the given arguments. 

327 

328 Notes 

329 ----- 

330 This is a `staticmethod` and hence does not respect the config passed 

331 in at construction when called on an instance. This is to support 

332 usage in contexts where construction (which requires a 

333 `lsst.daf.butler.DimensionUniverse`) is inconvenient or impossible. 

334 """ 

335 day_obs, seq_num, controller = LsstBaseTranslator.unpack_exposure_id( 

336 exposure_id 

337 ) 

338 return RubinDimensionPacker.pack_decomposition( 

339 int(day_obs), 

340 seq_num, 

341 detector=detector, 

342 controller=controller, 

343 is_one_to_one_reinterpretation=is_one_to_one_reinterpretation, 

344 config=config, 

345 ) 

346 

347 @staticmethod 

348 def unpack_id_pair( 

349 packed_id: int, config: RubinDimensionPackerConfig | None = None 

350 ) -> tuple[int, int, bool]: 

351 """Unpack data ID values directly. 

352 

353 Parameters 

354 ---------- 

355 packed_id : `int` 

356 Integer produced by one of the methods of this class using the same 

357 configuration. 

358 

359 Returns 

360 ------- 

361 exposure_id : `int` 

362 Integer that uniquely identifies an exposure. 

363 detector : `int` 

364 Integer that uniquely identifies a detector. 

365 is_one_to_one_reinterpretation : `bool`, optional 

366 If `True`, instead of packing the given ``exposure_id``, the packed 

367 ID corresponds to the visit that represents the alternate 

368 interpretation of the first snap in a multi-snap sequence as a 

369 standalone visit. 

370 

371 Notes 

372 ----- 

373 This is a `staticmethod` and hence does not respect the config passed 

374 in at construction when called on an instance. This is to support 

375 usage in contexts where construction (which requires a 

376 `lsst.daf.butler.DimensionUniverse`) is inconvenient or impossible. 

377 """ 

378 ( 

379 day_obs, 

380 seq_num, 

381 detector, 

382 controller, 

383 is_one_to_one_reinterpretation, 

384 ) = RubinDimensionPacker.unpack_decomposition(packed_id, config=config) 

385 return ( 

386 LsstBaseTranslator.compute_exposure_id(str(day_obs), seq_num, controller), 

387 detector, 

388 is_one_to_one_reinterpretation, 

389 ) 

390 

391 @staticmethod 

392 def pack_decomposition( 

393 day_obs: int, 

394 seq_num: int, 

395 detector: int, 

396 controller: str = "O", 

397 is_one_to_one_reinterpretation: bool = False, 

398 config: RubinDimensionPackerConfig | None = None, 

399 ) -> int: 

400 """Pack Rubin-specific identifiers directly into an integer. 

401 

402 Parameters 

403 ---------- 

404 day_obs : `int` 

405 Day of observation as a YYYYMMDD decimal integer. 

406 seq_num : `int` 

407 Sequence number 

408 detector : `int` 

409 Detector ID. 

410 controller : `str`, optional 

411 Single-character controller code defined in 

412 `RubinDimensionPackerConfig.controllers`. 

413 is_one_to_one_reinterpretation : `bool`, optional 

414 If `True`, this is a visit ID that differs from the exposure ID of 

415 its first snap because it is the alternate interpretation of that 

416 first snap as a standalone visit. 

417 config : `RubinDimensionPackerConfig`, optional 

418 Configuration, including upper bounds on all arguments. 

419 

420 Returns 

421 ------- 

422 packed_id : `int` 

423 Integer that reversibly combines all of the given arguments. 

424 

425 Notes 

426 ----- 

427 This is a `staticmethod` and hence does not respect the config passed 

428 in at construction when called on an instance. This is to support 

429 usage in contexts where construction (which requires a 

430 `lsst.daf.butler.DimensionUniverse`) is inconvenient or impossible. 

431 """ 

432 if config is None: 

433 config = RubinDimensionPackerConfig() 

434 day_obs_ordinal_begin = convert_day_obs_to_ordinal(config.day_obs_begin) 

435 result = int(is_one_to_one_reinterpretation) 

436 result *= config.n_controllers 

437 try: 

438 result += config.controllers[controller] 

439 except KeyError: 

440 raise ValueError(f"Unrecognized controller code {controller!r}.") from None 

441 day_obs_ordinal = convert_day_obs_to_ordinal(day_obs) - day_obs_ordinal_begin 

442 if day_obs_ordinal < 0: 

443 raise ValueError( 

444 f"day_obs {day_obs} is out of bounds; must be >= " 

445 f"{convert_ordinal_to_day_obs(day_obs_ordinal_begin)}." 

446 ) 

447 if day_obs_ordinal > config.n_days: 

448 raise ValueError( 

449 f"day_obs {day_obs} is out of bounds; must be < " 

450 f"{convert_ordinal_to_day_obs(day_obs_ordinal_begin + config.n_days)}." 

451 ) 

452 result *= config.n_days 

453 result += day_obs_ordinal 

454 if seq_num < 0: 

455 raise ValueError(f"seq_num {seq_num} is negative.") 

456 if seq_num >= config.n_seq_nums: 

457 raise ValueError( 

458 f"seq_num is out of bounds; must be < {config.n_seq_nums}." 

459 ) 

460 result *= config.n_seq_nums 

461 result += seq_num 

462 if detector < 0: 

463 raise ValueError(f"detector {detector} is out of bounds; must be >= 0.") 

464 if detector >= config.n_detectors: 

465 raise ValueError( 

466 f"detector {detector} is out of bounds; must be < {config.n_detectors}." 

467 ) 

468 result *= config.n_detectors 

469 result += detector 

470 return result 

471 

472 @staticmethod 

473 def unpack_decomposition( 

474 packed_id: int, config: RubinDimensionPackerConfig | None = None 

475 ) -> tuple[int, int, int, str, bool]: 

476 """Unpack an integer into Rubin-specific identifiers. 

477 

478 Parameters 

479 ---------- 

480 packed_id : `int` 

481 Integer produced by one of the methods of this class using the same 

482 configuration. 

483 config : `RubinDimensionPackerConfig`, optional 

484 Configuration, including upper bounds on all arguments. 

485 

486 Returns 

487 ------- 

488 day_obs : `int` 

489 Day of observation as a YYYYMMDD decimal integer. 

490 seq_num : `int` 

491 Sequence number 

492 detector : `int` 

493 Detector ID. 

494 controller : `str` 

495 Single-character controller code defined in 

496 `RubinDimensionPackerConfig.controllers`. 

497 is_one_to_one_reinterpretation : `bool` 

498 If `True`, this is a visit ID that differs from the exposure ID of 

499 its first snap because it is the alternate interpretation of that 

500 first snap as a standalone visit. 

501 

502 Notes 

503 ----- 

504 This is a `staticmethod` and hence does not respect the config passed 

505 in at construction when called on an instance. This is to support 

506 usage in contexts where construction (which requires a 

507 `lsst.daf.butler.DimensionUniverse`) is inconvenient or impossible. 

508 """ 

509 if config is None: 

510 config = RubinDimensionPackerConfig() 

511 rest, detector = divmod(packed_id, config.n_detectors) 

512 rest, seq_num = divmod(rest, config.n_seq_nums) 

513 rest, day_obs_ordinal = divmod(rest, config.n_days) 

514 rest, controller_int = divmod(rest, config.n_controllers) 

515 rest, is_one_to_one_reintepretation_int = divmod( 

516 rest, config.n_visit_definitions 

517 ) 

518 if rest: 

519 raise ValueError( 

520 f"Unexpected overall factor {rest} in packed data ID {packed_id}." 

521 ) 

522 for controller_code, index in config.controllers.items(): 

523 if index == controller_int: 

524 break 

525 else: 

526 raise ValueError( 

527 f"Unrecognized controller index {controller_int} in packed data ID {packed_id}." 

528 ) 

529 return ( 

530 convert_ordinal_to_day_obs(day_obs_ordinal + convert_day_obs_to_ordinal(config.day_obs_begin)), 

531 seq_num, 

532 detector, 

533 controller_code, 

534 bool(is_one_to_one_reintepretation_int), 

535 ) 

536 

537 

538# The double-registration guard here would be unnecessary if not for 

539# pytest-flake8 and some horribleness it must be doing to circumvent Python's 

540# own guards against importing the same module twice in the same process. 

541if "rubin" not in observation_packer_registry: 541 ↛ exitline 541 didn't jump to the function exit

542 observation_packer_registry.register("rubin", RubinDimensionPacker)