Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING 

29import astropy.time 

30from functools import lru_cache 

31 

32from lsst.afw.cameraGeom import Camera 

33from lsst.daf.butler import ( 

34 Butler, 

35 CollectionType, 

36 DataCoordinate, 

37 DataId, 

38 DatasetType, 

39 Timespan, 

40) 

41from lsst.utils import getPackageDir, doImport 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .gen2to3 import TranslatorFactory 

45 from lsst.daf.butler import Registry 

46 

47# To be a standard text curated calibration means that we use a 

48# standard definition for the corresponding DatasetType. 

49StandardCuratedCalibrationDatasetTypes = { 

50 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

51 "storageClass": "Defects"}, 

52 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

53 "storageClass": "QECurve"}, 

54 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"), 

55 "storageClass": "CrosstalkCalib"}, 

56} 

57 

58 

59class Instrument(metaclass=ABCMeta): 

60 """Base class for instrument-specific logic for the Gen3 Butler. 

61 

62 Concrete instrument subclasses should be directly constructable with no 

63 arguments. 

64 """ 

65 

66 configPaths: Sequence[str] = () 

67 """Paths to config files to read for specific Tasks. 

68 

69 The paths in this list should contain files of the form `task.py`, for 

70 each of the Tasks that requires special configuration. 

71 """ 

72 

73 policyName: Optional[str] = None 

74 """Instrument specific name to use when locating a policy or configuration 

75 file in the file system.""" 

76 

77 obsDataPackage: Optional[str] = None 

78 """Name of the package containing the text curated calibration files. 

79 Usually a obs _data package. If `None` no curated calibration files 

80 will be read. (`str`)""" 

81 

82 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes) 

83 """The dataset types expected to be obtained from the obsDataPackage. 

84 

85 These dataset types are all required to have standard definitions and 

86 must be known to the base class. Clearing this list will prevent 

87 any of these calibrations from being stored. If a dataset type is not 

88 known to a specific instrument it can still be included in this list 

89 since the data package is the source of truth. (`set` of `str`) 

90 """ 

91 

92 additionalCuratedDatasetTypes: Set[str] = frozenset() 

93 """Curated dataset types specific to this particular instrument that do 

94 not follow the standard organization found in obs data packages. 

95 

96 These are the instrument-specific dataset types written by 

97 `writeAdditionalCuratedCalibrations` in addition to the calibrations 

98 found in obs data packages that follow the standard scheme. 

99 (`set` of `str`)""" 

100 

101 @property 

102 @abstractmethod 

103 def filterDefinitions(self): 

104 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

105 for this instrument. 

106 """ 

107 return None 

108 

109 def __init__(self): 

110 self.filterDefinitions.reset() 

111 self.filterDefinitions.defineFilters() 

112 

113 @classmethod 

114 @abstractmethod 

115 def getName(cls): 

116 """Return the short (dimension) name for this instrument. 

117 

118 This is not (in general) the same as the class name - it's what is used 

119 as the value of the "instrument" field in data IDs, and is usually an 

120 abbreviation of the full name. 

121 """ 

122 raise NotImplementedError() 

123 

124 @classmethod 

125 @lru_cache() 

126 def getCuratedCalibrationNames(cls) -> Set[str]: 

127 """Return the names of all the curated calibration dataset types. 

128 

129 Returns 

130 ------- 

131 names : `set` of `str` 

132 The dataset type names of all curated calibrations. This will 

133 include the standard curated calibrations even if the particular 

134 instrument does not support them. 

135 

136 Notes 

137 ----- 

138 The returned list does not indicate whether a particular dataset 

139 is present in the Butler repository, simply that these are the 

140 dataset types that are handled by ``writeCuratedCalibrations``. 

141 """ 

142 

143 # Camera is a special dataset type that is also handled as a 

144 # curated calibration. 

145 curated = {"camera"} 

146 

147 # Make a cursory attempt to filter out curated dataset types 

148 # that are not present for this instrument 

149 for datasetTypeName in cls.standardCuratedDatasetTypes: 

150 calibPath = cls._getSpecificCuratedCalibrationPath(datasetTypeName) 

151 if calibPath is not None: 

152 curated.add(datasetTypeName) 

153 

154 curated.update(cls.additionalCuratedDatasetTypes) 

155 return frozenset(curated) 

156 

157 @abstractmethod 

158 def getCamera(self): 

159 """Retrieve the cameraGeom representation of this instrument. 

160 

161 This is a temporary API that should go away once ``obs_`` packages have 

162 a standardized approach to writing versioned cameras to a Gen3 repo. 

163 """ 

164 raise NotImplementedError() 

165 

166 @abstractmethod 

167 def register(self, registry): 

168 """Insert instrument, physical_filter, and detector entries into a 

169 `Registry`. 

170 """ 

171 raise NotImplementedError() 

172 

173 @classmethod 

174 @lru_cache() 

175 def getObsDataPackageDir(cls): 

176 """The root of the obs data package that provides specializations for 

177 this instrument. 

178 

179 returns 

180 ------- 

181 dir : `str` 

182 The root of the relevat obs data package. 

183 """ 

184 if cls.obsDataPackage is None: 

185 return None 

186 return getPackageDir(cls.obsDataPackage) 

187 

188 @staticmethod 

189 def fromName(name: str, registry: Registry) -> Instrument: 

190 """Given an instrument name and a butler, retrieve a corresponding 

191 instantiated instrument object. 

192 

193 Parameters 

194 ---------- 

195 name : `str` 

196 Name of the instrument (must match the return value of `getName`). 

197 registry : `lsst.daf.butler.Registry` 

198 Butler registry to query to find the information. 

199 

200 Returns 

201 ------- 

202 instrument : `Instrument` 

203 An instance of the relevant `Instrument`. 

204 

205 Notes 

206 ----- 

207 The instrument must be registered in the corresponding butler. 

208 

209 Raises 

210 ------ 

211 LookupError 

212 Raised if the instrument is not known to the supplied registry. 

213 ModuleNotFoundError 

214 Raised if the class could not be imported. This could mean 

215 that the relevant obs package has not been setup. 

216 TypeError 

217 Raised if the class name retrieved is not a string. 

218 """ 

219 records = list(registry.queryDimensionRecords("instrument", instrument=name)) 

220 if not records: 

221 raise LookupError(f"No registered instrument with name '{name}'.") 

222 cls = records[0].class_name 

223 if not isinstance(cls, str): 

224 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

225 instrument = doImport(cls) 

226 return instrument() 

227 

228 @staticmethod 

229 def importAll(registry: Registry) -> None: 

230 """Import all the instruments known to this registry. 

231 

232 This will ensure that all metadata translators have been registered. 

233 

234 Parameters 

235 ---------- 

236 registry : `lsst.daf.butler.Registry` 

237 Butler registry to query to find the information. 

238 

239 Notes 

240 ----- 

241 It is allowed for a particular instrument class to fail on import. 

242 This might simply indicate that a particular obs package has 

243 not been setup. 

244 """ 

245 records = list(registry.queryDimensionRecords("instrument")) 

246 for record in records: 

247 cls = record.class_name 

248 try: 

249 doImport(cls) 

250 except Exception: 

251 pass 

252 

253 def _registerFilters(self, registry): 

254 """Register the physical and abstract filter Dimension relationships. 

255 This should be called in the ``register`` implementation. 

256 

257 Parameters 

258 ---------- 

259 registry : `lsst.daf.butler.core.Registry` 

260 The registry to add dimensions to. 

261 """ 

262 for filter in self.filterDefinitions: 

263 # fix for undefined abstract filters causing trouble in the registry: 

264 if filter.band is None: 

265 band = filter.physical_filter 

266 else: 

267 band = filter.band 

268 

269 registry.insertDimensionData("physical_filter", 

270 {"instrument": self.getName(), 

271 "name": filter.physical_filter, 

272 "band": band 

273 }) 

274 

275 @abstractmethod 

276 def getRawFormatter(self, dataId): 

277 """Return the Formatter class that should be used to read a particular 

278 raw file. 

279 

280 Parameters 

281 ---------- 

282 dataId : `DataCoordinate` 

283 Dimension-based ID for the raw file or files being ingested. 

284 

285 Returns 

286 ------- 

287 formatter : `Formatter` class 

288 Class to be used that reads the file into an 

289 `lsst.afw.image.Exposure` instance. 

290 """ 

291 raise NotImplementedError() 

292 

293 def writeCuratedCalibrations(self, butler, run=None): 

294 """Write human-curated calibration Datasets to the given Butler with 

295 the appropriate validity ranges. 

296 

297 Parameters 

298 ---------- 

299 butler : `lsst.daf.butler.Butler` 

300 Butler to use to store these calibrations. 

301 run : `str` 

302 Run to use for this collection of calibrations. If `None` the 

303 collection name is worked out automatically from the instrument 

304 name and other metadata. 

305 

306 Notes 

307 ----- 

308 Expected to be called from subclasses. The base method calls 

309 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

310 """ 

311 # Need to determine the run for ingestion based on the instrument 

312 # name and eventually the data package version. The camera geom 

313 # is currently special in that it is not in the _data package. 

314 if run is None: 

315 run = self.makeCollectionName("calib") 

316 butler.registry.registerCollection(run, type=CollectionType.RUN) 

317 self.writeCameraGeom(butler, run=run) 

318 self.writeStandardTextCuratedCalibrations(butler, run=run) 

319 self.writeAdditionalCuratedCalibrations(butler, run=run) 

320 

321 def writeAdditionalCuratedCalibrations(self, butler, run=None): 

322 """Write additional curated calibrations that might be instrument 

323 specific and are not part of the standard set. 

324 

325 Default implementation does nothing. 

326 

327 Parameters 

328 ---------- 

329 butler : `lsst.daf.butler.Butler` 

330 Butler to use to store these calibrations. 

331 run : `str`, optional 

332 Name of the run to use to override the default run associated 

333 with this Butler. 

334 """ 

335 return 

336 

337 def applyConfigOverrides(self, name, config): 

338 """Apply instrument-specific overrides for a task config. 

339 

340 Parameters 

341 ---------- 

342 name : `str` 

343 Name of the object being configured; typically the _DefaultName 

344 of a Task. 

345 config : `lsst.pex.config.Config` 

346 Config instance to which overrides should be applied. 

347 """ 

348 for root in self.configPaths: 

349 path = os.path.join(root, f"{name}.py") 

350 if os.path.exists(path): 

351 config.load(path) 

352 

353 def writeCameraGeom(self, butler, run=None): 

354 """Write the default camera geometry to the butler repository 

355 with an infinite validity range. 

356 

357 Parameters 

358 ---------- 

359 butler : `lsst.daf.butler.Butler` 

360 Butler to receive these calibration datasets. 

361 run : `str`, optional 

362 Name of the run to use to override the default run associated 

363 with this Butler. 

364 """ 

365 

366 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

367 universe=butler.registry.dimensions) 

368 butler.registry.registerDatasetType(datasetType) 

369 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

370 camera = self.getCamera() 

371 butler.put(camera, datasetType, unboundedDataId, run=run) 

372 

373 def writeStandardTextCuratedCalibrations(self, butler, run=None): 

374 """Write the set of standardized curated text calibrations to 

375 the repository. 

376 

377 Parameters 

378 ---------- 

379 butler : `lsst.daf.butler.Butler` 

380 Butler to receive these calibration datasets. 

381 run : `str`, optional 

382 Name of the run to use to override the default run associated 

383 with this Butler. 

384 """ 

385 

386 for datasetTypeName in self.standardCuratedDatasetTypes: 

387 # We need to define the dataset types. 

388 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

389 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

390 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

391 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

392 datasetType = DatasetType(datasetTypeName, 

393 universe=butler.registry.dimensions, 

394 **definition) 

395 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run) 

396 

397 @classmethod 

398 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName): 

399 """Return the path of the curated calibration directory. 

400 

401 Parameters 

402 ---------- 

403 datasetTypeName : `str` 

404 The name of the standard dataset type to find. 

405 

406 Returns 

407 ------- 

408 path : `str` 

409 The path to the standard curated data directory. `None` if the 

410 dataset type is not found or the obs data package is not 

411 available. 

412 """ 

413 if cls.getObsDataPackageDir() is None: 

414 # if there is no data package then there can't be datasets 

415 return None 

416 

417 calibPath = os.path.join(cls.getObsDataPackageDir(), cls.policyName, 

418 datasetTypeName) 

419 

420 if os.path.exists(calibPath): 

421 return calibPath 

422 

423 return None 

424 

425 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None): 

426 """Write standardized curated calibration datasets for this specific 

427 dataset type from an obs data package. 

428 

429 Parameters 

430 ---------- 

431 butler : `lsst.daf.butler.Butler` 

432 Gen3 butler in which to put the calibrations. 

433 datasetType : `lsst.daf.butler.DatasetType` 

434 Dataset type to be put. 

435 run : `str`, optional 

436 Name of the run to use to override the default run associated 

437 with this Butler. 

438 

439 Notes 

440 ----- 

441 This method scans the location defined in the ``obsDataPackageDir`` 

442 class attribute for curated calibrations corresponding to the 

443 supplied dataset type. The directory name in the data package must 

444 match the name of the dataset type. They are assumed to use the 

445 standard layout and can be read by 

446 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

447 metadata. 

448 """ 

449 calibPath = self._getSpecificCuratedCalibrationPath(datasetType.name) 

450 if calibPath is None: 

451 return 

452 

453 # Register the dataset type 

454 butler.registry.registerDatasetType(datasetType) 

455 

456 # obs_base can't depend on pipe_tasks but concrete obs packages 

457 # can -- we therefore have to defer import 

458 from lsst.pipe.tasks.read_curated_calibs import read_all 

459 

460 camera = self.getCamera() 

461 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

462 dimensionRecords = [] 

463 datasetRecords = [] 

464 for det in calibsDict: 

465 times = sorted([k for k in calibsDict[det]]) 

466 calibs = [calibsDict[det][time] for time in times] 

467 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

468 times += [None] 

469 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

470 md = calib.getMetadata() 

471 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

472 dataId = DataCoordinate.standardize( 

473 universe=butler.registry.dimensions, 

474 instrument=self.getName(), 

475 calibration_label=calibrationLabel, 

476 detector=md["DETECTOR"], 

477 ) 

478 datasetRecords.append((calib, dataId)) 

479 dimensionRecords.append({ 

480 "instrument": self.getName(), 

481 "name": calibrationLabel, 

482 "timespan": Timespan(beginTime, endTime), 

483 }) 

484 

485 # Second loop actually does the inserts and filesystem writes. 

486 with butler.transaction(): 

487 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

488 # TODO: vectorize these puts, once butler APIs for that become 

489 # available. 

490 for calib, dataId in datasetRecords: 

491 butler.put(calib, datasetType, dataId, run=run) 

492 

493 @abstractmethod 

494 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

495 """Return a factory for creating Gen2->Gen3 data ID translators, 

496 specialized for this instrument. 

497 

498 Derived class implementations should generally call 

499 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

500 arguments, but are not required to (and may not be able to if their 

501 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

502 norm). 

503 

504 Returns 

505 ------- 

506 factory : `TranslatorFactory`. 

507 Factory for `Translator` objects. 

508 """ 

509 raise NotImplementedError("Must be implemented by derived classes.") 

510 

511 @classmethod 

512 def makeDefaultRawIngestRunName(cls) -> str: 

513 """Make the default instrument-specific run collection string for raw 

514 data ingest. 

515 

516 Returns 

517 ------- 

518 coll : `str` 

519 Run collection name to be used as the default for ingestion of 

520 raws. 

521 """ 

522 return cls.makeCollectionName("raw/all") 

523 

524 @classmethod 

525 def makeCollectionName(cls, label: str) -> str: 

526 """Get the instrument-specific collection string to use as derived 

527 from the supplied label. 

528 

529 Parameters 

530 ---------- 

531 label : `str` 

532 String to be combined with the instrument name to form a 

533 collection name. 

534 

535 Returns 

536 ------- 

537 name : `str` 

538 Collection name to use that includes the instrument name. 

539 """ 

540 return f"{cls.getName()}/{label}" 

541 

542 

543def makeExposureRecordFromObsInfo(obsInfo, universe): 

544 """Construct an exposure DimensionRecord from 

545 `astro_metadata_translator.ObservationInfo`. 

546 

547 Parameters 

548 ---------- 

549 obsInfo : `astro_metadata_translator.ObservationInfo` 

550 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

551 the exposure. 

552 universe : `DimensionUniverse` 

553 Set of all known dimensions. 

554 

555 Returns 

556 ------- 

557 record : `DimensionRecord` 

558 A record containing exposure metadata, suitable for insertion into 

559 a `Registry`. 

560 """ 

561 dimension = universe["exposure"] 

562 

563 ra, dec, sky_angle, zenith_angle = (None, None, None, None) 

564 if obsInfo.tracking_radec is not None: 

565 icrs = obsInfo.tracking_radec.icrs 

566 ra = icrs.ra.degree 

567 dec = icrs.dec.degree 

568 if obsInfo.boresight_rotation_coord == "sky": 

569 sky_angle = obsInfo.boresight_rotation_angle.degree 

570 if obsInfo.altaz_begin is not None: 

571 zenith_angle = obsInfo.altaz_begin.zen.degree 

572 

573 return dimension.RecordClass( 

574 instrument=obsInfo.instrument, 

575 id=obsInfo.exposure_id, 

576 name=obsInfo.observation_id, 

577 group_name=obsInfo.exposure_group, 

578 group_id=obsInfo.visit_id, 

579 datetime_begin=obsInfo.datetime_begin, 

580 datetime_end=obsInfo.datetime_end, 

581 exposure_time=obsInfo.exposure_time.to_value("s"), 

582 dark_time=obsInfo.dark_time.to_value("s"), 

583 observation_type=obsInfo.observation_type, 

584 physical_filter=obsInfo.physical_filter, 

585 science_program=obsInfo.science_program, 

586 target_name=obsInfo.object, 

587 tracking_ra=ra, 

588 tracking_dec=dec, 

589 sky_angle=sky_angle, 

590 zenith_angle=zenith_angle, 

591 ) 

592 

593 

594def addUnboundedCalibrationLabel(registry, instrumentName): 

595 """Add a special 'unbounded' calibration_label dimension entry for the 

596 given camera that is valid for any exposure. 

597 

598 If such an entry already exists, this function just returns a `DataId` 

599 for the existing entry. 

600 

601 Parameters 

602 ---------- 

603 registry : `Registry` 

604 Registry object in which to insert the dimension entry. 

605 instrumentName : `str` 

606 Name of the instrument this calibration label is associated with. 

607 

608 Returns 

609 ------- 

610 dataId : `DataId` 

611 New or existing data ID for the unbounded calibration. 

612 """ 

613 d = dict(instrument=instrumentName, calibration_label="unbounded") 

614 try: 

615 return registry.expandDataId(d) 

616 except LookupError: 

617 pass 

618 entry = d.copy() 

619 entry["timespan"] = Timespan(None, None) 

620 registry.insertDimensionData("calibration_label", entry) 

621 return registry.expandDataId(d) 

622 

623 

624def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

625 """Attempt to load versioned camera geometry from a butler, but fall back 

626 to obtaining a nominal camera from the `Instrument` class if that fails. 

627 

628 Parameters 

629 ---------- 

630 butler : `lsst.daf.butler.Butler` 

631 Butler instance to attempt to query for and load a ``camera`` dataset 

632 from. 

633 dataId : `dict` or `DataCoordinate` 

634 Data ID that identifies at least the ``instrument`` and ``exposure`` 

635 dimensions. 

636 collections : Any, optional 

637 Collections to be searched, overriding ``self.butler.collections``. 

638 Can be any of the types supported by the ``collections`` argument 

639 to butler construction. 

640 

641 Returns 

642 ------- 

643 camera : `lsst.afw.cameraGeom.Camera` 

644 Camera object. 

645 versioned : `bool` 

646 If `True`, the camera was obtained from the butler and should represent 

647 a versioned camera from a calibration repository. If `False`, no 

648 camera datasets were found, and the returned camera was produced by 

649 instantiating the appropriate `Instrument` class and calling 

650 `Instrument.getCamera`. 

651 """ 

652 if collections is None: 

653 collections = butler.collections 

654 # Registry would do data ID expansion internally if we didn't do it first, 

655 # but we might want an expanded data ID ourselves later, so we do it here 

656 # to ensure it only happens once. 

657 # This will also catch problems with the data ID not having keys we need. 

658 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

659 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

660 deduplicate=True)) 

661 if cameraRefs: 

662 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

663 return butler.getDirect(cameraRefs[0]), True 

664 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

665 return instrument.getCamera(), False