Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Tuple, TYPE_CHECKING 

29import astropy.time 

30 

31from lsst.afw.cameraGeom import Camera 

32from lsst.daf.butler import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate 

33from lsst.utils import getPackageDir, doImport 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from .gen2to3 import TranslatorFactory 

37 from lsst.daf.butler import Registry 

38 

39# To be a standard text curated calibration means that we use a 

40# standard definition for the corresponding DatasetType. 

41StandardCuratedCalibrationDatasetTypes = { 

42 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

43 "storageClass": "Defects"}, 

44 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

45 "storageClass": "QECurve"}, 

46} 

47 

48 

49class Instrument(metaclass=ABCMeta): 

50 """Base class for instrument-specific logic for the Gen3 Butler. 

51 

52 Concrete instrument subclasses should be directly constructable with no 

53 arguments. 

54 """ 

55 

56 configPaths = () 

57 """Paths to config files to read for specific Tasks. 

58 

59 The paths in this list should contain files of the form `task.py`, for 

60 each of the Tasks that requires special configuration. 

61 """ 

62 

63 policyName = None 

64 """Instrument specific name to use when locating a policy or configuration 

65 file in the file system.""" 

66 

67 obsDataPackage = None 

68 """Name of the package containing the text curated calibration files. 

69 Usually a obs _data package. If `None` no curated calibration files 

70 will be read. (`str`)""" 

71 

72 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

73 """The dataset types expected to be obtained from the obsDataPackage. 

74 These dataset types are all required to have standard definitions and 

75 must be known to the base class. Clearing this list will prevent 

76 any of these calibrations from being stored. If a dataset type is not 

77 known to a specific instrument it can still be included in this list 

78 since the data package is the source of truth. 

79 """ 

80 

81 @property 

82 @abstractmethod 

83 def filterDefinitions(self): 

84 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

85 for this instrument. 

86 """ 

87 return None 

88 

89 def __init__(self): 

90 self.filterDefinitions.reset() 

91 self.filterDefinitions.defineFilters() 

92 self._obsDataPackageDir = None 

93 

94 @classmethod 

95 @abstractmethod 

96 def getName(cls): 

97 """Return the short (dimension) name for this instrument. 

98 

99 This is not (in general) the same as the class name - it's what is used 

100 as the value of the "instrument" field in data IDs, and is usually an 

101 abbreviation of the full name. 

102 """ 

103 raise NotImplementedError() 

104 

105 @abstractmethod 

106 def getCamera(self): 

107 """Retrieve the cameraGeom representation of this instrument. 

108 

109 This is a temporary API that should go away once ``obs_`` packages have 

110 a standardized approach to writing versioned cameras to a Gen3 repo. 

111 """ 

112 raise NotImplementedError() 

113 

114 @abstractmethod 

115 def register(self, registry): 

116 """Insert instrument, physical_filter, and detector entries into a 

117 `Registry`. 

118 """ 

119 raise NotImplementedError() 

120 

121 @property 

122 def obsDataPackageDir(self): 

123 """The root of the obs package that provides specializations for 

124 this instrument (`str`). 

125 """ 

126 if self.obsDataPackage is None: 

127 return None 

128 if self._obsDataPackageDir is None: 

129 # Defer any problems with locating the package until 

130 # we need to find it. 

131 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

132 return self._obsDataPackageDir 

133 

134 @staticmethod 

135 def fromName(name: str, registry: Registry) -> Instrument: 

136 """Given an instrument name and a butler, retrieve a corresponding 

137 instantiated instrument object. 

138 

139 Parameters 

140 ---------- 

141 name : `str` 

142 Name of the instrument (must match the return value of `getName`). 

143 registry : `lsst.daf.butler.Registry` 

144 Butler registry to query to find the information. 

145 

146 Returns 

147 ------- 

148 instrument : `Instrument` 

149 An instance of the relevant `Instrument`. 

150 

151 Notes 

152 ----- 

153 The instrument must be registered in the corresponding butler. 

154 

155 Raises 

156 ------ 

157 LookupError 

158 Raised if the instrument is not known to the supplied registry. 

159 ModuleNotFoundError 

160 Raised if the class could not be imported. This could mean 

161 that the relevant obs package has not been setup. 

162 TypeError 

163 Raised if the class name retrieved is not a string. 

164 """ 

165 dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name})) 

166 cls = dimensions[0].records["instrument"].class_name 

167 if not isinstance(cls, str): 

168 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

169 instrument = doImport(cls) 

170 return instrument() 

171 

172 @staticmethod 

173 def importAll(registry: Registry) -> None: 

174 """Import all the instruments known to this registry. 

175 

176 This will ensure that all metadata translators have been registered. 

177 

178 Parameters 

179 ---------- 

180 registry : `lsst.daf.butler.Registry` 

181 Butler registry to query to find the information. 

182 

183 Notes 

184 ----- 

185 It is allowed for a particular instrument class to fail on import. 

186 This might simply indicate that a particular obs package has 

187 not been setup. 

188 """ 

189 dimensions = list(registry.queryDimensions("instrument")) 

190 for dim in dimensions: 

191 cls = dim.records["instrument"].class_name 

192 try: 

193 doImport(cls) 

194 except Exception: 

195 pass 

196 

197 def _registerFilters(self, registry): 

198 """Register the physical and abstract filter Dimension relationships. 

199 This should be called in the ``register`` implementation. 

200 

201 Parameters 

202 ---------- 

203 registry : `lsst.daf.butler.core.Registry` 

204 The registry to add dimensions to. 

205 """ 

206 for filter in self.filterDefinitions: 

207 # fix for undefined abstract filters causing trouble in the registry: 

208 if filter.abstract_filter is None: 

209 abstract_filter = filter.physical_filter 

210 else: 

211 abstract_filter = filter.abstract_filter 

212 

213 registry.insertDimensionData("physical_filter", 

214 {"instrument": self.getName(), 

215 "name": filter.physical_filter, 

216 "abstract_filter": abstract_filter 

217 }) 

218 

219 @abstractmethod 

220 def getRawFormatter(self, dataId): 

221 """Return the Formatter class that should be used to read a particular 

222 raw file. 

223 

224 Parameters 

225 ---------- 

226 dataId : `DataCoordinate` 

227 Dimension-based ID for the raw file or files being ingested. 

228 

229 Returns 

230 ------- 

231 formatter : `Formatter` class 

232 Class to be used that reads the file into an 

233 `lsst.afw.image.Exposure` instance. 

234 """ 

235 raise NotImplementedError() 

236 

237 def writeCuratedCalibrations(self, butler): 

238 """Write human-curated calibration Datasets to the given Butler with 

239 the appropriate validity ranges. 

240 

241 Parameters 

242 ---------- 

243 butler : `lsst.daf.butler.Butler` 

244 Butler to use to store these calibrations. 

245 

246 Notes 

247 ----- 

248 Expected to be called from subclasses. The base method calls 

249 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

250 """ 

251 self.writeCameraGeom(butler) 

252 self.writeStandardTextCuratedCalibrations(butler) 

253 

254 def applyConfigOverrides(self, name, config): 

255 """Apply instrument-specific overrides for a task config. 

256 

257 Parameters 

258 ---------- 

259 name : `str` 

260 Name of the object being configured; typically the _DefaultName 

261 of a Task. 

262 config : `lsst.pex.config.Config` 

263 Config instance to which overrides should be applied. 

264 """ 

265 for root in self.configPaths: 

266 path = os.path.join(root, f"{name}.py") 

267 if os.path.exists(path): 

268 config.load(path) 

269 

270 def writeCameraGeom(self, butler): 

271 """Write the default camera geometry to the butler repository 

272 with an infinite validity range. 

273 

274 Parameters 

275 ---------- 

276 butler : `lsst.daf.butler.Butler` 

277 Butler to receive these calibration datasets. 

278 """ 

279 

280 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

281 universe=butler.registry.dimensions) 

282 butler.registry.registerDatasetType(datasetType) 

283 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

284 camera = self.getCamera() 

285 butler.put(camera, datasetType, unboundedDataId) 

286 

287 def writeStandardTextCuratedCalibrations(self, butler): 

288 """Write the set of standardized curated text calibrations to 

289 the repository. 

290 

291 Parameters 

292 ---------- 

293 butler : `lsst.daf.butler.Butler` 

294 Butler to receive these calibration datasets. 

295 """ 

296 

297 for datasetTypeName in self.standardCuratedDatasetTypes: 

298 # We need to define the dataset types. 

299 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

300 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

301 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

302 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

303 datasetType = DatasetType(datasetTypeName, 

304 universe=butler.registry.dimensions, 

305 **definition) 

306 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType) 

307 

308 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType): 

309 """Write standardized curated calibration datasets for this specific 

310 dataset type from an obs data package. 

311 

312 Parameters 

313 ---------- 

314 butler : `lsst.daf.butler.Butler` 

315 Gen3 butler in which to put the calibrations. 

316 datasetType : `lsst.daf.butler.DatasetType` 

317 Dataset type to be put. 

318 

319 Notes 

320 ----- 

321 This method scans the location defined in the ``obsDataPackageDir`` 

322 class attribute for curated calibrations corresponding to the 

323 supplied dataset type. The directory name in the data package must 

324 match the name of the dataset type. They are assumed to use the 

325 standard layout and can be read by 

326 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

327 metadata. 

328 """ 

329 if self.obsDataPackageDir is None: 

330 # if there is no data package then there can't be datasets 

331 return 

332 

333 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

334 datasetType.name) 

335 

336 if not os.path.exists(calibPath): 

337 return 

338 

339 # Register the dataset type 

340 butler.registry.registerDatasetType(datasetType) 

341 

342 # obs_base can't depend on pipe_tasks but concrete obs packages 

343 # can -- we therefore have to defer import 

344 from lsst.pipe.tasks.read_curated_calibs import read_all 

345 

346 camera = self.getCamera() 

347 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

348 endOfTime = TIMESPAN_MAX 

349 dimensionRecords = [] 

350 datasetRecords = [] 

351 for det in calibsDict: 

352 times = sorted([k for k in calibsDict[det]]) 

353 calibs = [calibsDict[det][time] for time in times] 

354 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

355 times += [endOfTime] 

356 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

357 md = calib.getMetadata() 

358 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

359 dataId = DataCoordinate.standardize( 

360 universe=butler.registry.dimensions, 

361 instrument=self.getName(), 

362 calibration_label=calibrationLabel, 

363 detector=md["DETECTOR"], 

364 ) 

365 datasetRecords.append((calib, dataId)) 

366 dimensionRecords.append({ 

367 "instrument": self.getName(), 

368 "name": calibrationLabel, 

369 "datetime_begin": beginTime, 

370 "datetime_end": endTime, 

371 }) 

372 

373 # Second loop actually does the inserts and filesystem writes. 

374 with butler.transaction(): 

375 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

376 # TODO: vectorize these puts, once butler APIs for that become 

377 # available. 

378 for calib, dataId in datasetRecords: 

379 butler.put(calib, datasetType, dataId) 

380 

381 @abstractmethod 

382 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

383 """Return a factory for creating Gen2->Gen3 data ID translators, 

384 specialized for this instrument. 

385 

386 Derived class implementations should generally call 

387 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

388 arguments, but are not required to (and may not be able to if their 

389 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

390 norm). 

391 

392 Returns 

393 ------- 

394 factory : `TranslatorFactory`. 

395 Factory for `Translator` objects. 

396 """ 

397 raise NotImplementedError("Must be implemented by derived classes.") 

398 

399 

400def makeExposureRecordFromObsInfo(obsInfo, universe): 

401 """Construct an exposure DimensionRecord from 

402 `astro_metadata_translator.ObservationInfo`. 

403 

404 Parameters 

405 ---------- 

406 obsInfo : `astro_metadata_translator.ObservationInfo` 

407 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

408 the exposure. 

409 universe : `DimensionUniverse` 

410 Set of all known dimensions. 

411 

412 Returns 

413 ------- 

414 record : `DimensionRecord` 

415 A record containing exposure metadata, suitable for insertion into 

416 a `Registry`. 

417 """ 

418 dimension = universe["exposure"] 

419 return dimension.RecordClass.fromDict({ 

420 "instrument": obsInfo.instrument, 

421 "id": obsInfo.exposure_id, 

422 "name": obsInfo.observation_id, 

423 "group_name": obsInfo.exposure_group, 

424 "group_id": obsInfo.visit_id, 

425 "datetime_begin": obsInfo.datetime_begin, 

426 "datetime_end": obsInfo.datetime_end, 

427 "exposure_time": obsInfo.exposure_time.to_value("s"), 

428 "dark_time": obsInfo.dark_time.to_value("s"), 

429 "observation_type": obsInfo.observation_type, 

430 "physical_filter": obsInfo.physical_filter, 

431 }) 

432 

433 

434def addUnboundedCalibrationLabel(registry, instrumentName): 

435 """Add a special 'unbounded' calibration_label dimension entry for the 

436 given camera that is valid for any exposure. 

437 

438 If such an entry already exists, this function just returns a `DataId` 

439 for the existing entry. 

440 

441 Parameters 

442 ---------- 

443 registry : `Registry` 

444 Registry object in which to insert the dimension entry. 

445 instrumentName : `str` 

446 Name of the instrument this calibration label is associated with. 

447 

448 Returns 

449 ------- 

450 dataId : `DataId` 

451 New or existing data ID for the unbounded calibration. 

452 """ 

453 d = dict(instrument=instrumentName, calibration_label="unbounded") 

454 try: 

455 return registry.expandDataId(d) 

456 except LookupError: 

457 pass 

458 entry = d.copy() 

459 entry["datetime_begin"] = TIMESPAN_MIN 

460 entry["datetime_end"] = TIMESPAN_MAX 

461 registry.insertDimensionData("calibration_label", entry) 

462 return registry.expandDataId(d) 

463 

464 

465def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

466 """Attempt to load versioned camera geometry from a butler, but fall back 

467 to obtaining a nominal camera from the `Instrument` class if that fails. 

468 

469 Parameters 

470 ---------- 

471 butler : `lsst.daf.butler.Butler` 

472 Butler instance to attempt to query for and load a ``camera`` dataset 

473 from. 

474 dataId : `dict` or `DataCoordinate` 

475 Data ID that identifies at least the ``instrument`` and ``exposure`` 

476 dimensions. 

477 collections : Any, optional 

478 Collections to be searched, overriding ``self.butler.collections``. 

479 Can be any of the types supported by the ``collections`` argument 

480 to butler construction. 

481 

482 Returns 

483 ------- 

484 camera : `lsst.afw.cameraGeom.Camera` 

485 Camera object. 

486 versioned : `bool` 

487 If `True`, the camera was obtained from the butler and should represent 

488 a versioned camera from a calibration repository. If `False`, no 

489 camera datasets were found, and the returned camera was produced by 

490 instantiating the appropriate `Instrument` class and calling 

491 `Instrument.getCamera`. 

492 """ 

493 if collections is None: 

494 collections = butler.collections 

495 # Registry would do data ID expansion internally if we didn't do it first, 

496 # but we might want an expanded data ID ourselves later, so we do it here 

497 # to ensure it only happens once. 

498 # This will also catch problems with the data ID not having keys we need. 

499 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

500 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

501 deduplicate=True)) 

502 if cameraRefs: 

503 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

504 return butler.getDirect(cameraRefs[0]), True 

505 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

506 return instrument.getCamera(), False