Coverage for python/lsst/pipe/base/_instrument.py: 33%

104 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-08 14:36 -0800

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument",) 

25 

26import datetime 

27import os.path 

28from abc import ABCMeta, abstractmethod 

29from typing import TYPE_CHECKING, Optional, Sequence, Type, Union 

30 

31from lsst.daf.butler import DataId, Formatter 

32from lsst.daf.butler.registry import DataIdError 

33from lsst.utils import doImportType 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from lsst.daf.butler import Registry 

37 from lsst.pex.config import Config 

38 

39 

40class Instrument(metaclass=ABCMeta): 

41 """Base class for instrument-specific logic for the Gen3 Butler. 

42 

43 Parameters 

44 ---------- 

45 collection_prefix : `str`, optional 

46 Prefix for collection names to use instead of the intrument's own name. 

47 This is primarily for use in simulated-data repositories, where the 

48 instrument name may not be necessary and/or sufficient to distinguish 

49 between collections. 

50 

51 Notes 

52 ----- 

53 Concrete instrument subclasses must have the same construction signature as 

54 the base class. 

55 """ 

56 

57 configPaths: Sequence[str] = () 

58 """Paths to config files to read for specific Tasks. 

59 

60 The paths in this list should contain files of the form `task.py`, for 

61 each of the Tasks that requires special configuration. 

62 """ 

63 

64 policyName: Optional[str] = None 

65 """Instrument specific name to use when locating a policy or configuration 

66 file in the file system.""" 

67 

68 def __init__(self, collection_prefix: Optional[str] = None): 

69 if collection_prefix is None: 

70 collection_prefix = self.getName() 

71 self.collection_prefix = collection_prefix 

72 

73 @classmethod 

74 @abstractmethod 

75 def getName(cls) -> str: 

76 """Return the short (dimension) name for this instrument. 

77 

78 This is not (in general) the same as the class name - it's what is used 

79 as the value of the "instrument" field in data IDs, and is usually an 

80 abbreviation of the full name. 

81 """ 

82 raise NotImplementedError() 

83 

84 @abstractmethod 

85 def register(self, registry: Registry, *, update: bool = False) -> None: 

86 """Insert instrument, and other relevant records into `Registry`. 

87 

88 Parameters 

89 ---------- 

90 registry : `lsst.daf.butler.Registry` 

91 Registry client for the data repository to modify. 

92 update : `bool`, optional 

93 If `True` (`False` is default), update existing records if they 

94 differ from the new ones. 

95 

96 Raises 

97 ------ 

98 lsst.daf.butler.registry.ConflictingDefinitionError 

99 Raised if any existing record has the same key but a different 

100 definition as one being registered. 

101 

102 Notes 

103 ----- 

104 New records can always be added by calling this method multiple times, 

105 as long as no existing records have changed (if existing records have 

106 changed, ``update=True`` must be used). Old records can never be 

107 removed by this method. 

108 

109 Implementations should guarantee that registration is atomic (the 

110 registry should not be modified if any error occurs) and idempotent at 

111 the level of individual dimension entries; new detectors and filters 

112 should be added, but changes to any existing record should not be. 

113 This can generally be achieved via a block like 

114 

115 .. code-block:: python 

116 

117 with registry.transaction(): 

118 registry.syncDimensionData("instrument", ...) 

119 registry.syncDimensionData("detector", ...) 

120 self.registerFilters(registry) 

121 

122 """ 

123 raise NotImplementedError() 

124 

125 @staticmethod 

126 def fromName(name: str, registry: Registry, collection_prefix: Optional[str] = None) -> Instrument: 

127 """Given an instrument name and a butler registry, retrieve a 

128 corresponding instantiated instrument object. 

129 

130 Parameters 

131 ---------- 

132 name : `str` 

133 Name of the instrument (must match the return value of `getName`). 

134 registry : `lsst.daf.butler.Registry` 

135 Butler registry to query to find the information. 

136 collection_prefix : `str`, optional 

137 Prefix for collection names to use instead of the intrument's own 

138 name. This is primarily for use in simulated-data repositories, 

139 where the instrument name may not be necessary and/or sufficient to 

140 distinguish between collections. 

141 

142 Returns 

143 ------- 

144 instrument : `Instrument` 

145 An instance of the relevant `Instrument`. 

146 

147 Notes 

148 ----- 

149 The instrument must be registered in the corresponding butler. 

150 

151 Raises 

152 ------ 

153 LookupError 

154 Raised if the instrument is not known to the supplied registry. 

155 ModuleNotFoundError 

156 Raised if the class could not be imported. This could mean 

157 that the relevant obs package has not been setup. 

158 TypeError 

159 Raised if the class name retrieved is not a string or the imported 

160 symbol is not an `Instrument` subclass. 

161 """ 

162 try: 

163 records = list(registry.queryDimensionRecords("instrument", instrument=name)) 

164 except DataIdError: 

165 records = None 

166 if not records: 

167 raise LookupError(f"No registered instrument with name '{name}'.") 

168 cls_name = records[0].class_name 

169 if not isinstance(cls_name, str): 

170 raise TypeError( 

171 f"Unexpected class name retrieved from {name} instrument dimension (got {cls_name})" 

172 ) 

173 instrument_cls: type = doImportType(cls_name) 

174 if not issubclass(instrument_cls, Instrument): 

175 raise TypeError( 

176 f"{instrument_cls!r}, obtained from importing {cls_name}, is not an Instrument subclass." 

177 ) 

178 return instrument_cls(collection_prefix=collection_prefix) 

179 

180 @staticmethod 

181 def from_string( 

182 name: str, registry: Optional[Registry] = None, collection_prefix: Optional[str] = None 

183 ) -> Instrument: 

184 """Return an instance from the short name or class name. 

185 

186 If the instrument name is not qualified (does not contain a '.') and a 

187 butler registry is provided, this will attempt to load the instrument 

188 using `Instrument.fromName()`. Otherwise the instrument will be 

189 imported and instantiated. 

190 

191 Parameters 

192 ---------- 

193 name : `str` 

194 The name or fully-qualified class name of an instrument. 

195 registry : `lsst.daf.butler.Registry`, optional 

196 Butler registry to query to find information about the instrument, 

197 by default `None`. 

198 collection_prefix : `str`, optional 

199 Prefix for collection names to use instead of the intrument's own 

200 name. This is primarily for use in simulated-data repositories, 

201 where the instrument name may not be necessary and/or sufficient 

202 to distinguish between collections. 

203 

204 Returns 

205 ------- 

206 instrument : `Instrument` 

207 The instantiated instrument. 

208 

209 Raises 

210 ------ 

211 RuntimeError 

212 Raised if the instrument can not be imported, instantiated, or 

213 obtained from the registry. 

214 TypeError 

215 Raised if the instrument is not a subclass of 

216 `~lsst.pipe.base.Instrument`. 

217 

218 See Also 

219 -------- 

220 Instrument.fromName 

221 """ 

222 if "." not in name and registry is not None: 

223 try: 

224 instr = Instrument.fromName(name, registry, collection_prefix=collection_prefix) 

225 except Exception as err: 

226 raise RuntimeError( 

227 f"Could not get instrument from name: {name}. Failed with exception: {err}" 

228 ) from err 

229 else: 

230 try: 

231 instr_class = doImportType(name) 

232 except Exception as err: 

233 raise RuntimeError( 

234 f"Could not import instrument: {name}. Failed with exception: {err}" 

235 ) from err 

236 instr = instr_class(collection_prefix=collection_prefix) 

237 if not isinstance(instr, Instrument): 

238 raise TypeError(f"{name} is not an Instrument subclass.") 

239 return instr 

240 

241 @staticmethod 

242 def importAll(registry: Registry) -> None: 

243 """Import all the instruments known to this registry. 

244 

245 This will ensure that all metadata translators have been registered. 

246 

247 Parameters 

248 ---------- 

249 registry : `lsst.daf.butler.Registry` 

250 Butler registry to query to find the information. 

251 

252 Notes 

253 ----- 

254 It is allowed for a particular instrument class to fail on import. 

255 This might simply indicate that a particular obs package has 

256 not been setup. 

257 """ 

258 records = list(registry.queryDimensionRecords("instrument")) 

259 for record in records: 

260 cls = record.class_name 

261 try: 

262 doImportType(cls) 

263 except Exception: 

264 pass 

265 

266 @abstractmethod 

267 def getRawFormatter(self, dataId: DataId) -> Type[Formatter]: 

268 """Return the Formatter class that should be used to read a particular 

269 raw file. 

270 

271 Parameters 

272 ---------- 

273 dataId : `DataId` 

274 Dimension-based ID for the raw file or files being ingested. 

275 

276 Returns 

277 ------- 

278 formatter : `lsst.daf.butler.Formatter` class 

279 Class to be used that reads the file into the correct 

280 Python object for the raw data. 

281 """ 

282 raise NotImplementedError() 

283 

284 def applyConfigOverrides(self, name: str, config: Config) -> None: 

285 """Apply instrument-specific overrides for a task config. 

286 

287 Parameters 

288 ---------- 

289 name : `str` 

290 Name of the object being configured; typically the _DefaultName 

291 of a Task. 

292 config : `lsst.pex.config.Config` 

293 Config instance to which overrides should be applied. 

294 """ 

295 for root in self.configPaths: 

296 path = os.path.join(root, f"{name}.py") 

297 if os.path.exists(path): 

298 config.load(path) 

299 

300 @staticmethod 

301 def formatCollectionTimestamp(timestamp: Union[str, datetime.datetime]) -> str: 

302 """Format a timestamp for use in a collection name. 

303 

304 Parameters 

305 ---------- 

306 timestamp : `str` or `datetime.datetime` 

307 Timestamp to format. May be a date or datetime string in extended 

308 ISO format (assumed UTC), with or without a timezone specifier, a 

309 datetime string in basic ISO format with a timezone specifier, a 

310 naive `datetime.datetime` instance (assumed UTC) or a 

311 timezone-aware `datetime.datetime` instance (converted to UTC). 

312 This is intended to cover all forms that string ``CALIBDATE`` 

313 metadata values have taken in the past, as well as the format this 

314 method itself writes out (to enable round-tripping). 

315 

316 Returns 

317 ------- 

318 formatted : `str` 

319 Standardized string form for the timestamp. 

320 """ 

321 if isinstance(timestamp, str): 

322 if "-" in timestamp: 

323 # extended ISO format, with - and : delimiters 

324 timestamp = datetime.datetime.fromisoformat(timestamp) 

325 else: 

326 # basic ISO format, with no delimiters (what this method 

327 # returns) 

328 timestamp = datetime.datetime.strptime(timestamp, "%Y%m%dT%H%M%S%z") 

329 if not isinstance(timestamp, datetime.datetime): 

330 raise TypeError(f"Unexpected date/time object: {timestamp!r}.") 

331 if timestamp.tzinfo is not None: 

332 timestamp = timestamp.astimezone(datetime.timezone.utc) 

333 return f"{timestamp:%Y%m%dT%H%M%S}Z" 

334 

335 @staticmethod 

336 def makeCollectionTimestamp() -> str: 

337 """Create a timestamp string for use in a collection name from the 

338 current time. 

339 

340 Returns 

341 ------- 

342 formatted : `str` 

343 Standardized string form of the current time. 

344 """ 

345 return Instrument.formatCollectionTimestamp(datetime.datetime.now(tz=datetime.timezone.utc)) 

346 

347 def makeDefaultRawIngestRunName(self) -> str: 

348 """Make the default instrument-specific run collection string for raw 

349 data ingest. 

350 

351 Returns 

352 ------- 

353 coll : `str` 

354 Run collection name to be used as the default for ingestion of 

355 raws. 

356 """ 

357 return self.makeCollectionName("raw", "all") 

358 

359 def makeUnboundedCalibrationRunName(self, *labels: str) -> str: 

360 """Make a RUN collection name appropriate for inserting calibration 

361 datasets whose validity ranges are unbounded. 

362 

363 Parameters 

364 ---------- 

365 *labels : `str` 

366 Extra strings to be included in the base name, using the default 

367 delimiter for collection names. Usually this is the name of the 

368 ticket on which the calibration collection is being created. 

369 

370 Returns 

371 ------- 

372 name : `str` 

373 Run collection name. 

374 """ 

375 return self.makeCollectionName("calib", *labels, "unbounded") 

376 

377 def makeCuratedCalibrationRunName(self, calibDate: str, *labels: str) -> str: 

378 """Make a RUN collection name appropriate for inserting curated 

379 calibration datasets with the given ``CALIBDATE`` metadata value. 

380 

381 Parameters 

382 ---------- 

383 calibDate : `str` 

384 The ``CALIBDATE`` metadata value. 

385 *labels : `str` 

386 Strings to be included in the collection name (before 

387 ``calibDate``, but after all other terms), using the default 

388 delimiter for collection names. Usually this is the name of the 

389 ticket on which the calibration collection is being created. 

390 

391 Returns 

392 ------- 

393 name : `str` 

394 Run collection name. 

395 """ 

396 return self.makeCollectionName("calib", *labels, "curated", self.formatCollectionTimestamp(calibDate)) 

397 

398 def makeCalibrationCollectionName(self, *labels: str) -> str: 

399 """Make a CALIBRATION collection name appropriate for associating 

400 calibration datasets with validity ranges. 

401 

402 Parameters 

403 ---------- 

404 *labels : `str` 

405 Strings to be appended to the base name, using the default 

406 delimiter for collection names. Usually this is the name of the 

407 ticket on which the calibration collection is being created. 

408 

409 Returns 

410 ------- 

411 name : `str` 

412 Calibration collection name. 

413 """ 

414 return self.makeCollectionName("calib", *labels) 

415 

416 @staticmethod 

417 def makeRefCatCollectionName(*labels: str) -> str: 

418 """Return a global (not instrument-specific) name for a collection that 

419 holds reference catalogs. 

420 

421 With no arguments, this returns the name of the collection that holds 

422 all reference catalogs (usually a ``CHAINED`` collection, at least in 

423 long-lived repos that may contain more than one reference catalog). 

424 

425 Parameters 

426 ---------- 

427 *labels : `str` 

428 Strings to be added to the global collection name, in order to 

429 define a collection name for one or more reference catalogs being 

430 ingested at the same time. 

431 

432 Returns 

433 ------- 

434 name : `str` 

435 Collection name. 

436 

437 Notes 

438 ----- 

439 This is a ``staticmethod``, not a ``classmethod``, because it should 

440 be the same for all instruments. 

441 """ 

442 return "/".join(("refcats",) + labels) 

443 

444 def makeUmbrellaCollectionName(self) -> str: 

445 """Return the name of the umbrella ``CHAINED`` collection for this 

446 instrument that combines all standard recommended input collections. 

447 

448 This method should almost never be overridden by derived classes. 

449 

450 Returns 

451 ------- 

452 name : `str` 

453 Name for the umbrella collection. 

454 """ 

455 return self.makeCollectionName("defaults") 

456 

457 def makeCollectionName(self, *labels: str) -> str: 

458 """Get the instrument-specific collection string to use as derived 

459 from the supplied labels. 

460 

461 Parameters 

462 ---------- 

463 *labels : `str` 

464 Strings to be combined with the instrument name to form a 

465 collection name. 

466 

467 Returns 

468 ------- 

469 name : `str` 

470 Collection name to use that includes the instrument's recommended 

471 prefix. 

472 """ 

473 return "/".join((self.collection_prefix,) + labels)