Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("Translator", "KeyHandler", "CopyKeyHandler", "ConstantKeyHandler", 

23 "CalibKeyHandler", "AbstractToPhysicalFilterKeyHandler", "PhysicalToAbstractFilterKeyHandler", 

24 "makeCalibrationLabel") 

25 

26import itertools 

27from typing import Optional, Any, Dict, Tuple, FrozenSet, Iterable, List 

28from abc import ABCMeta, abstractmethod 

29 

30from lsst.log import Log 

31from lsst.skymap import BaseSkyMap 

32 

33 

34def makeCalibrationLabel(datasetTypeName: str, calibDate: str, ccd: Optional[int] = None, 

35 filter: Optional[str] = None) -> str: 

36 """Make a Gen3 calibration_label string corresponding to a Gen2 data ID. 

37 

38 Parameters 

39 ---------- 

40 datasetTypeName : `str` 

41 Name of the dataset type this calibration label identifies. 

42 calibDate : `str` 

43 Date string used in the Gen2 template. 

44 ccd : `int`, optional 

45 Detector ID used in the Gen2 template. 

46 filter : `str`, optional 

47 Filter used in the Gen2 template. 

48 

49 Returns 

50 ------- 

51 label : `str` 

52 Calibration label string. 

53 """ 

54 # TODO: this function is probably HSC-specific, but I don't know how other 

55 # obs calib registries behave so I don't know (yet) how to generalize it. 

56 elements = [datasetTypeName, calibDate] 

57 if ccd is not None: 

58 elements.append(f"{ccd:03d}") 

59 if filter is not None: 

60 elements.append(filter) 

61 return "gen2/{}".format("_".join(elements)) 

62 

63 

64class KeyHandler(metaclass=ABCMeta): 

65 """Base class for Translator helpers that each handle just one Gen3 Data 

66 ID key. 

67 

68 Parameters 

69 ---------- 

70 dimension : `str` 

71 Name of the Gen3 dimension (data ID key) populated by 

72 this handler (e.g. "visit" or "abstract_filter"). 

73 """ 

74 def __init__(self, dimension: str): 

75 self.dimension = dimension 

76 

77 __slots__ = ("dimension",) 

78 

79 def __str__(self): 

80 return f"{type(self).__name__}({self.dimension})" 

81 

82 def translate(self, gen2id: dict, gen3id: dict, 

83 skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

84 datasetTypeName: str): 

85 """Update a Gen3 data ID dict with a single key-value pair from a Gen2 

86 data ID. 

87 

88 This method is implemented by the base class and is not expected to 

89 be re-implemented by subclasses. 

90 

91 Parameters 

92 ---------- 

93 gen2id: `dict` 

94 Gen2 data ID from which to draw key-value pairs from. 

95 gen3id: `dict` 

96 Gen3 data ID to update in-place. 

97 skyMap: `BaseSkyMap`, optional 

98 SkyMap that defines the tracts and patches used in the Gen2 data 

99 ID, if any. 

100 skyMapName: `str` 

101 Name of the Gen3 skymap dimension that defines the tracts and 

102 patches used in the Gen3 data ID. 

103 datasetTypeName: `str` 

104 Name of the dataset type. 

105 """ 

106 gen3id[self.dimension] = self.extract(gen2id, skyMap=skyMap, skyMapName=skyMapName, 

107 datasetTypeName=datasetTypeName) 

108 

109 @abstractmethod 

110 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

111 datasetTypeName: str) -> Any: 

112 """Extract a Gen3 data ID value from a Gen2 data ID. 

113 

114 Parameters 

115 ---------- 

116 gen2id: `dict` 

117 Gen2 data ID from which to draw key-value pairs from. 

118 skyMap: `BaseSkyMap`, optional 

119 SkyMap that defines the tracts and patches used in the Gen2 data 

120 ID, if any. 

121 skyMapName: `str` 

122 Name of the Gen3 skymap dimension that defines the tracts and 

123 patches used in the Gen3 data ID. 

124 datasetTypeName: `str` 

125 Name of the dataset type. 

126 """ 

127 raise NotImplementedError() 

128 

129 

130class ConstantKeyHandler(KeyHandler): 

131 """A KeyHandler that adds a constant key-value pair to the Gen3 data ID. 

132 

133 Parameters 

134 ---------- 

135 dimension : `str` 

136 Name of the Gen3 dimension (data ID key) populated by 

137 this handler (e.g. "visit" or "abstract_filter"). 

138 value : `object` 

139 Data ID value. 

140 """ 

141 def __init__(self, dimension: str, value: Any): 

142 super().__init__(dimension) 

143 self.value = value 

144 

145 __slots__ = ("value",) 

146 

147 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

148 datasetTypeName: str) -> Any: 

149 # Docstring inherited from KeyHandler.extract. 

150 return self.value 

151 

152 

153class CopyKeyHandler(KeyHandler): 

154 """A KeyHandler that simply copies a value from a Gen3 data ID. 

155 

156 Parameters 

157 ---------- 

158 dimension : `str` 

159 Name of the Gen3 dimension produced by this handler. 

160 dtype : `type`, optional 

161 If not `None`, the type that values for this key must be an 

162 instance of. 

163 """ 

164 def __init__(self, dimension: str, gen2key: Optional[str] = None, 

165 dtype: Optional[type] = None): 

166 super().__init__(dimension) 

167 self.gen2key = gen2key if gen2key is not None else dimension 

168 self.dtype = dtype 

169 

170 __slots__ = ("gen2key", "dtype") 

171 

172 def __str__(self): 

173 return f"{type(self).__name__}({self.gen2key}, {self.dtype})" 

174 

175 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

176 datasetTypeName: str) -> Any: 

177 # Docstring inherited from KeyHandler.extract. 

178 r = gen2id[self.gen2key] 

179 if self.dtype is not None: 

180 try: 

181 r = self.dtype(r) 

182 except ValueError as err: 

183 raise TypeError( 

184 f"'{r}' is not a valid value for {self.dimension}; " 

185 f"expected {self.dtype.__name__}, got {type(r).__name__}." 

186 ) from err 

187 return r 

188 

189 

190class PatchKeyHandler(KeyHandler): 

191 """A KeyHandler for skymap patches. 

192 """ 

193 def __init__(self): 

194 super().__init__("patch") 

195 

196 __slots__ = () 

197 

198 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

199 datasetTypeName: str) -> Any: 

200 # Docstring inherited from KeyHandler.extract. 

201 tract = gen2id["tract"] 

202 tractInfo = skyMap[tract] 

203 x, y = gen2id["patch"].split(",") 

204 patchInfo = tractInfo[int(x), int(y)] 

205 return tractInfo.getSequentialPatchIndex(patchInfo) 

206 

207 

208class SkyMapKeyHandler(KeyHandler): 

209 """A KeyHandler for skymaps.""" 

210 def __init__(self): 

211 super().__init__("skymap") 

212 

213 __slots__ = () 

214 

215 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

216 datasetTypeName: str) -> Any: 

217 # Docstring inherited from KeyHandler.extract. 

218 return skyMapName 

219 

220 

221class CalibKeyHandler(KeyHandler): 

222 """A KeyHandler for master calibration datasets. 

223 """ 

224 __slots__ = ("ccdKey",) 

225 

226 def __init__(self, ccdKey="ccd"): 

227 self.ccdKey = ccdKey 

228 super().__init__("calibration_label") 

229 

230 def extract(self, gen2id: dict, skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

231 datasetTypeName: str) -> Any: 

232 # Docstring inherited from KeyHandler.extract. 

233 return makeCalibrationLabel(datasetTypeName, gen2id["calibDate"], 

234 ccd=gen2id.get(self.ccdKey), filter=gen2id.get("filter")) 

235 

236 

237class PhysicalToAbstractFilterKeyHandler(KeyHandler): 

238 """KeyHandler for gen2 ``filter`` keys that match ``physical_filter`` 

239 keys in gen3 but should be mapped to ``abstract_filter``. 

240 

241 Note that multiple physical filter can potentially map to one abstract 

242 filter, so be careful to only use this translator on obs packages where 

243 there is a one-to-one mapping. 

244 """ 

245 

246 __slots__ = ("_map",) 

247 

248 def __init__(self, filterDefinitions): 

249 super().__init__("abstract_filter") 

250 self._map = {d.physical_filter: d.abstract_filter for d in filterDefinitions 

251 if d.physical_filter is not None} 

252 

253 def extract(self, gen2id, *args, **kwargs): 

254 physical = gen2id["filter"] 

255 return self._map.get(physical, physical) 

256 

257 

258class AbstractToPhysicalFilterKeyHandler(KeyHandler): 

259 """KeyHandler for gen2 ``filter`` keys that match ``abstract_filter`` 

260 keys in gen3 but should be mapped to ``physical_filter``. 

261 

262 Note that one abstract filter can potentially map to multiple physical 

263 filters, so be careful to only use this translator on obs packages where 

264 there is a one-to-one mapping. 

265 """ 

266 

267 __slots__ = ("_map",) 

268 

269 def __init__(self, filterDefinitions): 

270 super().__init__("physical_filter") 

271 self._map = {d.abstract_filter: d.physical_filter for d in filterDefinitions 

272 if d.abstract_filter is not None} 

273 

274 def extract(self, gen2id, *args, **kwargs): 

275 abstract = gen2id["filter"] 

276 return self._map.get(abstract, abstract) 

277 

278 

279class Translator: 

280 """Callable object that translates Gen2 Data IDs to Gen3 Data IDs for a 

281 particular DatasetType. 

282 

283 Translators should usually be constructed via the `makeMatching` method. 

284 

285 Parameters 

286 ---------- 

287 handlers : `list` 

288 A list of KeyHandlers this Translator should use. 

289 skyMap : `BaseSkyMap`, optional 

290 SkyMap instance used to define any tract or patch Dimensions. 

291 skyMapName : `str` 

292 Gen3 SkyMap Dimension name to be associated with any tract or patch 

293 Dimensions. 

294 datasetTypeName : `str` 

295 Name of the dataset type whose data IDs this translator handles. 

296 """ 

297 def __init__(self, handlers: List[KeyHandler], skyMap: Optional[BaseSkyMap], skyMapName: Optional[str], 

298 datasetTypeName: str): 

299 self.handlers = handlers 

300 self.skyMap = skyMap 

301 self.skyMapName = skyMapName 

302 self.datasetTypeName = datasetTypeName 

303 

304 __slots__ = ("handlers", "skyMap", "skyMapName", "datasetTypeName") 

305 

306 # Rules used to match Handlers when constring a Translator. 

307 # outer key is instrument name, or None for any 

308 # inner key is DatasetType name, or None for any 

309 # values are 3-tuples of (frozenset(gen2keys), handler, consume) 

310 _rules: Dict[ 

311 Optional[str], 

312 Dict[ 

313 Optional[str], 

314 Tuple[FrozenSet[str], KeyHandler, bool] 

315 ] 

316 ] = { 

317 None: { 

318 None: [] 

319 } 

320 } 

321 

322 def __str__(self): 

323 hstr = ",".join(str(h) for h in self.handlers) 

324 return f"{type(self).__name__}(dtype={self.datasetTypeName}, handlers=[{hstr}])" 

325 

326 @classmethod 

327 def addRule(cls, handler: KeyHandler, instrument: Optional[str] = None, 

328 datasetTypeName: Optional[str] = None, gen2keys: Iterable[str] = (), 

329 consume: bool = True): 

330 """Add a KeyHandler and an associated matching rule. 

331 

332 Parameters 

333 ---------- 

334 handler : `KeyHandler` 

335 A KeyHandler instance to add to a Translator when this rule 

336 matches. 

337 instrument : `str` 

338 Gen3 instrument name the Gen2 repository must be associated with 

339 for this rule to match, or None to match any instrument. 

340 datasetTypeName : `str` 

341 Name of the DatasetType this rule matches, or None to match any 

342 DatasetType. 

343 gen2Keys : sequence 

344 Sequence of Gen2 data ID keys that must all be present for this 

345 rule to match. 

346 consume : `bool` or `tuple` 

347 If True (default), remove all entries in gen2keys from the set of 

348 keys being matched to in order to prevent less-specific handlers 

349 from matching them. 

350 May also be a `tuple` listing only the keys to consume. 

351 """ 

352 # Ensure consume is always a frozenset, so we can process it uniformly 

353 # from here on. 

354 if consume is True: 

355 consume = frozenset(gen2keys) 

356 elif consume: 356 ↛ 357line 356 didn't jump to line 357, because the condition on line 356 was never true

357 consume = frozenset(consume) 

358 else: 

359 consume = frozenset() 

360 # find the rules for this instrument, or if we haven't seen it before, 

361 # add a nested dictionary that matches any DatasetType name and then 

362 # append this rule. 

363 rulesForInstrument = cls._rules.setdefault(instrument, {None: []}) 

364 rulesForInstrumentAndDatasetType = rulesForInstrument.setdefault(datasetTypeName, []) 

365 rulesForInstrumentAndDatasetType.append((frozenset(gen2keys), handler, consume)) 

366 

367 @classmethod 

368 def makeMatching(cls, datasetTypeName: str, gen2keys: Dict[str, type], instrument: Optional[str] = None, 

369 skyMap: Optional[BaseSkyMap] = None, skyMapName: Optional[str] = None): 

370 """Construct a Translator appropriate for instances of the given 

371 dataset. 

372 

373 Parameters 

374 ---------- 

375 datasetTypeName : `str` 

376 Name of the dataset type. 

377 gen2keys: `dict` 

378 Keys of a Gen2 data ID for this dataset. 

379 instrument: `str`, optional 

380 Name of the Gen3 instrument dimension for translated data IDs. 

381 skyMap: `~lsst.skymap.BaseSkyMap`, optional 

382 The skymap instance that defines any tract/patch data IDs. 

383 `~lsst.skymap.BaseSkyMap` instances. 

384 skyMapName : `str`, optional 

385 Gen3 SkyMap Dimension name to be associated with any tract or patch 

386 Dimensions. 

387 

388 Returns 

389 ------- 

390 translator : `Translator` 

391 A translator whose translate() method can be used to transform Gen2 

392 data IDs to Gen3 dataIds. 

393 """ 

394 if instrument is not None: 

395 rulesForInstrument = cls._rules.get(instrument, {None: []}) 

396 else: 

397 rulesForInstrument = {None: []} 

398 rulesForAnyInstrument = cls._rules[None] 

399 candidateRules = itertools.chain( 

400 rulesForInstrument.get(datasetTypeName, []), # this instrument, this DatasetType 

401 rulesForInstrument[None], # this instrument, any DatasetType 

402 rulesForAnyInstrument.get(datasetTypeName, []), # any instrument, this DatasetType 

403 rulesForAnyInstrument[None], # any instrument, any DatasetType 

404 ) 

405 matchedHandlers = [] 

406 targetKeys = set(gen2keys) 

407 for ruleKeys, ruleHandlers, consume in candidateRules: 

408 if ruleKeys.issubset(targetKeys): 

409 matchedHandlers.append(ruleHandlers) 

410 targetKeys -= consume 

411 return Translator(matchedHandlers, skyMap=skyMap, skyMapName=skyMapName, 

412 datasetTypeName=datasetTypeName) 

413 

414 def __call__(self, gen2id: Dict[str, Any], *, partial: bool = False, log: Optional[Log] = None): 

415 """Return a Gen3 data ID that corresponds to the given Gen2 data ID. 

416 """ 

417 gen3id = {} 

418 for handler in self.handlers: 

419 try: 

420 handler.translate(gen2id, gen3id, skyMap=self.skyMap, skyMapName=self.skyMapName, 

421 datasetTypeName=self.datasetTypeName) 

422 except KeyError: 

423 if partial: 

424 if log is not None: 

425 log.debug("Failed to translate %s from %s.", handler.dimension, gen2id) 

426 continue 

427 else: 

428 raise 

429 return gen3id 

430 

431 @property 

432 def dimensionNames(self): 

433 """The names of the dimensions populated by this Translator 

434 (`frozenset`). 

435 """ 

436 return frozenset(h.dimension for h in self.handlers) 

437 

438 

439# Add "skymap" to Gen3 ID if Gen2 ID has a "tract" key. 

440Translator.addRule(SkyMapKeyHandler(), gen2keys=("tract",), consume=False) 

441 

442# Add "skymap" to Gen3 ID if DatasetType is one of a few specific ones 

443for coaddName in ("deep", "goodSeeing", "psfMatched", "dcr"): 

444 Translator.addRule(SkyMapKeyHandler(), datasetTypeName=f"{coaddName}Coadd_skyMap") 

445 

446# Translate Gen2 str patch IDs to Gen3 sequential integers. 

447Translator.addRule(PatchKeyHandler(), gen2keys=("patch",)) 

448 

449# Copy Gen2 "tract" to Gen3 "tract". 

450Translator.addRule(CopyKeyHandler("tract", dtype=int), gen2keys=("tract",)) 

451 

452# Add valid_first, valid_last to instrument-level transmission/ datasets; 

453# these are considered calibration products in Gen3. 

454for datasetTypeName in ("transmission_sensor", "transmission_optics", "transmission_filter"): 

455 Translator.addRule(ConstantKeyHandler("calibration_label", "unbounded"), 

456 datasetTypeName=datasetTypeName) 

457 

458# Translate Gen2 pixel_id to Gen3 skypix. 

459# TODO: For now, we just assume that the refcat indexer uses htm7, since that's 

460# what we have generated most of our refcats at. 

461Translator.addRule(CopyKeyHandler("htm7", gen2key="pixel_id", dtype=int), gen2keys=("pixel_id",))