Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning. 

24 

25import os 

26from functools import lru_cache 

27import lsst.log 

28import lsst.geom 

29import lsst.utils as utils 

30import lsst.afw.image as afwImage 

31from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo 

32import lsst.obs.base.yamlCamera as yamlCamera 

33import lsst.daf.persistence as dafPersist 

34from .translators import LsstCamTranslator 

35from ._fitsHeader import readRawFitsHeader 

36from ._instrument import LsstCam 

37 

38from .filters import LSSTCAM_FILTER_DEFINITIONS 

39from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble 

40 

41__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"] 

42 

43 

44class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo): 

45 """Make a VisitInfo from the FITS header of a raw image.""" 

46 

47 

48class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo): 

49 metadataTranslator = LsstCamTranslator 

50 

51 

52def assemble_raw(dataId, componentInfo, cls): 

53 """Called by the butler to construct the composite type "raw". 

54 

55 Note that we still need to define "_raw" and copy various fields over. 

56 

57 Parameters 

58 ---------- 

59 dataId : `lsst.daf.persistence.dataId.DataId` 

60 The data ID. 

61 componentInfo : `dict` 

62 dict containing the components, as defined by the composite definition 

63 in the mapper policy. 

64 cls : 'object' 

65 Unused. 

66 

67 Returns 

68 ------- 

69 exposure : `lsst.afw.image.Exposure` 

70 The assembled exposure. 

71 """ 

72 

73 ampExps = componentInfo['raw_amp'].obj 

74 exposure = fixAmpsAndAssemble(ampExps, str(dataId)) 

75 md = componentInfo['raw_hdu'].obj 

76 exposure.setMetadata(md) 

77 

78 if not attachRawWcsFromBoresight(exposure): 

79 logger = lsst.log.Log.getLogger("LsstCamMapper") 

80 logger.warn("Unable to set WCS for %s from header as RA/Dec/Angle are unavailable", dataId) 

81 

82 return exposure 

83 

84 

85class LsstCamBaseMapper(CameraMapper): 

86 """The Base Mapper for all LSST-style instruments. 

87 """ 

88 

89 packageName = 'obs_lsst' 

90 _cameraName = "lsstCam" 

91 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence 

92 # 

93 # do not set MakeRawVisitInfoClass or translatorClass to anything other 

94 # than None! 

95 # 

96 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know 

97 # its mapper and cannot use mapper.makeRawVisitInfo() 

98 # 

99 MakeRawVisitInfoClass = None 

100 translatorClass = None 

101 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

102 

103 def __init__(self, inputPolicy=None, **kwargs): 

104 # 

105 # Merge the list of .yaml files 

106 # 

107 policy = None 

108 for yamlFile in self.yamlFileList: 

109 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy") 

110 npolicy = dafPersist.Policy(policyFile) 

111 

112 if policy is None: 

113 policy = npolicy 

114 else: 

115 policy.merge(npolicy) 

116 # 

117 # Look for the calibrations root "root/CALIB" if not supplied 

118 # 

119 if kwargs.get('root', None) and not kwargs.get('calibRoot', None): 

120 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

121 if "repositoryCfg" in kwargs: 

122 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

123 hasattr(cfg, "root")] 

124 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

125 for calibRoot in calibSearch: 

126 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

127 kwargs['calibRoot'] = calibRoot 

128 break 

129 if not kwargs.get('calibRoot', None): 

130 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory") 

131 

132 super().__init__(policy, os.path.dirname(policyFile), **kwargs) 

133 # 

134 # The composite objects don't seem to set these 

135 # 

136 for d in (self.mappings, self.exposures): 

137 d['raw'] = d['_raw'] 

138 

139 self.filterDefinitions.reset() 

140 self.filterDefinitions.defineFilters() 

141 

142 LsstCamMapper._nbit_tract = 16 

143 LsstCamMapper._nbit_patch = 5 

144 LsstCamMapper._nbit_filter = 7 

145 

146 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch 

147 + LsstCamMapper._nbit_filter) 

148 

149 if len(afwImage.Filter.getNames()) >= 2**LsstCamMapper._nbit_filter: 

150 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

151 LsstCamMapper._nbit_filter) 

152 

153 @classmethod 

154 def getCameraName(cls): 

155 return cls._cameraName 

156 

157 @classmethod 

158 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None): 

159 """Make a camera describing the camera geometry. 

160 

161 policy : ignored 

162 repositoryDir : ignored 

163 cameraYamlFile : `str` 

164 The full path to a yaml file to be passed to `yamlCamera.makeCamera` 

165 

166 Returns 

167 ------- 

168 camera : `lsst.afw.cameraGeom.Camera` 

169 Camera geometry. 

170 """ 

171 return cls._makeYamlCamera(cameraYamlFile=cameraYamlFile) 

172 

173 @classmethod 

174 @lru_cache(maxsize=10) 

175 def _makeYamlCamera(cls, cameraYamlFile=None): 

176 """Helper function for _makeCamera that can be cached. 

177 """ 

178 if not cameraYamlFile: 

179 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy", 

180 ("%s.yaml" % cls.getCameraName())) 

181 

182 return yamlCamera.makeCamera(cameraYamlFile) 

183 

184 def _getRegistryValue(self, dataId, k): 

185 """Return a value from a dataId, or look it up in the registry if it 

186 isn't present.""" 

187 if k in dataId: 

188 return dataId[k] 

189 else: 

190 dataType = "bias" if "taiObs" in dataId else "raw" 

191 

192 try: 

193 return self.queryMetadata(dataType, [k], dataId)[0][0] 

194 except IndexError: 

195 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" % 

196 (k, dataType, dataId)) 

197 

198 def _extractDetectorName(self, dataId): 

199 if "channel" in dataId: # they specified a channel 

200 dataId = dataId.copy() 

201 del dataId["channel"] # Do not include in query 

202 raftName = self._getRegistryValue(dataId, "raftName") 

203 detectorName = self._getRegistryValue(dataId, "detectorName") 

204 

205 return "%s_%s" % (raftName, detectorName) 

206 

207 def _computeCcdExposureId(self, dataId): 

208 """Compute the 64-bit (long) identifier for a CCD exposure. 

209 

210 Parameters 

211 ---------- 

212 dataId : `dict` 

213 Data identifier including dayObs and seqNum. 

214 

215 Returns 

216 ------- 

217 id : `int` 

218 Integer identifier for a CCD exposure. 

219 """ 

220 try: 

221 visit = self._getRegistryValue(dataId, "visit") 

222 except Exception: 

223 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}") 

224 

225 if "detector" in dataId: 

226 detector = dataId["detector"] 

227 else: 

228 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'], 

229 dataId['detectorName']) 

230 

231 return self.translatorClass.compute_detector_exposure_id(visit, detector) 

232 

233 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

234 return self._computeCcdExposureId(dataId) 

235 

236 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

237 """How many bits are required for the maximum exposure ID""" 

238 # 52 for "C" controller and 51 for "O" 

239 return 52 # max detector_exposure_id ~ 3050121299999250 

240 

241 def _computeCoaddExposureId(self, dataId, singleFilter): 

242 """Compute the 64-bit (long) identifier for a coadd. 

243 

244 Parameters 

245 ---------- 

246 dataId : `dict` 

247 Data identifier with tract and patch. 

248 singleFilter : `bool` 

249 True means the desired ID is for a single-filter coadd, in which 

250 case ``dataId`` must contain filter. 

251 """ 

252 

253 tract = int(dataId['tract']) 

254 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract: 

255 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract)) 

256 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

257 for p in (patchX, patchY): 

258 if p < 0 or p >= 2**LsstCamMapper._nbit_patch: 

259 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch) 

260 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY 

261 if singleFilter: 

262 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

263 return oid 

264 

265 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

266 """The number of bits used up for patch ID bits.""" 

267 return 64 - LsstCamMapper._nbit_id 

268 

269 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

270 return self._computeCoaddExposureId(dataId, True) 

271 

272 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

273 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

274 

275 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

276 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

277 

278 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

279 """The number of bits used up for patch ID bits.""" 

280 return 64 - LsstCamMapper._nbit_id 

281 

282 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

283 return self._computeCoaddExposureId(dataId, False) 

284 

285 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

286 """The number of bits used up for patch ID bits.""" 

287 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

288 

289 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

290 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

291 

292 def query_raw_amp(self, format, dataId): 

293 """Return a list of tuples of values of the fields specified in 

294 format, in order. 

295 

296 Parameters 

297 ---------- 

298 format : `list` 

299 The desired set of keys. 

300 dataId : `dict` 

301 A possible-incomplete ``dataId``. 

302 

303 Returns 

304 ------- 

305 fields : `list` of `tuple` 

306 Values of the fields specified in ``format``. 

307 

308 Raises 

309 ------ 

310 ValueError 

311 The channel number requested in ``dataId`` is out of range. 

312 """ 

313 nChannel = 16 # number of possible channels, 1..nChannel 

314 

315 if "channel" in dataId: # they specified a channel 

316 dataId = dataId.copy() 

317 channel = dataId.pop('channel') # Do not include in query below 

318 if channel > nChannel or channel < 1: 

319 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}") 

320 channels = [channel] 

321 else: 

322 channels = range(1, nChannel + 1) # we want all possible channels 

323 

324 if "channel" in format: # they asked for a channel, but we mustn't query for it 

325 format = list(format) 

326 channelIndex = format.index('channel') # where channel values should go 

327 format.pop(channelIndex) 

328 else: 

329 channelIndex = None 

330 

331 dids = [] # returned list of dataIds 

332 for value in self.query_raw(format, dataId): 

333 if channelIndex is None: 

334 dids.append(value) 

335 else: 

336 for c in channels: 

337 did = list(value) 

338 did.insert(channelIndex, c) 

339 dids.append(tuple(did)) 

340 

341 return dids 

342 # 

343 # The composite type "raw" doesn't provide e.g. query_raw, so we defined 

344 # type _raw in the .paf file with the same template, and forward requests 

345 # as necessary 

346 # 

347 

348 def query_raw(self, *args, **kwargs): 

349 """Magic method that is called automatically if it exists. 

350 

351 This code redirects the call to the right place, necessary because of 

352 leading underscore on ``_raw``. 

353 """ 

354 return self.query__raw(*args, **kwargs) 

355 

356 def map_raw_md(self, *args, **kwargs): 

357 """Magic method that is called automatically if it exists. 

358 

359 This code redirects the call to the right place, necessary because of 

360 leading underscore on ``_raw``. 

361 """ 

362 return self.map__raw_md(*args, **kwargs) 

363 

364 def map_raw_filename(self, *args, **kwargs): 

365 """Magic method that is called automatically if it exists. 

366 

367 This code redirects the call to the right place, necessary because of 

368 leading underscore on ``_raw``. 

369 """ 

370 return self.map__raw_filename(*args, **kwargs) 

371 

372 def bypass_raw_filename(self, *args, **kwargs): 

373 """Magic method that is called automatically if it exists. 

374 

375 This code redirects the call to the right place, necessary because of 

376 leading underscore on ``_raw``. 

377 """ 

378 return self.bypass__raw_filename(*args, **kwargs) 

379 

380 def map_raw_visitInfo(self, *args, **kwargs): 

381 """Magic method that is called automatically if it exists. 

382 

383 This code redirects the call to the right place, necessary because of 

384 leading underscore on ``_raw``. 

385 """ 

386 return self.map__raw_visitInfo(*args, **kwargs) 

387 

388 def bypass_raw_md(self, datasetType, pythonType, location, dataId): 

389 fileName = location.getLocationsWithRoot()[0] 

390 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

391 return md 

392 

393 def bypass_raw_hdu(self, datasetType, pythonType, location, dataId): 

394 # We need to override raw_hdu so that we can trap a request 

395 # for the primary HDU and merge it with the default content. 

396 fileName = location.getLocationsWithRoot()[0] 

397 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

398 return md 

399 

400 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId): 

401 fileName = location.getLocationsWithRoot()[0] 

402 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

403 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log) 

404 return makeVisitInfo(md) 

405 

406 def std_raw_amp(self, item, dataId): 

407 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId, 

408 trimmed=False, setVisitInfo=False, 

409 filter=False) # Don't set the filter for an amp 

410 

411 def std_raw(self, item, dataId, filter=True): 

412 """Standardize a raw dataset by converting it to an 

413 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`.""" 

414 

415 return self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False, 

416 setVisitInfo=False, # it's already set, and the metadata's stripped 

417 filter=filter) 

418 

419 

420class LsstCamMapper(LsstCamBaseMapper): 

421 """The mapper for lsstCam.""" 

422 translatorClass = LsstCamTranslator 

423 MakeRawVisitInfoClass = LsstCamRawVisitInfo 

424 _cameraName = "lsstCam" 

425 _gen3instrument = LsstCam