Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning. 

24 

25import os 

26import lsst.log 

27import lsst.geom 

28import lsst.utils as utils 

29import lsst.afw.image as afwImage 

30from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo 

31import lsst.obs.base.yamlCamera as yamlCamera 

32import lsst.daf.persistence as dafPersist 

33from .translators import LsstCamTranslator 

34from ._fitsHeader import readRawFitsHeader 

35 

36from .filters import LSSTCAM_FILTER_DEFINITIONS 

37from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble 

38 

39__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"] 

40 

41 

42class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo): 

43 """Make a VisitInfo from the FITS header of a raw image.""" 

44 

45 

46class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo): 

47 metadataTranslator = LsstCamTranslator 

48 

49 

50def assemble_raw(dataId, componentInfo, cls): 

51 """Called by the butler to construct the composite type "raw". 

52 

53 Note that we still need to define "_raw" and copy various fields over. 

54 

55 Parameters 

56 ---------- 

57 dataId : `lsst.daf.persistence.dataId.DataId` 

58 The data ID. 

59 componentInfo : `dict` 

60 dict containing the components, as defined by the composite definition 

61 in the mapper policy. 

62 cls : 'object' 

63 Unused. 

64 

65 Returns 

66 ------- 

67 exposure : `lsst.afw.image.Exposure` 

68 The assembled exposure. 

69 """ 

70 

71 ampExps = componentInfo['raw_amp'].obj 

72 exposure = fixAmpsAndAssemble(ampExps, str(dataId)) 

73 md = componentInfo['raw_hdu'].obj 

74 exposure.setMetadata(md) 

75 

76 if not attachRawWcsFromBoresight(exposure): 

77 logger = lsst.log.Log.getLogger("LsstCamMapper") 

78 logger.warn("Unable to set WCS for %s from header as RA/Dec/Angle are unavailable", dataId) 

79 

80 return exposure 

81 

82 

83class LsstCamBaseMapper(CameraMapper): 

84 """The Base Mapper for all LSST-style instruments. 

85 """ 

86 

87 packageName = 'obs_lsst' 

88 _cameraName = "lsstCam" 

89 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence 

90 # 

91 # do not set MakeRawVisitInfoClass or translatorClass to anything other 

92 # than None! 

93 # 

94 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know 

95 # its mapper and cannot use mapper.makeRawVisitInfo() 

96 # 

97 MakeRawVisitInfoClass = None 

98 translatorClass = None 

99 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

100 

101 def __init__(self, inputPolicy=None, **kwargs): 

102 # 

103 # Merge the list of .yaml files 

104 # 

105 policy = None 

106 for yamlFile in self.yamlFileList: 

107 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy") 

108 npolicy = dafPersist.Policy(policyFile) 

109 

110 if policy is None: 

111 policy = npolicy 

112 else: 

113 policy.merge(npolicy) 

114 # 

115 # Look for the calibrations root "root/CALIB" if not supplied 

116 # 

117 if kwargs.get('root', None) and not kwargs.get('calibRoot', None): 

118 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

119 if "repositoryCfg" in kwargs: 

120 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

121 hasattr(cfg, "root")] 

122 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

123 for calibRoot in calibSearch: 

124 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

125 kwargs['calibRoot'] = calibRoot 

126 break 

127 if not kwargs.get('calibRoot', None): 

128 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory") 

129 

130 super().__init__(policy, os.path.dirname(policyFile), **kwargs) 

131 # 

132 # The composite objects don't seem to set these 

133 # 

134 for d in (self.mappings, self.exposures): 

135 d['raw'] = d['_raw'] 

136 

137 self.filterDefinitions.reset() 

138 self.filterDefinitions.defineFilters() 

139 

140 LsstCamMapper._nbit_tract = 16 

141 LsstCamMapper._nbit_patch = 5 

142 LsstCamMapper._nbit_filter = 7 

143 

144 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch 

145 + LsstCamMapper._nbit_filter) 

146 

147 if len(afwImage.Filter.getNames()) >= 2**LsstCamMapper._nbit_filter: 

148 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

149 LsstCamMapper._nbit_filter) 

150 

151 @classmethod 

152 def getCameraName(cls): 

153 return cls._cameraName 

154 

155 @classmethod 

156 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None): 

157 """Make a camera describing the camera geometry. 

158 

159 policy : ignored 

160 repositoryDir : ignored 

161 cameraYamlFile : `str` 

162 The full path to a yaml file to be passed to `yamlCamera.makeCamera` 

163 

164 Returns 

165 ------- 

166 camera : `lsst.afw.cameraGeom.Camera` 

167 Camera geometry. 

168 """ 

169 

170 if not cameraYamlFile: 

171 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy", 

172 ("%s.yaml" % cls.getCameraName())) 

173 

174 return yamlCamera.makeCamera(cameraYamlFile) 

175 

176 def _getRegistryValue(self, dataId, k): 

177 """Return a value from a dataId, or look it up in the registry if it 

178 isn't present.""" 

179 if k in dataId: 

180 return dataId[k] 

181 else: 

182 dataType = "bias" if "taiObs" in dataId else "raw" 

183 

184 try: 

185 return self.queryMetadata(dataType, [k], dataId)[0][0] 

186 except IndexError: 

187 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" % 

188 (k, dataType, dataId)) 

189 

190 def _extractDetectorName(self, dataId): 

191 if "channel" in dataId: # they specified a channel 

192 dataId = dataId.copy() 

193 del dataId["channel"] # Do not include in query 

194 raftName = self._getRegistryValue(dataId, "raftName") 

195 detectorName = self._getRegistryValue(dataId, "detectorName") 

196 

197 return "%s_%s" % (raftName, detectorName) 

198 

199 def _computeCcdExposureId(self, dataId): 

200 """Compute the 64-bit (long) identifier for a CCD exposure. 

201 

202 Parameters 

203 ---------- 

204 dataId : `dict` 

205 Data identifier including dayObs and seqNum. 

206 

207 Returns 

208 ------- 

209 id : `int` 

210 Integer identifier for a CCD exposure. 

211 """ 

212 try: 

213 visit = self._getRegistryValue(dataId, "visit") 

214 except Exception: 

215 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}") 

216 

217 if "detector" in dataId: 

218 detector = dataId["detector"] 

219 else: 

220 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'], 

221 dataId['detectorName']) 

222 

223 return self.translatorClass.compute_detector_exposure_id(visit, detector) 

224 

225 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

226 return self._computeCcdExposureId(dataId) 

227 

228 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

229 """How many bits are required for the maximum exposure ID""" 

230 # 52 for "C" controller and 51 for "O" 

231 return 52 # max detector_exposure_id ~ 3050121299999250 

232 

233 def _computeCoaddExposureId(self, dataId, singleFilter): 

234 """Compute the 64-bit (long) identifier for a coadd. 

235 

236 Parameters 

237 ---------- 

238 dataId : `dict` 

239 Data identifier with tract and patch. 

240 singleFilter : `bool` 

241 True means the desired ID is for a single-filter coadd, in which 

242 case ``dataId`` must contain filter. 

243 """ 

244 

245 tract = int(dataId['tract']) 

246 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract: 

247 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract)) 

248 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

249 for p in (patchX, patchY): 

250 if p < 0 or p >= 2**LsstCamMapper._nbit_patch: 

251 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch) 

252 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY 

253 if singleFilter: 

254 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

255 return oid 

256 

257 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

258 """The number of bits used up for patch ID bits.""" 

259 return 64 - LsstCamMapper._nbit_id 

260 

261 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

262 return self._computeCoaddExposureId(dataId, True) 

263 

264 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

265 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

266 

267 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

268 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

269 

270 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

271 """The number of bits used up for patch ID bits.""" 

272 return 64 - LsstCamMapper._nbit_id 

273 

274 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

275 return self._computeCoaddExposureId(dataId, False) 

276 

277 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

278 """The number of bits used up for patch ID bits.""" 

279 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

280 

281 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

282 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

283 

284 def query_raw_amp(self, format, dataId): 

285 """Return a list of tuples of values of the fields specified in 

286 format, in order. 

287 

288 Parameters 

289 ---------- 

290 format : `list` 

291 The desired set of keys. 

292 dataId : `dict` 

293 A possible-incomplete ``dataId``. 

294 

295 Returns 

296 ------- 

297 fields : `list` of `tuple` 

298 Values of the fields specified in ``format``. 

299 

300 Raises 

301 ------ 

302 ValueError 

303 The channel number requested in ``dataId`` is out of range. 

304 """ 

305 nChannel = 16 # number of possible channels, 1..nChannel 

306 

307 if "channel" in dataId: # they specified a channel 

308 dataId = dataId.copy() 

309 channel = dataId.pop('channel') # Do not include in query below 

310 if channel > nChannel or channel < 1: 

311 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}") 

312 channels = [channel] 

313 else: 

314 channels = range(1, nChannel + 1) # we want all possible channels 

315 

316 if "channel" in format: # they asked for a channel, but we mustn't query for it 

317 format = list(format) 

318 channelIndex = format.index('channel') # where channel values should go 

319 format.pop(channelIndex) 

320 else: 

321 channelIndex = None 

322 

323 dids = [] # returned list of dataIds 

324 for value in self.query_raw(format, dataId): 

325 if channelIndex is None: 

326 dids.append(value) 

327 else: 

328 for c in channels: 

329 did = list(value) 

330 did.insert(channelIndex, c) 

331 dids.append(tuple(did)) 

332 

333 return dids 

334 # 

335 # The composite type "raw" doesn't provide e.g. query_raw, so we defined 

336 # type _raw in the .paf file with the same template, and forward requests 

337 # as necessary 

338 # 

339 

340 def query_raw(self, *args, **kwargs): 

341 """Magic method that is called automatically if it exists. 

342 

343 This code redirects the call to the right place, necessary because of 

344 leading underscore on ``_raw``. 

345 """ 

346 return self.query__raw(*args, **kwargs) 

347 

348 def map_raw_md(self, *args, **kwargs): 

349 """Magic method that is called automatically if it exists. 

350 

351 This code redirects the call to the right place, necessary because of 

352 leading underscore on ``_raw``. 

353 """ 

354 return self.map__raw_md(*args, **kwargs) 

355 

356 def map_raw_filename(self, *args, **kwargs): 

357 """Magic method that is called automatically if it exists. 

358 

359 This code redirects the call to the right place, necessary because of 

360 leading underscore on ``_raw``. 

361 """ 

362 return self.map__raw_filename(*args, **kwargs) 

363 

364 def bypass_raw_filename(self, *args, **kwargs): 

365 """Magic method that is called automatically if it exists. 

366 

367 This code redirects the call to the right place, necessary because of 

368 leading underscore on ``_raw``. 

369 """ 

370 return self.bypass__raw_filename(*args, **kwargs) 

371 

372 def map_raw_visitInfo(self, *args, **kwargs): 

373 """Magic method that is called automatically if it exists. 

374 

375 This code redirects the call to the right place, necessary because of 

376 leading underscore on ``_raw``. 

377 """ 

378 return self.map__raw_visitInfo(*args, **kwargs) 

379 

380 def bypass_raw_md(self, datasetType, pythonType, location, dataId): 

381 fileName = location.getLocationsWithRoot()[0] 

382 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

383 return md 

384 

385 def bypass_raw_hdu(self, datasetType, pythonType, location, dataId): 

386 # We need to override raw_hdu so that we can trap a request 

387 # for the primary HDU and merge it with the default content. 

388 fileName = location.getLocationsWithRoot()[0] 

389 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

390 return md 

391 

392 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId): 

393 fileName = location.getLocationsWithRoot()[0] 

394 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

395 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log) 

396 return makeVisitInfo(md) 

397 

398 def std_raw_amp(self, item, dataId): 

399 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId, 

400 trimmed=False, setVisitInfo=False, 

401 filter=False) # Don't set the filter for an amp 

402 

403 def std_raw(self, item, dataId, filter=True): 

404 """Standardize a raw dataset by converting it to an 

405 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`.""" 

406 

407 return self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False, 

408 setVisitInfo=False, # it's already set, and the metadata's stripped 

409 filter=filter) 

410 

411 

412class LsstCamMapper(LsstCamBaseMapper): 

413 """The mapper for lsstCam.""" 

414 translatorClass = LsstCamTranslator 

415 MakeRawVisitInfoClass = LsstCamRawVisitInfo 

416 _cameraName = "lsstCam"