Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning. 

24 

25import os 

26import re 

27import lsst.log 

28import lsst.geom 

29import lsst.utils as utils 

30import lsst.afw.image as afwImage 

31from lsst.afw.fits import readMetadata 

32from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo 

33import lsst.obs.base.yamlCamera as yamlCamera 

34import lsst.daf.persistence as dafPersist 

35from .translators import LsstCamTranslator 

36from astro_metadata_translator import fix_header 

37 

38from .filters import LSSTCAM_FILTER_DEFINITIONS 

39from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble 

40 

41__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"] 

42 

43 

44class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo): 

45 """Make a VisitInfo from the FITS header of a raw image.""" 

46 

47 

48class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo): 

49 metadataTranslator = LsstCamTranslator 

50 

51 

52def assemble_raw(dataId, componentInfo, cls): 

53 """Called by the butler to construct the composite type "raw". 

54 

55 Note that we still need to define "_raw" and copy various fields over. 

56 

57 Parameters 

58 ---------- 

59 dataId : `lsst.daf.persistence.dataId.DataId` 

60 The data ID. 

61 componentInfo : `dict` 

62 dict containing the components, as defined by the composite definition 

63 in the mapper policy. 

64 cls : 'object' 

65 Unused. 

66 

67 Returns 

68 ------- 

69 exposure : `lsst.afw.image.Exposure` 

70 The assembled exposure. 

71 """ 

72 

73 ampExps = componentInfo['raw_amp'].obj 

74 exposure = fixAmpsAndAssemble(ampExps, str(dataId)) 

75 md = componentInfo['raw_hdu'].obj 

76 fix_header(md) # No mapper so cannot specify the translator class 

77 exposure.setMetadata(md) 

78 

79 if not attachRawWcsFromBoresight(exposure): 

80 logger = lsst.log.Log.getLogger("LsstCamMapper") 

81 logger.warn("Unable to set WCS for %s from header as RA/Dec/Angle are unavailable", dataId) 

82 

83 return exposure 

84 

85 

86class LsstCamBaseMapper(CameraMapper): 

87 """The Base Mapper for all LSST-style instruments. 

88 """ 

89 

90 packageName = 'obs_lsst' 

91 _cameraName = "lsstCam" 

92 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence 

93 # 

94 # do not set MakeRawVisitInfoClass or translatorClass to anything other 

95 # than None! 

96 # 

97 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know 

98 # its mapper and cannot use mapper.makeRawVisitInfo() 

99 # 

100 MakeRawVisitInfoClass = None 

101 translatorClass = None 

102 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

103 

104 def __init__(self, inputPolicy=None, **kwargs): 

105 # 

106 # Merge the list of .yaml files 

107 # 

108 policy = None 

109 for yamlFile in self.yamlFileList: 

110 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy") 

111 npolicy = dafPersist.Policy(policyFile) 

112 

113 if policy is None: 

114 policy = npolicy 

115 else: 

116 policy.merge(npolicy) 

117 # 

118 # Look for the calibrations root "root/CALIB" if not supplied 

119 # 

120 if kwargs.get('root', None) and not kwargs.get('calibRoot', None): 

121 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

122 if "repositoryCfg" in kwargs: 

123 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

124 hasattr(cfg, "root")] 

125 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

126 for calibRoot in calibSearch: 

127 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

128 kwargs['calibRoot'] = calibRoot 

129 break 

130 if not kwargs.get('calibRoot', None): 

131 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory") 

132 

133 super().__init__(policy, os.path.dirname(policyFile), **kwargs) 

134 # 

135 # The composite objects don't seem to set these 

136 # 

137 for d in (self.mappings, self.exposures): 

138 d['raw'] = d['_raw'] 

139 

140 self.filterDefinitions.reset() 

141 self.filterDefinitions.defineFilters() 

142 

143 LsstCamMapper._nbit_tract = 16 

144 LsstCamMapper._nbit_patch = 5 

145 LsstCamMapper._nbit_filter = 7 

146 

147 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch 

148 + LsstCamMapper._nbit_filter) 

149 

150 if len(afwImage.Filter.getNames()) >= 2**LsstCamMapper._nbit_filter: 

151 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

152 LsstCamMapper._nbit_filter) 

153 

154 @classmethod 

155 def getCameraName(cls): 

156 return cls._cameraName 

157 

158 @classmethod 

159 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None): 

160 """Make a camera describing the camera geometry. 

161 

162 policy : ignored 

163 repositoryDir : ignored 

164 cameraYamlFile : `str` 

165 The full path to a yaml file to be passed to `yamlCamera.makeCamera` 

166 

167 Returns 

168 ------- 

169 camera : `lsst.afw.cameraGeom.Camera` 

170 Camera geometry. 

171 """ 

172 

173 if not cameraYamlFile: 

174 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy", 

175 ("%s.yaml" % cls.getCameraName())) 

176 

177 return yamlCamera.makeCamera(cameraYamlFile) 

178 

179 def _getRegistryValue(self, dataId, k): 

180 """Return a value from a dataId, or look it up in the registry if it 

181 isn't present.""" 

182 if k in dataId: 

183 return dataId[k] 

184 else: 

185 dataType = "bias" if "taiObs" in dataId else "raw" 

186 

187 try: 

188 return self.queryMetadata(dataType, [k], dataId)[0][0] 

189 except IndexError: 

190 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" % 

191 (k, dataType, dataId)) 

192 

193 def _extractDetectorName(self, dataId): 

194 if "channel" in dataId: # they specified a channel 

195 dataId = dataId.copy() 

196 del dataId["channel"] # Do not include in query 

197 raftName = self._getRegistryValue(dataId, "raftName") 

198 detectorName = self._getRegistryValue(dataId, "detectorName") 

199 

200 return "%s_%s" % (raftName, detectorName) 

201 

202 def _computeCcdExposureId(self, dataId): 

203 """Compute the 64-bit (long) identifier for a CCD exposure. 

204 

205 Parameters 

206 ---------- 

207 dataId : `dict` 

208 Data identifier including dayObs and seqNum. 

209 

210 Returns 

211 ------- 

212 id : `int` 

213 Integer identifier for a CCD exposure. 

214 """ 

215 try: 

216 visit = self._getRegistryValue(dataId, "visit") 

217 except Exception: 

218 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}") 

219 

220 if "detector" in dataId: 

221 detector = dataId["detector"] 

222 else: 

223 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'], 

224 dataId['detectorName']) 

225 

226 return self.translatorClass.compute_detector_exposure_id(visit, detector) 

227 

228 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

229 return self._computeCcdExposureId(dataId) 

230 

231 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

232 """How many bits are required for the maximum exposure ID""" 

233 # 52 for "C" controller and 51 for "O" 

234 return 52 # max detector_exposure_id ~ 3050121299999250 

235 

236 def _computeCoaddExposureId(self, dataId, singleFilter): 

237 """Compute the 64-bit (long) identifier for a coadd. 

238 

239 Parameters 

240 ---------- 

241 dataId : `dict` 

242 Data identifier with tract and patch. 

243 singleFilter : `bool` 

244 True means the desired ID is for a single-filter coadd, in which 

245 case ``dataId`` must contain filter. 

246 """ 

247 

248 tract = int(dataId['tract']) 

249 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract: 

250 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract)) 

251 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

252 for p in (patchX, patchY): 

253 if p < 0 or p >= 2**LsstCamMapper._nbit_patch: 

254 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch) 

255 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY 

256 if singleFilter: 

257 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

258 return oid 

259 

260 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

261 """The number of bits used up for patch ID bits.""" 

262 return 64 - LsstCamMapper._nbit_id 

263 

264 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

265 return self._computeCoaddExposureId(dataId, True) 

266 

267 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

268 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

269 

270 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

271 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

272 

273 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

274 """The number of bits used up for patch ID bits.""" 

275 return 64 - LsstCamMapper._nbit_id 

276 

277 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

278 return self._computeCoaddExposureId(dataId, False) 

279 

280 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

281 """The number of bits used up for patch ID bits.""" 

282 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

283 

284 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

285 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

286 

287 def query_raw_amp(self, format, dataId): 

288 """Return a list of tuples of values of the fields specified in 

289 format, in order. 

290 

291 Parameters 

292 ---------- 

293 format : `list` 

294 The desired set of keys. 

295 dataId : `dict` 

296 A possible-incomplete ``dataId``. 

297 

298 Returns 

299 ------- 

300 fields : `list` of `tuple` 

301 Values of the fields specified in ``format``. 

302 

303 Raises 

304 ------ 

305 ValueError 

306 The channel number requested in ``dataId`` is out of range. 

307 """ 

308 nChannel = 16 # number of possible channels, 1..nChannel 

309 

310 if "channel" in dataId: # they specified a channel 

311 dataId = dataId.copy() 

312 channel = dataId.pop('channel') # Do not include in query below 

313 if channel > nChannel or channel < 1: 

314 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}") 

315 channels = [channel] 

316 else: 

317 channels = range(1, nChannel + 1) # we want all possible channels 

318 

319 if "channel" in format: # they asked for a channel, but we mustn't query for it 

320 format = list(format) 

321 channelIndex = format.index('channel') # where channel values should go 

322 format.pop(channelIndex) 

323 else: 

324 channelIndex = None 

325 

326 dids = [] # returned list of dataIds 

327 for value in self.query_raw(format, dataId): 

328 if channelIndex is None: 

329 dids.append(value) 

330 else: 

331 for c in channels: 

332 did = list(value) 

333 did.insert(channelIndex, c) 

334 dids.append(tuple(did)) 

335 

336 return dids 

337 # 

338 # The composite type "raw" doesn't provide e.g. query_raw, so we defined 

339 # type _raw in the .paf file with the same template, and forward requests 

340 # as necessary 

341 # 

342 

343 def query_raw(self, *args, **kwargs): 

344 """Magic method that is called automatically if it exists. 

345 

346 This code redirects the call to the right place, necessary because of 

347 leading underscore on ``_raw``. 

348 """ 

349 return self.query__raw(*args, **kwargs) 

350 

351 def map_raw_md(self, *args, **kwargs): 

352 """Magic method that is called automatically if it exists. 

353 

354 This code redirects the call to the right place, necessary because of 

355 leading underscore on ``_raw``. 

356 """ 

357 return self.map__raw_md(*args, **kwargs) 

358 

359 def map_raw_filename(self, *args, **kwargs): 

360 """Magic method that is called automatically if it exists. 

361 

362 This code redirects the call to the right place, necessary because of 

363 leading underscore on ``_raw``. 

364 """ 

365 return self.map__raw_filename(*args, **kwargs) 

366 

367 def bypass_raw_filename(self, *args, **kwargs): 

368 """Magic method that is called automatically if it exists. 

369 

370 This code redirects the call to the right place, necessary because of 

371 leading underscore on ``_raw``. 

372 """ 

373 return self.bypass__raw_filename(*args, **kwargs) 

374 

375 def map_raw_visitInfo(self, *args, **kwargs): 

376 """Magic method that is called automatically if it exists. 

377 

378 This code redirects the call to the right place, necessary because of 

379 leading underscore on ``_raw``. 

380 """ 

381 return self.map__raw_visitInfo(*args, **kwargs) 

382 

383 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId): 

384 fileName = location.getLocationsWithRoot()[0] 

385 mat = re.search(r"\[(\d+)\]$", fileName) 

386 if mat: 

387 hdu = int(mat.group(1)) 

388 md = readMetadata(fileName, hdu=hdu) 

389 else: 

390 md = readMetadata(fileName) # or hdu = INT_MIN; -(1 << 31) 

391 

392 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log) 

393 fix_header(md, translator_class=self.translatorClass) 

394 return makeVisitInfo(md) 

395 

396 def std_raw_amp(self, item, dataId): 

397 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId, 

398 trimmed=False, setVisitInfo=False, 

399 filter=False) # Don't set the filter for an amp 

400 

401 def std_raw(self, item, dataId, filter=True): 

402 """Standardize a raw dataset by converting it to an 

403 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`.""" 

404 

405 return self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False, 

406 setVisitInfo=False, # it's already set, and the metadata's stripped 

407 filter=filter) 

408 

409 

410class LsstCamMapper(LsstCamBaseMapper): 

411 """The mapper for lsstCam.""" 

412 translatorClass = LsstCamTranslator 

413 MakeRawVisitInfoClass = LsstCamRawVisitInfo 

414 _cameraName = "lsstCam"