Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <http://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""The LsstCam Mapper.""" # necessary to suppress D100 flake8 warning. 

24 

25import os 

26import lsst.log 

27import lsst.geom 

28import lsst.utils as utils 

29import lsst.afw.image as afwImage 

30from lsst.obs.base import CameraMapper, MakeRawVisitInfoViaObsInfo 

31import lsst.obs.base.yamlCamera as yamlCamera 

32import lsst.daf.persistence as dafPersist 

33from .translators import LsstCamTranslator 

34from ._fitsHeader import readRawFitsHeader 

35from ._instrument import LsstCam 

36 

37from .filters import LSSTCAM_FILTER_DEFINITIONS 

38from .assembly import attachRawWcsFromBoresight, fixAmpsAndAssemble 

39 

40__all__ = ["LsstCamMapper", "LsstCamMakeRawVisitInfo"] 

41 

42 

43class LsstCamMakeRawVisitInfo(MakeRawVisitInfoViaObsInfo): 

44 """Make a VisitInfo from the FITS header of a raw image.""" 

45 

46 

47class LsstCamRawVisitInfo(LsstCamMakeRawVisitInfo): 

48 metadataTranslator = LsstCamTranslator 

49 

50 

51def assemble_raw(dataId, componentInfo, cls): 

52 """Called by the butler to construct the composite type "raw". 

53 

54 Note that we still need to define "_raw" and copy various fields over. 

55 

56 Parameters 

57 ---------- 

58 dataId : `lsst.daf.persistence.dataId.DataId` 

59 The data ID. 

60 componentInfo : `dict` 

61 dict containing the components, as defined by the composite definition 

62 in the mapper policy. 

63 cls : 'object' 

64 Unused. 

65 

66 Returns 

67 ------- 

68 exposure : `lsst.afw.image.Exposure` 

69 The assembled exposure. 

70 """ 

71 

72 ampExps = componentInfo['raw_amp'].obj 

73 exposure = fixAmpsAndAssemble(ampExps, str(dataId)) 

74 md = componentInfo['raw_hdu'].obj 

75 exposure.setMetadata(md) 

76 

77 attachRawWcsFromBoresight(exposure, dataId) 

78 

79 return exposure 

80 

81 

82class LsstCamBaseMapper(CameraMapper): 

83 """The Base Mapper for all LSST-style instruments. 

84 """ 

85 

86 packageName = 'obs_lsst' 

87 _cameraName = "lsstCam" 

88 yamlFileList = ("lsstCamMapper.yaml",) # list of yaml files to load, keeping the first occurrence 

89 # 

90 # do not set MakeRawVisitInfoClass or translatorClass to anything other 

91 # than None! 

92 # 

93 # assemble_raw relies on autodetect as in butler Gen2 it doesn't know 

94 # its mapper and cannot use mapper.makeRawVisitInfo() 

95 # 

96 MakeRawVisitInfoClass = None 

97 translatorClass = None 

98 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS 

99 

100 def __init__(self, inputPolicy=None, **kwargs): 

101 # 

102 # Merge the list of .yaml files 

103 # 

104 policy = None 

105 for yamlFile in self.yamlFileList: 

106 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, yamlFile, "policy") 

107 npolicy = dafPersist.Policy(policyFile) 

108 

109 if policy is None: 

110 policy = npolicy 

111 else: 

112 policy.merge(npolicy) 

113 # 

114 # Look for the calibrations root "root/CALIB" if not supplied 

115 # 

116 if kwargs.get('root', None) and not kwargs.get('calibRoot', None): 

117 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

118 if "repositoryCfg" in kwargs: 

119 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

120 hasattr(cfg, "root")] 

121 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

122 for calibRoot in calibSearch: 

123 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

124 kwargs['calibRoot'] = calibRoot 

125 break 

126 if not kwargs.get('calibRoot', None): 

127 lsst.log.Log.getLogger("LsstCamMapper").warn("Unable to find valid calib root directory") 

128 

129 super().__init__(policy, os.path.dirname(policyFile), **kwargs) 

130 # 

131 # The composite objects don't seem to set these 

132 # 

133 for d in (self.mappings, self.exposures): 

134 d['raw'] = d['_raw'] 

135 

136 self.filterDefinitions.reset() 

137 self.filterDefinitions.defineFilters() 

138 

139 LsstCamMapper._nbit_tract = 16 

140 LsstCamMapper._nbit_patch = 5 

141 LsstCamMapper._nbit_filter = 7 

142 

143 LsstCamMapper._nbit_id = 64 - (LsstCamMapper._nbit_tract + 2*LsstCamMapper._nbit_patch 

144 + LsstCamMapper._nbit_filter) 

145 

146 if len(afwImage.Filter.getNames()) >= 2**LsstCamMapper._nbit_filter: 

147 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

148 LsstCamMapper._nbit_filter) 

149 

150 @classmethod 

151 def getCameraName(cls): 

152 return cls._cameraName 

153 

154 @classmethod 

155 def _makeCamera(cls, policy=None, repositoryDir=None, cameraYamlFile=None): 

156 """Make a camera describing the camera geometry. 

157 

158 policy : ignored 

159 repositoryDir : ignored 

160 cameraYamlFile : `str` 

161 The full path to a yaml file to be passed to `yamlCamera.makeCamera` 

162 

163 Returns 

164 ------- 

165 camera : `lsst.afw.cameraGeom.Camera` 

166 Camera geometry. 

167 """ 

168 if not cameraYamlFile: 

169 cameraYamlFile = os.path.join(utils.getPackageDir(cls.packageName), "policy", 

170 ("%s.yaml" % cls.getCameraName())) 

171 

172 return yamlCamera.makeCamera(cameraYamlFile) 

173 

174 def _getRegistryValue(self, dataId, k): 

175 """Return a value from a dataId, or look it up in the registry if it 

176 isn't present.""" 

177 if k in dataId: 

178 return dataId[k] 

179 else: 

180 dataType = "bias" if "taiObs" in dataId else "raw" 

181 

182 try: 

183 return self.queryMetadata(dataType, [k], dataId)[0][0] 

184 except IndexError: 

185 raise RuntimeError("Unable to lookup %s in \"%s\" registry for dataId %s" % 

186 (k, dataType, dataId)) 

187 

188 def _extractDetectorName(self, dataId): 

189 if "channel" in dataId: # they specified a channel 

190 dataId = dataId.copy() 

191 del dataId["channel"] # Do not include in query 

192 raftName = self._getRegistryValue(dataId, "raftName") 

193 detectorName = self._getRegistryValue(dataId, "detectorName") 

194 

195 return "%s_%s" % (raftName, detectorName) 

196 

197 def _computeCcdExposureId(self, dataId): 

198 """Compute the 64-bit (long) identifier for a CCD exposure. 

199 

200 Parameters 

201 ---------- 

202 dataId : `dict` 

203 Data identifier including dayObs and seqNum. 

204 

205 Returns 

206 ------- 

207 id : `int` 

208 Integer identifier for a CCD exposure. 

209 """ 

210 try: 

211 visit = self._getRegistryValue(dataId, "visit") 

212 except Exception: 

213 raise KeyError(f"Require a visit ID to calculate detector exposure ID. Got: {dataId}") 

214 

215 if "detector" in dataId: 

216 detector = dataId["detector"] 

217 else: 

218 detector = self.translatorClass.compute_detector_num_from_name(dataId['raftName'], 

219 dataId['detectorName']) 

220 

221 return self.translatorClass.compute_detector_exposure_id(visit, detector) 

222 

223 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

224 return self._computeCcdExposureId(dataId) 

225 

226 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

227 """How many bits are required for the maximum exposure ID""" 

228 # 52 for "C" controller and 51 for "O" 

229 return 52 # max detector_exposure_id ~ 3050121299999250 

230 

231 def _computeCoaddExposureId(self, dataId, singleFilter): 

232 """Compute the 64-bit (long) identifier for a coadd. 

233 

234 Parameters 

235 ---------- 

236 dataId : `dict` 

237 Data identifier with tract and patch. 

238 singleFilter : `bool` 

239 True means the desired ID is for a single-filter coadd, in which 

240 case ``dataId`` must contain filter. 

241 """ 

242 

243 tract = int(dataId['tract']) 

244 if tract < 0 or tract >= 2**LsstCamMapper._nbit_tract: 

245 raise RuntimeError('tract not in range [0,%d)' % (2**LsstCamMapper._nbit_tract)) 

246 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

247 for p in (patchX, patchY): 

248 if p < 0 or p >= 2**LsstCamMapper._nbit_patch: 

249 raise RuntimeError('patch component not in range [0, %d)' % 2**LsstCamMapper._nbit_patch) 

250 oid = (((tract << LsstCamMapper._nbit_patch) + patchX) << LsstCamMapper._nbit_patch) + patchY 

251 if singleFilter: 

252 return (oid << LsstCamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

253 return oid 

254 

255 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

256 """The number of bits used up for patch ID bits.""" 

257 return 64 - LsstCamMapper._nbit_id 

258 

259 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

260 return self._computeCoaddExposureId(dataId, True) 

261 

262 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

263 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

264 

265 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

266 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

267 

268 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

269 """The number of bits used up for patch ID bits.""" 

270 return 64 - LsstCamMapper._nbit_id 

271 

272 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

273 return self._computeCoaddExposureId(dataId, False) 

274 

275 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

276 """The number of bits used up for patch ID bits.""" 

277 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

278 

279 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

280 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

281 

282 def query_raw_amp(self, format, dataId): 

283 """Return a list of tuples of values of the fields specified in 

284 format, in order. 

285 

286 Parameters 

287 ---------- 

288 format : `list` 

289 The desired set of keys. 

290 dataId : `dict` 

291 A possible-incomplete ``dataId``. 

292 

293 Returns 

294 ------- 

295 fields : `list` of `tuple` 

296 Values of the fields specified in ``format``. 

297 

298 Raises 

299 ------ 

300 ValueError 

301 The channel number requested in ``dataId`` is out of range. 

302 """ 

303 # set number of possible channels, 1..nChannel 

304 # The wave front chips are special, 4k x 2k with only 8 amps 

305 

306 if "detectorName" in dataId: 

307 detectorName = dataId.get("detectorName") 

308 elif "detector" in dataId: 

309 detector = dataId.get("detector") 

310 if detector in self.camera: 

311 name = self.camera[detector].getName() 

312 detectorName = name.split('_')[1] 

313 else: 

314 raise RuntimeError('Unable to find detector %s in camera' % detector) 

315 else: 

316 logger = lsst.log.Log.getLogger("LsstCamMapper") 

317 logger.debug('Unable to lookup either "detectorName" or "detector" in the dataId') 

318 detectorName = "unknown" 

319 

320 if detectorName in ["SW0", "SW1"]: 

321 nChannel = 8 

322 else: 

323 nChannel = 16 

324 

325 if "channel" in dataId: # they specified a channel 

326 dataId = dataId.copy() 

327 channel = dataId.pop('channel') # Do not include in query below 

328 if channel > nChannel or channel < 1: 

329 raise ValueError(f"Requested channel is out of range 0 < {channel} <= {nChannel}") 

330 channels = [channel] 

331 else: 

332 channels = range(1, nChannel + 1) # we want all possible channels 

333 

334 if "channel" in format: # they asked for a channel, but we mustn't query for it 

335 format = list(format) 

336 channelIndex = format.index('channel') # where channel values should go 

337 format.pop(channelIndex) 

338 else: 

339 channelIndex = None 

340 

341 dids = [] # returned list of dataIds 

342 for value in self.query_raw(format, dataId): 

343 if channelIndex is None: 

344 dids.append(value) 

345 else: 

346 for c in channels: 

347 did = list(value) 

348 did.insert(channelIndex, c) 

349 dids.append(tuple(did)) 

350 

351 return dids 

352 # 

353 # The composite type "raw" doesn't provide e.g. query_raw, so we defined 

354 # type _raw in the .paf file with the same template, and forward requests 

355 # as necessary 

356 # 

357 

358 def query_raw(self, *args, **kwargs): 

359 """Magic method that is called automatically if it exists. 

360 

361 This code redirects the call to the right place, necessary because of 

362 leading underscore on ``_raw``. 

363 """ 

364 return self.query__raw(*args, **kwargs) 

365 

366 def map_raw_md(self, *args, **kwargs): 

367 """Magic method that is called automatically if it exists. 

368 

369 This code redirects the call to the right place, necessary because of 

370 leading underscore on ``_raw``. 

371 """ 

372 return self.map__raw_md(*args, **kwargs) 

373 

374 def map_raw_filename(self, *args, **kwargs): 

375 """Magic method that is called automatically if it exists. 

376 

377 This code redirects the call to the right place, necessary because of 

378 leading underscore on ``_raw``. 

379 """ 

380 return self.map__raw_filename(*args, **kwargs) 

381 

382 def bypass_raw_filename(self, *args, **kwargs): 

383 """Magic method that is called automatically if it exists. 

384 

385 This code redirects the call to the right place, necessary because of 

386 leading underscore on ``_raw``. 

387 """ 

388 return self.bypass__raw_filename(*args, **kwargs) 

389 

390 def map_raw_visitInfo(self, *args, **kwargs): 

391 """Magic method that is called automatically if it exists. 

392 

393 This code redirects the call to the right place, necessary because of 

394 leading underscore on ``_raw``. 

395 """ 

396 return self.map__raw_visitInfo(*args, **kwargs) 

397 

398 def bypass_raw_md(self, datasetType, pythonType, location, dataId): 

399 fileName = location.getLocationsWithRoot()[0] 

400 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

401 return md 

402 

403 def bypass_raw_hdu(self, datasetType, pythonType, location, dataId): 

404 # We need to override raw_hdu so that we can trap a request 

405 # for the primary HDU and merge it with the default content. 

406 fileName = location.getLocationsWithRoot()[0] 

407 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

408 return md 

409 

410 def bypass_raw_visitInfo(self, datasetType, pythonType, location, dataId): 

411 fileName = location.getLocationsWithRoot()[0] 

412 md = readRawFitsHeader(fileName, translator_class=self.translatorClass) 

413 makeVisitInfo = self.MakeRawVisitInfoClass(log=self.log) 

414 return makeVisitInfo(md) 

415 

416 def std_raw_amp(self, item, dataId): 

417 return self._standardizeExposure(self.exposures['raw_amp'], item, dataId, 

418 trimmed=False, setVisitInfo=False, 

419 filter=False) # Don't set the filter for an amp 

420 

421 def std_raw(self, item, dataId, filter=True): 

422 """Standardize a raw dataset by converting it to an 

423 `~lsst.afw.image.Exposure` instead of an `~lsst.afw.image.Image`.""" 

424 

425 return self._standardizeExposure(self.exposures['raw'], item, dataId, trimmed=False, 

426 setVisitInfo=False, # it's already set, and the metadata's stripped 

427 filter=filter) 

428 

429 

430class LsstCamMapper(LsstCamBaseMapper): 

431 """The mapper for lsstCam.""" 

432 translatorClass = LsstCamTranslator 

433 MakeRawVisitInfoClass = LsstCamRawVisitInfo 

434 _cameraName = "lsstCam" 

435 _gen3instrument = LsstCam