Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012-2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import re 

24import traceback 

25 

26import numpy as np 

27 

28from lsst.utils import getPackageDir 

29from astro_metadata_translator import fix_header, DecamTranslator 

30import lsst.afw.image as afwImage 

31from lsst.afw.fits import readMetadata 

32from lsst.afw.geom import makeSkyWcs 

33from lsst.obs.base import CameraMapper 

34from lsst.obs.base.utils import createInitialSkyWcs, InitialSkyWcsError 

35from lsst.daf.persistence import ButlerLocation, Storage, Policy 

36from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo 

37from .decamFilters import DECAM_FILTER_DEFINITIONS 

38from ._instrument import DarkEnergyCamera 

39 

40np.seterr(divide="ignore") 

41 

42__all__ = ["DecamMapper"] 

43 

44 

45class DecamMapper(CameraMapper): 

46 packageName = 'obs_decam' 

47 _gen3instrument = DarkEnergyCamera 

48 

49 MakeRawVisitInfoClass = MakeDecamRawVisitInfo 

50 

51 detectorNames = { 

52 1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21', 

53 10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18', 

54 18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2', 

55 27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4', 

56 36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13', 

57 45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21', 

58 53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29', 

59 62: 'N31'} 

60 

61 def __init__(self, inputPolicy=None, **kwargs): 

62 policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") 

63 policy = Policy(policyFile) 

64 

65 super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

66 

67 DECAM_FILTER_DEFINITIONS.defineFilters() 

68 

69 # The data ID key ccdnum is not directly used in the current policy 

70 # template of the raw and instcal et al. datasets, so is not in its 

71 # keyDict automatically. Add it so the butler know about the data ID key 

72 # ccdnum. 

73 # Similarly, add "object" for raws. 

74 for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"): 

75 self.mappings[datasetType].keyDict.update({'ccdnum': int}) 

76 self.mappings["raw"].keyDict.update({'object': str}) 

77 

78 # The number of bits allocated for fields in object IDs 

79 # TODO: This needs to be updated; also see Trac #2797 

80 DecamMapper._nbit_tract = 10 

81 DecamMapper._nbit_patch = 10 

82 DecamMapper._nbit_filter = 4 

83 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract 

84 + 2*DecamMapper._nbit_patch 

85 + DecamMapper._nbit_filter) 

86 

87 def _extractDetectorName(self, dataId): 

88 copyId = self._transformId(dataId) 

89 try: 

90 return DecamMapper.detectorNames[copyId['ccdnum']] 

91 except KeyError: 

92 raise RuntimeError("No name found for dataId: %s"%(dataId)) 

93 

94 def _transformId(self, dataId): 

95 copyId = CameraMapper._transformId(self, dataId) 

96 if "ccd" in copyId: 

97 copyId.setdefault("ccdnum", copyId["ccd"]) 

98 return copyId 

99 

100 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

101 return self._computeCcdExposureId(dataId) 

102 

103 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

104 return 32 # not really, but this leaves plenty of space for sources 

105 

106 def _computeCcdExposureId(self, dataId): 

107 """Compute the 64-bit (long) identifier for a CCD exposure. 

108 

109 Parameters 

110 ---------- 

111 dataId : `dict` 

112 Data identifier with visit, ccd. 

113 

114 Returns 

115 ------- 

116 result : `int` 

117 Integer identifier for a CCD exposure. 

118 """ 

119 copyId = self._transformId(dataId) 

120 visit = copyId['visit'] 

121 ccdnum = copyId['ccdnum'] 

122 return int("%07d%02d" % (visit, ccdnum)) 

123 

124 def _computeCoaddExposureId(self, dataId, singleFilter): 

125 """Compute the 64-bit (long) identifier for a coadd. 

126 

127 Parameters 

128 ---------- 

129 dataId : `dict` 

130 Data identifier with tract and patch. 

131 singleFilter : `bool` 

132 True means the desired ID is for a single-filter coadd, 

133 in which case the dataId must contain filter. 

134 

135 Returns 

136 ------- 

137 oid : `int` 

138 Unique integer identifier. 

139 """ 

140 tract = int(dataId['tract']) 

141 if tract < 0 or tract >= 2**DecamMapper._nbit_tract: 

142 raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract)) 

143 patchX, patchY = [int(x) for x in dataId['patch'].split(',')] 

144 for p in (patchX, patchY): 

145 if p < 0 or p >= 2**DecamMapper._nbit_patch: 

146 raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch) 

147 oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY 

148 if singleFilter: 

149 return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

150 return oid 

151 

152 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

153 return self._computeCoaddExposureId(dataId, True) 

154 

155 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

156 return 64 - DecamMapper._nbit_id 

157 

158 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

159 return self._computeCoaddExposureId(dataId, False) 

160 

161 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

162 return 64 - DecamMapper._nbit_id 

163 

164 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

165 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

166 

167 def bypass_dcrCoaddId_bits(self, *args, **kwargs): 

168 return self.bypass_deepCoaddId_bits(*args, **kwargs) 

169 

170 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

171 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

172 

173 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

174 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

175 

176 def translate_dqmask(self, dqmask): 

177 # TODO: make a class member variable that knows the mappings 

178 # below instead of hard-coding them 

179 dqmArr = dqmask.getArray() 

180 mask = afwImage.Mask(dqmask.getDimensions()) 

181 mArr = mask.getArray() 

182 idxBad = np.where(dqmArr & 1) 

183 idxSat = np.where(dqmArr & 2) 

184 idxIntrp = np.where(dqmArr & 4) 

185 idxCr = np.where(dqmArr & 16) 

186 idxBleed = np.where(dqmArr & 64) 

187 idxEdge = np.where(dqmArr & 512) 

188 mArr[idxBad] |= mask.getPlaneBitMask("BAD") 

189 mArr[idxSat] |= mask.getPlaneBitMask("SAT") 

190 mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP") 

191 mArr[idxCr] |= mask.getPlaneBitMask("CR") 

192 mArr[idxBleed] |= mask.getPlaneBitMask("SAT") 

193 mArr[idxEdge] |= mask.getPlaneBitMask("EDGE") 

194 return mask 

195 

196 def translate_wtmap(self, wtmap): 

197 wtmArr = wtmap.getArray() 

198 idxUndefWeight = np.where(wtmArr <= 0) 

199 # Reassign weights to be finite but small: 

200 wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)])) 

201 var = 1.0 / wtmArr 

202 varim = afwImage.ImageF(var) 

203 return varim 

204 

205 def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId): 

206 # Workaround until I can access the butler 

207 instcalMap = self.map_instcal(dataId) 

208 dqmaskMap = self.map_dqmask(dataId) 

209 wtmapMap = self.map_wtmap(dataId) 

210 instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1]) 

211 dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1]) 

212 wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1]) 

213 instcal = instcalType(instcalMap.getLocationsWithRoot()[0]) 

214 dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0]) 

215 wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0]) 

216 

217 mask = self.translate_dqmask(dqmask) 

218 variance = self.translate_wtmap(wtmap) 

219 

220 mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance) 

221 md = readMetadata(instcalMap.getLocationsWithRoot()[0]) 

222 fix_header(md, translator_class=DecamTranslator) 

223 wcs = makeSkyWcs(md, strip=True) 

224 exp = afwImage.ExposureF(mi, wcs) 

225 

226 exp.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(10**(0.4 * md.getScalar("MAGZERO")), 0)) 

227 visitInfo = self.makeRawVisitInfo(md=md) 

228 exp.getInfo().setVisitInfo(visitInfo) 

229 

230 for kw in ('LTV1', 'LTV2'): 

231 md.remove(kw) 

232 

233 exp.setMetadata(md) 

234 return exp 

235 

236 def std_raw(self, item, dataId): 

237 """Standardize a raw dataset by converting it to an Exposure. 

238 

239 Raw images are MEF files with one HDU for each detector. 

240 

241 Parameters 

242 ---------- 

243 item : `lsst.afw.image.DecoratedImage` 

244 The image read by the butler. 

245 dataId : data ID 

246 Data identifier. 

247 

248 Returns 

249 ------- 

250 result : `lsst.afw.image.Exposure` 

251 The standardized Exposure. 

252 """ 

253 return self._standardizeExposure(self.exposures['raw'], item, dataId, 

254 trimmed=False) 

255 

256 def _createInitialSkyWcs(self, exposure): 

257 # DECam has a coordinate system flipped on X with respect to our 

258 # VisitInfo definition of the field angle orientation. 

259 # We have to override this method until RFC-605 is implemented, to pass 

260 # `flipX=True` to createInitialSkyWcs below. 

261 self._createSkyWcsFromMetadata(exposure) 

262 

263 if exposure.getInfo().getVisitInfo() is None: 

264 msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs." 

265 self.log.warn(msg) 

266 return 

267 try: 

268 newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector(), 

269 flipX=True) 

270 exposure.setWcs(newSkyWcs) 

271 except InitialSkyWcsError as e: 

272 msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s" 

273 self.log.warn(msg, e) 

274 self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e)) 

275 if e.__context__ is not None: 

276 self.log.debug("Root-cause Exception was: %s", 

277 traceback.TracebackException.from_exception(e.__context__)) 

278 

279 def std_dark(self, item, dataId): 

280 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

281 rawPath = self.map_raw(dataId).getLocations()[0] 

282 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath) 

283 md0 = readMetadata(headerPath) 

284 fix_header(md0, translator_class=DecamTranslator) 

285 visitInfo = self.makeRawVisitInfo(md0) 

286 exp.getInfo().setVisitInfo(visitInfo) 

287 return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False) 

288 

289 def std_bias(self, item, dataId): 

290 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

291 return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False) 

292 

293 def std_flat(self, item, dataId): 

294 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

295 return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True) 

296 

297 def std_illumcor(self, item, dataId): 

298 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

299 return self._standardizeExposure(self.calibrations["illumcor"], exp, dataId, filter=True) 

300 

301 def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False): 

302 """Standardize a MasterCal image obtained from NOAO archive into Exposure 

303 

304 These MasterCal images are MEF files with one HDU for each detector. 

305 Some WCS header, eg CTYPE1, exists only in the zeroth extensionr, 

306 so info in the zeroth header need to be copied over to metadata. 

307 

308 Parameters 

309 ---------- 

310 datasetType : `str` 

311 Dataset type ("bias", "flat", or "illumcor"). 

312 item : `lsst.afw.image.DecoratedImage` 

313 The image read by the butler. 

314 dataId : data ID 

315 Data identifier. 

316 setFilter : `bool` 

317 Whether to set the filter in the Exposure. 

318 

319 Returns 

320 ------- 

321 result : `lsst.afw.image.Exposure` 

322 The standardized Exposure. 

323 """ 

324 mi = afwImage.makeMaskedImage(item.getImage()) 

325 md = item.getMetadata() 

326 masterCalMap = getattr(self, "map_" + datasetType) 

327 masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0] 

328 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath) 

329 md0 = readMetadata(headerPath) 

330 fix_header(md0, translator_class=DecamTranslator) 

331 for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2', 

332 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'): 

333 if kw in md0.paramNames() and kw not in md.paramNames(): 

334 md.add(kw, md0.getScalar(kw)) 

335 wcs = makeSkyWcs(md, strip=True) 

336 exp = afwImage.makeExposure(mi, wcs) 

337 exp.setMetadata(md) 

338 return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter) 

339 

340 def std_cpBias(self, item, dataId): 

341 return self._standardizeCpMasterCal("cpBias", item, dataId, setFilter=False) 

342 

343 def std_cpFlat(self, item, dataId): 

344 return self._standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True) 

345 

346 def std_fringe(self, item, dataId): 

347 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

348 return self._standardizeExposure(self.calibrations["fringe"], exp, dataId) 

349 

350 def std_cpIllumcor(self, item, dataId): 

351 return self._standardizeCpMasterCal("cpIllumcor", item, dataId, setFilter=True) 

352 

353 @classmethod 

354 def getLinearizerDir(cls): 

355 """Directory containing linearizers""" 

356 packageName = cls.getPackageName() 

357 packageDir = getPackageDir(packageName) 

358 return os.path.join(packageDir, "decam", "linearizer") 

359 

360 def map_linearizer(self, dataId, write=False): 

361 """Map a linearizer""" 

362 actualId = self._transformId(dataId) 

363 location = "%02d.fits" % (dataId["ccdnum"]) 

364 return ButlerLocation( 

365 pythonType="lsst.ip.isr.LinearizeSquared", 

366 cppType="Config", 

367 storageName="PickleStorage", 

368 locationList=[location], 

369 dataId=actualId, 

370 mapper=self, 

371 storage=Storage.makeFromURI(self.getLinearizerDir()) 

372 ) 

373 

374 @classmethod 

375 def getCrosstalkDir(cls): 

376 """Directory containing crosstalk tables. 

377 """ 

378 packageName = cls.getPackageName() 

379 packageDir = getPackageDir(packageName) 

380 return os.path.join(packageDir, "decam", "crosstalk")