Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012-2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import re 

24import traceback 

25 

26import numpy as np 

27 

28from lsst.utils import getPackageDir 

29from astro_metadata_translator import fix_header, DecamTranslator 

30import lsst.afw.image as afwImage 

31from lsst.afw.fits import readMetadata 

32from lsst.afw.geom import makeSkyWcs 

33from lsst.obs.base import CameraMapper 

34from lsst.obs.base.utils import createInitialSkyWcs, InitialSkyWcsError 

35from lsst.daf.persistence import ButlerLocation, Storage, Policy 

36from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo 

37from .decamFilters import DECAM_FILTER_DEFINITIONS 

38 

39np.seterr(divide="ignore") 

40 

41__all__ = ["DecamMapper"] 

42 

43 

44class DecamMapper(CameraMapper): 

45 packageName = 'obs_decam' 

46 

47 MakeRawVisitInfoClass = MakeDecamRawVisitInfo 

48 

49 detectorNames = { 

50 1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21', 

51 10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18', 

52 18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2', 

53 27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4', 

54 36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13', 

55 45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21', 

56 53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29', 

57 62: 'N31'} 

58 

59 def __init__(self, inputPolicy=None, **kwargs): 

60 policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") 

61 policy = Policy(policyFile) 

62 

63 super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

64 

65 DECAM_FILTER_DEFINITIONS.defineFilters() 

66 

67 # The data ID key ccdnum is not directly used in the current policy 

68 # template of the raw and instcal et al. datasets, so is not in its 

69 # keyDict automatically. Add it so the butler know about the data ID key 

70 # ccdnum. 

71 # Similarly, add "object" for raws. 

72 for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"): 

73 self.mappings[datasetType].keyDict.update({'ccdnum': int}) 

74 self.mappings["raw"].keyDict.update({'object': str}) 

75 

76 # The number of bits allocated for fields in object IDs 

77 # TODO: This needs to be updated; also see Trac #2797 

78 DecamMapper._nbit_tract = 10 

79 DecamMapper._nbit_patch = 10 

80 DecamMapper._nbit_filter = 4 

81 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract 

82 + 2*DecamMapper._nbit_patch 

83 + DecamMapper._nbit_filter) 

84 

85 def _extractDetectorName(self, dataId): 

86 copyId = self._transformId(dataId) 

87 try: 

88 return DecamMapper.detectorNames[copyId['ccdnum']] 

89 except KeyError: 

90 raise RuntimeError("No name found for dataId: %s"%(dataId)) 

91 

92 def _transformId(self, dataId): 

93 copyId = CameraMapper._transformId(self, dataId) 

94 if "ccd" in copyId: 

95 copyId.setdefault("ccdnum", copyId["ccd"]) 

96 return copyId 

97 

98 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

99 return self._computeCcdExposureId(dataId) 

100 

101 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

102 return 32 # not really, but this leaves plenty of space for sources 

103 

104 def _computeCcdExposureId(self, dataId): 

105 """Compute the 64-bit (long) identifier for a CCD exposure. 

106 

107 Parameters 

108 ---------- 

109 dataId : `dict` 

110 Data identifier with visit, ccd. 

111 

112 Returns 

113 ------- 

114 result : `int` 

115 Integer identifier for a CCD exposure. 

116 """ 

117 copyId = self._transformId(dataId) 

118 visit = copyId['visit'] 

119 ccdnum = copyId['ccdnum'] 

120 return int("%07d%02d" % (visit, ccdnum)) 

121 

122 def _computeCoaddExposureId(self, dataId, singleFilter): 

123 """Compute the 64-bit (long) identifier for a coadd. 

124 

125 Parameters 

126 ---------- 

127 dataId : `dict` 

128 Data identifier with tract and patch. 

129 singleFilter : `bool` 

130 True means the desired ID is for a single-filter coadd, 

131 in which case the dataId must contain filter. 

132 

133 Returns 

134 ------- 

135 oid : `int` 

136 Unique integer identifier. 

137 """ 

138 tract = int(dataId['tract']) 

139 if tract < 0 or tract >= 2**DecamMapper._nbit_tract: 

140 raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract)) 

141 patchX, patchY = [int(x) for x in dataId['patch'].split(',')] 

142 for p in (patchX, patchY): 

143 if p < 0 or p >= 2**DecamMapper._nbit_patch: 

144 raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch) 

145 oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY 

146 if singleFilter: 

147 return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

148 return oid 

149 

150 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

151 return self._computeCoaddExposureId(dataId, True) 

152 

153 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

154 return 64 - DecamMapper._nbit_id 

155 

156 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

157 return self._computeCoaddExposureId(dataId, False) 

158 

159 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

160 return 64 - DecamMapper._nbit_id 

161 

162 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

163 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

164 

165 def bypass_dcrCoaddId_bits(self, *args, **kwargs): 

166 return self.bypass_deepCoaddId_bits(*args, **kwargs) 

167 

168 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

169 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

170 

171 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

172 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

173 

174 def translate_dqmask(self, dqmask): 

175 # TODO: make a class member variable that knows the mappings 

176 # below instead of hard-coding them 

177 dqmArr = dqmask.getArray() 

178 mask = afwImage.Mask(dqmask.getDimensions()) 

179 mArr = mask.getArray() 

180 idxBad = np.where(dqmArr & 1) 

181 idxSat = np.where(dqmArr & 2) 

182 idxIntrp = np.where(dqmArr & 4) 

183 idxCr = np.where(dqmArr & 16) 

184 idxBleed = np.where(dqmArr & 64) 

185 idxEdge = np.where(dqmArr & 512) 

186 mArr[idxBad] |= mask.getPlaneBitMask("BAD") 

187 mArr[idxSat] |= mask.getPlaneBitMask("SAT") 

188 mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP") 

189 mArr[idxCr] |= mask.getPlaneBitMask("CR") 

190 mArr[idxBleed] |= mask.getPlaneBitMask("SAT") 

191 mArr[idxEdge] |= mask.getPlaneBitMask("EDGE") 

192 return mask 

193 

194 def translate_wtmap(self, wtmap): 

195 wtmArr = wtmap.getArray() 

196 idxUndefWeight = np.where(wtmArr <= 0) 

197 # Reassign weights to be finite but small: 

198 wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)])) 

199 var = 1.0 / wtmArr 

200 varim = afwImage.ImageF(var) 

201 return varim 

202 

203 def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId): 

204 # Workaround until I can access the butler 

205 instcalMap = self.map_instcal(dataId) 

206 dqmaskMap = self.map_dqmask(dataId) 

207 wtmapMap = self.map_wtmap(dataId) 

208 instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1]) 

209 dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1]) 

210 wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1]) 

211 instcal = instcalType(instcalMap.getLocationsWithRoot()[0]) 

212 dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0]) 

213 wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0]) 

214 

215 mask = self.translate_dqmask(dqmask) 

216 variance = self.translate_wtmap(wtmap) 

217 

218 mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance) 

219 md = readMetadata(instcalMap.getLocationsWithRoot()[0]) 

220 fix_header(md, translator_class=DecamTranslator) 

221 wcs = makeSkyWcs(md, strip=True) 

222 exp = afwImage.ExposureF(mi, wcs) 

223 

224 exp.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(10**(0.4 * md.getScalar("MAGZERO")), 0)) 

225 visitInfo = self.makeRawVisitInfo(md=md) 

226 exp.getInfo().setVisitInfo(visitInfo) 

227 

228 for kw in ('LTV1', 'LTV2'): 

229 md.remove(kw) 

230 

231 exp.setMetadata(md) 

232 return exp 

233 

234 def std_raw(self, item, dataId): 

235 """Standardize a raw dataset by converting it to an Exposure. 

236 

237 Raw images are MEF files with one HDU for each detector. 

238 

239 Parameters 

240 ---------- 

241 item : `lsst.afw.image.DecoratedImage` 

242 The image read by the butler. 

243 dataId : data ID 

244 Data identifier. 

245 

246 Returns 

247 ------- 

248 result : `lsst.afw.image.Exposure` 

249 The standardized Exposure. 

250 """ 

251 return self._standardizeExposure(self.exposures['raw'], item, dataId, 

252 trimmed=False) 

253 

254 def _createInitialSkyWcs(self, exposure): 

255 # DECam has a coordinate system flipped on X with respect to our 

256 # VisitInfo definition of the field angle orientation. 

257 # We have to override this method until RFC-605 is implemented, to pass 

258 # `flipX=True` to createInitialSkyWcs below. 

259 self._createSkyWcsFromMetadata(exposure) 

260 

261 if exposure.getInfo().getVisitInfo() is None: 

262 msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs." 

263 self.log.warn(msg) 

264 return 

265 try: 

266 newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector(), 

267 flipX=True) 

268 exposure.setWcs(newSkyWcs) 

269 except InitialSkyWcsError as e: 

270 msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s" 

271 self.log.warn(msg, e) 

272 self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e)) 

273 if e.__context__ is not None: 

274 self.log.debug("Root-cause Exception was: %s", 

275 traceback.TracebackException.from_exception(e.__context__)) 

276 

277 def std_dark(self, item, dataId): 

278 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

279 rawPath = self.map_raw(dataId).getLocations()[0] 

280 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath) 

281 md0 = readMetadata(headerPath) 

282 fix_header(md0, translator_class=DecamTranslator) 

283 visitInfo = self.makeRawVisitInfo(md0) 

284 exp.getInfo().setVisitInfo(visitInfo) 

285 return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False) 

286 

287 def std_bias(self, item, dataId): 

288 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

289 return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False) 

290 

291 def std_flat(self, item, dataId): 

292 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

293 return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True) 

294 

295 def std_illumcor(self, item, dataId): 

296 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

297 return self._standardizeExposure(self.calibrations["illumcor"], exp, dataId, filter=True) 

298 

299 def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False): 

300 """Standardize a MasterCal image obtained from NOAO archive into Exposure 

301 

302 These MasterCal images are MEF files with one HDU for each detector. 

303 Some WCS header, eg CTYPE1, exists only in the zeroth extensionr, 

304 so info in the zeroth header need to be copied over to metadata. 

305 

306 Parameters 

307 ---------- 

308 datasetType : `str` 

309 Dataset type ("bias", "flat", or "illumcor"). 

310 item : `lsst.afw.image.DecoratedImage` 

311 The image read by the butler. 

312 dataId : data ID 

313 Data identifier. 

314 setFilter : `bool` 

315 Whether to set the filter in the Exposure. 

316 

317 Returns 

318 ------- 

319 result : `lsst.afw.image.Exposure` 

320 The standardized Exposure. 

321 """ 

322 mi = afwImage.makeMaskedImage(item.getImage()) 

323 md = item.getMetadata() 

324 masterCalMap = getattr(self, "map_" + datasetType) 

325 masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0] 

326 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath) 

327 md0 = readMetadata(headerPath) 

328 fix_header(md0, translator_class=DecamTranslator) 

329 for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2', 

330 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'): 

331 if kw in md0.paramNames() and kw not in md.paramNames(): 

332 md.add(kw, md0.getScalar(kw)) 

333 wcs = makeSkyWcs(md, strip=True) 

334 exp = afwImage.makeExposure(mi, wcs) 

335 exp.setMetadata(md) 

336 return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter) 

337 

338 def std_cpBias(self, item, dataId): 

339 return self._standardizeCpMasterCal("cpBias", item, dataId, setFilter=False) 

340 

341 def std_cpFlat(self, item, dataId): 

342 return self._standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True) 

343 

344 def std_fringe(self, item, dataId): 

345 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

346 return self._standardizeExposure(self.calibrations["fringe"], exp, dataId) 

347 

348 def std_cpIllumcor(self, item, dataId): 

349 return self._standardizeCpMasterCal("cpIllumcor", item, dataId, setFilter=True) 

350 

351 @classmethod 

352 def getLinearizerDir(cls): 

353 """Directory containing linearizers""" 

354 packageName = cls.getPackageName() 

355 packageDir = getPackageDir(packageName) 

356 return os.path.join(packageDir, "decam", "linearizer") 

357 

358 def map_linearizer(self, dataId, write=False): 

359 """Map a linearizer""" 

360 actualId = self._transformId(dataId) 

361 location = "%02d.fits" % (dataId["ccdnum"]) 

362 return ButlerLocation( 

363 pythonType="lsst.ip.isr.LinearizeSquared", 

364 cppType="Config", 

365 storageName="PickleStorage", 

366 locationList=[location], 

367 dataId=actualId, 

368 mapper=self, 

369 storage=Storage.makeFromURI(self.getLinearizerDir()) 

370 ) 

371 

372 @classmethod 

373 def getCrosstalkDir(cls): 

374 """Directory containing crosstalk tables. 

375 """ 

376 packageName = cls.getPackageName() 

377 packageDir = getPackageDir(packageName) 

378 return os.path.join(packageDir, "decam", "crosstalk")