Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

# 

# LSST Data Management System 

# Copyright 2012-2016 LSST Corporation. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

import os 

import re 

 

import numpy as np 

 

from lsst.utils import getPackageDir 

import lsst.afw.image as afwImage 

import lsst.afw.image.utils as afwImageUtils 

from lsst.afw.fits import readMetadata 

from lsst.afw.geom import makeSkyWcs 

from lsst.obs.base import CameraMapper, exposureFromImage 

from lsst.daf.persistence import ButlerLocation, Storage, Policy 

import lsst.ip.isr as isr 

from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo 

 

np.seterr(divide="ignore") 

 

 

class DecamMapper(CameraMapper): 

packageName = 'obs_decam' 

 

MakeRawVisitInfoClass = MakeDecamRawVisitInfo 

 

detectorNames = { 

1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21', 

10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18', 

18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2', 

27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4', 

36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13', 

45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21', 

53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29', 

62: 'N31'} 

 

def __init__(self, inputPolicy=None, **kwargs): 

policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy") 

policy = Policy(policyFile) 

 

super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

 

# lambdaMin and lambda max are chosen to be where the filter rises above 1% 

# from http://www.ctio.noao.edu/noao/sites/default/files/DECam/DECam_filters_transmission.txt 

afwImageUtils.defineFilter('u', lambdaEff=350, lambdaMin=305, lambdaMax=403, 

alias=['u DECam c0006 3500.0 1000.0']) 

afwImageUtils.defineFilter('g', lambdaEff=450, lambdaMin=394, lambdaMax=555, 

alias=['g DECam SDSS c0001 4720.0 1520.0']) 

afwImageUtils.defineFilter('r', lambdaEff=600, lambdaMin=562, lambdaMax=725, 

alias=['r DECam SDSS c0002 6415.0 1480.0']) 

afwImageUtils.defineFilter('i', lambdaEff=750, lambdaMin=699, lambdaMax=870, 

alias=['i DECam SDSS c0003 7835.0 1470.0']) 

afwImageUtils.defineFilter('z', lambdaEff=900, lambdaMin=837, lambdaMax=1016, 

alias=['z DECam SDSS c0004 9260.0 1520.0']) 

afwImageUtils.defineFilter('y', lambdaEff=1000, lambdaMin=941, lambdaMax=1080, 

alias=['Y DECam c0005 10095.0 1130.0', 'Y']) 

afwImageUtils.defineFilter('VR', lambdaEff=630, lambdaMin=490, lambdaMax=765, 

alias=['VR DECam c0007 6300.0 2600.0']) 

afwImageUtils.defineFilter('N964', lambdaEff=964, alias=['N964 DECam c0008 9645.0 94.0']) 

afwImageUtils.defineFilter('SOLID', lambdaEff=0, alias=['solid']) 

 

# The data ID key ccdnum is not directly used in the current policy 

# template of the raw and instcal et al. datasets, so is not in its 

# keyDict automatically. Add it so the butler know about the data ID key 

# ccdnum. 

for datasetType in ("raw", "instcal", "dqmask", "wtmap"): 

self.mappings[datasetType].keyDict.update({'ccdnum': int}) 

self.mappings["raw"].keyDict.update({'object': str}) 

 

# The number of bits allocated for fields in object IDs 

# TODO: This needs to be updated; also see Trac #2797 

DecamMapper._nbit_tract = 10 

DecamMapper._nbit_patch = 10 

DecamMapper._nbit_filter = 4 

DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract + 

2*DecamMapper._nbit_patch + 

DecamMapper._nbit_filter) 

 

def _extractDetectorName(self, dataId): 

copyId = self._transformId(dataId) 

try: 

return DecamMapper.detectorNames[copyId['ccdnum']] 

except KeyError: 

raise RuntimeError("No name found for dataId: %s"%(dataId)) 

 

def _transformId(self, dataId): 

copyId = CameraMapper._transformId(self, dataId) 

106 ↛ 107line 106 didn't jump to line 107, because the condition on line 106 was never true if "ccd" in copyId: 

copyId.setdefault("ccdnum", copyId["ccd"]) 

return copyId 

 

def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

return self._computeCcdExposureId(dataId) 

 

def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

return 32 # not really, but this leaves plenty of space for sources 

 

def _computeCcdExposureId(self, dataId): 

"""Compute the 64-bit (long) identifier for a CCD exposure. 

 

@param dataId (dict) Data identifier with visit, ccd 

""" 

copyId = self._transformId(dataId) 

visit = copyId['visit'] 

ccdnum = copyId['ccdnum'] 

return int("%07d%02d" % (visit, ccdnum)) 

 

def _computeCoaddExposureId(self, dataId, singleFilter): 

"""Compute the 64-bit (long) identifier for a coadd. 

 

@param dataId (dict) Data identifier with tract and patch. 

@param singleFilter (bool) True means the desired ID is for a single- 

filter coadd, in which case dataId 

must contain filter. 

""" 

tract = int(dataId['tract']) 

if tract < 0 or tract >= 2**DecamMapper._nbit_tract: 

raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract)) 

patchX, patchY = [int(x) for x in dataId['patch'].split(',')] 

for p in (patchX, patchY): 

if p < 0 or p >= 2**DecamMapper._nbit_patch: 

raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch) 

oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY 

if singleFilter: 

return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

return oid 

 

def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

return self._computeCoaddExposureId(dataId, True) 

 

def bypass_deepCoaddId_bits(self, *args, **kwargs): 

return 64 - DecamMapper._nbit_id 

 

def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

return self._computeCoaddExposureId(dataId, False) 

 

def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

return 64 - DecamMapper._nbit_id 

 

def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

 

def bypass_dcrCoaddId_bits(self, *args, **kwargs): 

return self.bypass_deepCoaddId_bits(*args, **kwargs) 

 

def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

 

def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

 

def translate_dqmask(self, dqmask): 

# TODO: make a class member variable that knows the mappings 

# below instead of hard-coding them 

dqmArr = dqmask.getArray() 

mask = afwImage.Mask(dqmask.getDimensions()) 

mArr = mask.getArray() 

idxBad = np.where(dqmArr & 1) 

idxSat = np.where(dqmArr & 2) 

idxIntrp = np.where(dqmArr & 4) 

idxCr = np.where(dqmArr & 16) 

idxBleed = np.where(dqmArr & 64) 

idxEdge = np.where(dqmArr & 512) 

mArr[idxBad] |= mask.getPlaneBitMask("BAD") 

mArr[idxSat] |= mask.getPlaneBitMask("SAT") 

mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP") 

mArr[idxCr] |= mask.getPlaneBitMask("CR") 

mArr[idxBleed] |= mask.getPlaneBitMask("SAT") 

mArr[idxEdge] |= mask.getPlaneBitMask("EDGE") 

return mask 

 

def translate_wtmap(self, wtmap): 

wtmArr = wtmap.getArray() 

idxUndefWeight = np.where(wtmArr <= 0) 

# Reassign weights to be finite but small: 

wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)])) 

var = 1.0 / wtmArr 

varim = afwImage.ImageF(var) 

return varim 

 

def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId): 

# Workaround until I can access the butler 

instcalMap = self.map_instcal(dataId) 

dqmaskMap = self.map_dqmask(dataId) 

wtmapMap = self.map_wtmap(dataId) 

instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1]) 

dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1]) 

wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1]) 

instcal = instcalType(instcalMap.getLocationsWithRoot()[0]) 

dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0]) 

wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0]) 

 

mask = self.translate_dqmask(dqmask) 

variance = self.translate_wtmap(wtmap) 

 

mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance) 

md = instcal.getMetadata() 

wcs = makeSkyWcs(md, strip=True) 

exp = afwImage.ExposureF(mi, wcs) 

 

# Set the calib by hand; need to grab the zeroth extension 

header = re.sub(r'[\[](\d+)[\]]$', "[0]", instcalMap.getLocationsWithRoot()[0]) 

md0 = readMetadata(header) 

calib = afwImage.Calib() 

calib.setFluxMag0(10**(0.4 * md0.get("MAGZERO"))) 

exp.setCalib(calib) 

exposureId = self._computeCcdExposureId(dataId) 

visitInfo = self.makeRawVisitInfo(md=md0, exposureId=exposureId) 

exp.getInfo().setVisitInfo(visitInfo) 

 

for kw in ('LTV1', 'LTV2'): 

md.remove(kw) 

 

exp.setMetadata(md) 

return exp 

 

def std_raw(self, item, dataId): 

"""Standardize a raw dataset by converting it to an Exposure. 

 

Raw images are MEF files with one HDU for each detector. 

Header keywords EXPTIME and MJD-OBS exist only in the zeroth 

extension and are copied to metadata. 

 

@param item: The image read by the butler 

@param dataId: Data identifier 

@return (lsst.afw.image.Exposure) the standardized Exposure 

""" 

# Convert the raw DecoratedImage to an Exposure, set metadata and wcs. 

exp = exposureFromImage(item, logger=self.log) 

md = exp.getMetadata() 

rawPath = self.map_raw(dataId).getLocationsWithRoot()[0] 

headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath) 

md0 = readMetadata(headerPath) 

# extra keywords to copy to the exposure 

for kw in ('DARKTIME', ): 

if kw in md0.paramNames() and kw not in md.paramNames(): 

md.add(kw, md0.get(kw)) 

exposureId = self._computeCcdExposureId(dataId) 

visitInfo = self.makeRawVisitInfo(md=md0, exposureId=exposureId) 

exp.getInfo().setVisitInfo(visitInfo) 

 

# Standardize an Exposure, including setting the calib object 

return self._standardizeExposure(self.exposures['raw'], exp, dataId, 

trimmed=False) 

 

def std_dark(self, item, dataId): 

exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

rawPath = self.map_raw(dataId).getLocations()[0] 

headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath) 

md0 = readMetadata(headerPath) 

visitInfo = self.makeRawVisitInfo(md0) 

exp.getInfo().setVisitInfo(visitInfo) 

return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False) 

 

def std_bias(self, item, dataId): 

exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False) 

 

def std_flat(self, item, dataId): 

exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True) 

 

def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False): 

"""Standardize a MasterCal image obtained from NOAO archive into Exposure 

 

These MasterCal images are MEF files with one HDU for each detector. 

Some WCS header, eg CTYPE1, exists only in the zeroth extensionr, 

so info in the zeroth header need to be copied over to metadata. 

 

@param datasetType: Dataset type ("bias" or "flat") 

@param item: The image read by the butler 

@param dataId: Data identifier 

@param setFilter: Whether to set the filter in the Exposure 

@return (lsst.afw.image.Exposure) the standardized Exposure 

""" 

mi = afwImage.makeMaskedImage(item.getImage()) 

md = item.getMetadata() 

masterCalMap = getattr(self, "map_" + datasetType) 

masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0] 

headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath) 

md0 = readMetadata(headerPath) 

for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2', 

'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'): 

if kw in md0.paramNames() and kw not in md.paramNames(): 

md.add(kw, md0.get(kw)) 

wcs = makeSkyWcs(md, strip=True) 

exp = afwImage.makeExposure(mi, wcs) 

exp.setMetadata(md) 

return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter) 

 

def std_cpBias(self, item, dataId): 

return self._standardizeCpMasterCal("cpBias", item, dataId, setFilter=False) 

 

def std_cpFlat(self, item, dataId): 

return self._standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True) 

 

def std_fringe(self, item, dataId): 

exp = afwImage.makeExposure(afwImage.makeMaskedImage(item)) 

return self._standardizeExposure(self.calibrations["fringe"], exp, dataId) 

 

def map_defects(self, dataId, write=False): 

"""Map defects dataset with the calibration registry. 

 

Overriding the method so to use CalibrationMapping policy, 

instead of looking up the path in defectRegistry as currently 

implemented in CameraMapper. 

 

@param dataId (dict) Dataset identifier 

@return daf.persistence.ButlerLocation 

""" 

return self.mappings["defects"].map(self, dataId=dataId, write=write) 

 

def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

"""Return a defect list based on butlerLocation returned by map_defects. 

 

Use all nonzero pixels in the Community Pipeline Bad Pixel Masks. 

 

@param[in] butlerLocation: Butler Location with path to defects FITS 

@param[in] dataId: data identifier 

@return meas.algorithms.DefectListT 

""" 

bpmFitsPath = butlerLocation.getLocationsWithRoot()[0] 

bpmImg = afwImage.ImageU(bpmFitsPath) 

idxBad = np.nonzero(bpmImg.getArray()) 

mim = afwImage.MaskedImageU(bpmImg.getDimensions()) 

mim.getMask().getArray()[idxBad] |= mim.getMask().getPlaneBitMask("BAD") 

return isr.getDefectListFromMask(mim, "BAD") 

 

def std_defects(self, item, dataId): 

"""Return the defect list as it is. 

 

Do not standardize it to Exposure. 

""" 

return item 

 

@classmethod 

def getLinearizerDir(cls): 

"""Directory containing linearizers""" 

packageName = cls.getPackageName() 

packageDir = getPackageDir(packageName) 

return os.path.join(packageDir, "decam", "linearizer") 

 

def map_linearizer(self, dataId, write=False): 

"""Map a linearizer""" 

actualId = self._transformId(dataId) 

location = "%02d.fits" % (dataId["ccdnum"]) 

return ButlerLocation( 

pythonType="lsst.ip.isr.LinearizeSquared", 

cppType="Config", 

storageName="PickleStorage", 

locationList=[location], 

dataId=actualId, 

mapper=self, 

storage=Storage.makeFromURI(self.getLinearizerDir()) 

)