Coverage for python/lsst/obs/decam/decamMapper.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2012-2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import re
24import traceback
26import numpy as np
28from lsst.utils import getPackageDir
29from astro_metadata_translator import fix_header, DecamTranslator
30import lsst.afw.image as afwImage
31from lsst.afw.fits import readMetadata
32from lsst.afw.geom import makeSkyWcs
33from lsst.obs.base import CameraMapper
34from lsst.obs.base.utils import createInitialSkyWcs, InitialSkyWcsError
35from lsst.daf.persistence import ButlerLocation, Storage, Policy
36from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo
37from .decamFilters import DECAM_FILTER_DEFINITIONS
38from ._instrument import DarkEnergyCamera
40np.seterr(divide="ignore")
42__all__ = ["DecamMapper"]
45class DecamMapper(CameraMapper):
46 packageName = 'obs_decam'
47 _gen3instrument = DarkEnergyCamera
49 MakeRawVisitInfoClass = MakeDecamRawVisitInfo
51 detectorNames = {
52 1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21',
53 10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18',
54 18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2',
55 27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4',
56 36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13',
57 45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21',
58 53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29',
59 62: 'N31'}
61 def __init__(self, inputPolicy=None, **kwargs):
62 policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy")
63 policy = Policy(policyFile)
65 super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
67 DECAM_FILTER_DEFINITIONS.defineFilters()
69 # The data ID key ccdnum is not directly used in the current policy
70 # template of the raw and instcal et al. datasets, so is not in its
71 # keyDict automatically. Add it so the butler know about the data ID key
72 # ccdnum.
73 # Similarly, add "object" for raws.
74 for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"):
75 self.mappings[datasetType].keyDict.update({'ccdnum': int})
76 self.mappings["raw"].keyDict.update({'object': str})
78 # The number of bits allocated for fields in object IDs
79 # TODO: This needs to be updated; also see Trac #2797
80 DecamMapper._nbit_tract = 10
81 DecamMapper._nbit_patch = 10
82 DecamMapper._nbit_filter = 4
83 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract
84 + 2*DecamMapper._nbit_patch
85 + DecamMapper._nbit_filter)
87 def _extractDetectorName(self, dataId):
88 copyId = self._transformId(dataId)
89 try:
90 return DecamMapper.detectorNames[copyId['ccdnum']]
91 except KeyError:
92 raise RuntimeError("No name found for dataId: %s"%(dataId))
94 def _transformId(self, dataId):
95 copyId = CameraMapper._transformId(self, dataId)
96 if "ccd" in copyId:
97 copyId.setdefault("ccdnum", copyId["ccd"])
98 return copyId
100 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
101 return self._computeCcdExposureId(dataId)
103 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
104 return 32 # not really, but this leaves plenty of space for sources
106 def _computeCcdExposureId(self, dataId):
107 """Compute the 64-bit (long) identifier for a CCD exposure.
109 Parameters
110 ----------
111 dataId : `dict`
112 Data identifier with visit, ccd.
114 Returns
115 -------
116 result : `int`
117 Integer identifier for a CCD exposure.
118 """
119 copyId = self._transformId(dataId)
120 visit = copyId['visit']
121 ccdnum = copyId['ccdnum']
122 return int("%07d%02d" % (visit, ccdnum))
124 def _computeCoaddExposureId(self, dataId, singleFilter):
125 """Compute the 64-bit (long) identifier for a coadd.
127 Parameters
128 ----------
129 dataId : `dict`
130 Data identifier with tract and patch.
131 singleFilter : `bool`
132 True means the desired ID is for a single-filter coadd,
133 in which case the dataId must contain filter.
135 Returns
136 -------
137 oid : `int`
138 Unique integer identifier.
139 """
140 tract = int(dataId['tract'])
141 if tract < 0 or tract >= 2**DecamMapper._nbit_tract:
142 raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract))
143 patchX, patchY = [int(x) for x in dataId['patch'].split(',')]
144 for p in (patchX, patchY):
145 if p < 0 or p >= 2**DecamMapper._nbit_patch:
146 raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch)
147 oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY
148 if singleFilter:
149 return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
150 return oid
152 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
153 return self._computeCoaddExposureId(dataId, True)
155 def bypass_deepCoaddId_bits(self, *args, **kwargs):
156 return 64 - DecamMapper._nbit_id
158 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
159 return self._computeCoaddExposureId(dataId, False)
161 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
162 return 64 - DecamMapper._nbit_id
164 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
165 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
167 def bypass_dcrCoaddId_bits(self, *args, **kwargs):
168 return self.bypass_deepCoaddId_bits(*args, **kwargs)
170 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
171 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
173 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
174 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
176 def translate_dqmask(self, dqmask):
177 # TODO: make a class member variable that knows the mappings
178 # below instead of hard-coding them
179 dqmArr = dqmask.getArray()
180 mask = afwImage.Mask(dqmask.getDimensions())
181 mArr = mask.getArray()
182 idxBad = np.where(dqmArr & 1)
183 idxSat = np.where(dqmArr & 2)
184 idxIntrp = np.where(dqmArr & 4)
185 idxCr = np.where(dqmArr & 16)
186 idxBleed = np.where(dqmArr & 64)
187 idxEdge = np.where(dqmArr & 512)
188 mArr[idxBad] |= mask.getPlaneBitMask("BAD")
189 mArr[idxSat] |= mask.getPlaneBitMask("SAT")
190 mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP")
191 mArr[idxCr] |= mask.getPlaneBitMask("CR")
192 mArr[idxBleed] |= mask.getPlaneBitMask("SAT")
193 mArr[idxEdge] |= mask.getPlaneBitMask("EDGE")
194 return mask
196 def translate_wtmap(self, wtmap):
197 wtmArr = wtmap.getArray()
198 idxUndefWeight = np.where(wtmArr <= 0)
199 # Reassign weights to be finite but small:
200 wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)]))
201 var = 1.0 / wtmArr
202 varim = afwImage.ImageF(var)
203 return varim
205 def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
206 # Workaround until I can access the butler
207 instcalMap = self.map_instcal(dataId)
208 dqmaskMap = self.map_dqmask(dataId)
209 wtmapMap = self.map_wtmap(dataId)
210 instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1])
211 dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1])
212 wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
213 instcal = instcalType(instcalMap.getLocationsWithRoot()[0])
214 dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0])
215 wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0])
217 mask = self.translate_dqmask(dqmask)
218 variance = self.translate_wtmap(wtmap)
220 mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance)
221 md = readMetadata(instcalMap.getLocationsWithRoot()[0])
222 fix_header(md, translator_class=DecamTranslator)
223 wcs = makeSkyWcs(md, strip=True)
224 exp = afwImage.ExposureF(mi, wcs)
226 exp.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(10**(0.4 * md.getScalar("MAGZERO")), 0))
227 visitInfo = self.makeRawVisitInfo(md=md)
228 exp.getInfo().setVisitInfo(visitInfo)
230 for kw in ('LTV1', 'LTV2'):
231 md.remove(kw)
233 exp.setMetadata(md)
234 return exp
236 def std_raw(self, item, dataId):
237 """Standardize a raw dataset by converting it to an Exposure.
239 Raw images are MEF files with one HDU for each detector.
241 Parameters
242 ----------
243 item : `lsst.afw.image.DecoratedImage`
244 The image read by the butler.
245 dataId : data ID
246 Data identifier.
248 Returns
249 -------
250 result : `lsst.afw.image.Exposure`
251 The standardized Exposure.
252 """
253 return self._standardizeExposure(self.exposures['raw'], item, dataId,
254 trimmed=False)
256 def _createInitialSkyWcs(self, exposure):
257 # DECam has a coordinate system flipped on X with respect to our
258 # VisitInfo definition of the field angle orientation.
259 # We have to override this method until RFC-605 is implemented, to pass
260 # `flipX=True` to createInitialSkyWcs below.
261 self._createSkyWcsFromMetadata(exposure)
263 if exposure.getInfo().getVisitInfo() is None:
264 msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs."
265 self.log.warn(msg)
266 return
267 try:
268 newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector(),
269 flipX=True)
270 exposure.setWcs(newSkyWcs)
271 except InitialSkyWcsError as e:
272 msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s"
273 self.log.warn(msg, e)
274 self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e))
275 if e.__context__ is not None:
276 self.log.debug("Root-cause Exception was: %s",
277 traceback.TracebackException.from_exception(e.__context__))
279 def std_dark(self, item, dataId):
280 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
281 rawPath = self.map_raw(dataId).getLocations()[0]
282 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath)
283 md0 = readMetadata(headerPath)
284 fix_header(md0, translator_class=DecamTranslator)
285 visitInfo = self.makeRawVisitInfo(md0)
286 exp.getInfo().setVisitInfo(visitInfo)
287 return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False)
289 def std_bias(self, item, dataId):
290 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
291 return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False)
293 def std_flat(self, item, dataId):
294 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
295 return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True)
297 def std_illumcor(self, item, dataId):
298 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
299 return self._standardizeExposure(self.calibrations["illumcor"], exp, dataId, filter=True)
301 def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False):
302 """Standardize a MasterCal image obtained from NOAO archive into Exposure
304 These MasterCal images are MEF files with one HDU for each detector.
305 Some WCS header, eg CTYPE1, exists only in the zeroth extensionr,
306 so info in the zeroth header need to be copied over to metadata.
308 Parameters
309 ----------
310 datasetType : `str`
311 Dataset type ("bias", "flat", or "illumcor").
312 item : `lsst.afw.image.DecoratedImage`
313 The image read by the butler.
314 dataId : data ID
315 Data identifier.
316 setFilter : `bool`
317 Whether to set the filter in the Exposure.
319 Returns
320 -------
321 result : `lsst.afw.image.Exposure`
322 The standardized Exposure.
323 """
324 mi = afwImage.makeMaskedImage(item.getImage())
325 md = item.getMetadata()
326 masterCalMap = getattr(self, "map_" + datasetType)
327 masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0]
328 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath)
329 md0 = readMetadata(headerPath)
330 fix_header(md0, translator_class=DecamTranslator)
331 for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2',
332 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'):
333 if kw in md0.paramNames() and kw not in md.paramNames():
334 md.add(kw, md0.getScalar(kw))
335 wcs = makeSkyWcs(md, strip=True)
336 exp = afwImage.makeExposure(mi, wcs)
337 exp.setMetadata(md)
338 return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter)
340 def std_cpBias(self, item, dataId):
341 return self._standardizeCpMasterCal("cpBias", item, dataId, setFilter=False)
343 def std_cpFlat(self, item, dataId):
344 return self._standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True)
346 def std_fringe(self, item, dataId):
347 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
348 return self._standardizeExposure(self.calibrations["fringe"], exp, dataId)
350 def std_cpIllumcor(self, item, dataId):
351 return self._standardizeCpMasterCal("cpIllumcor", item, dataId, setFilter=True)
353 @classmethod
354 def getLinearizerDir(cls):
355 """Directory containing linearizers"""
356 packageName = cls.getPackageName()
357 packageDir = getPackageDir(packageName)
358 return os.path.join(packageDir, "decam", "linearizer")
360 def map_linearizer(self, dataId, write=False):
361 """Map a linearizer"""
362 actualId = self._transformId(dataId)
363 location = "%02d.fits" % (dataId["ccdnum"])
364 return ButlerLocation(
365 pythonType="lsst.ip.isr.LinearizeSquared",
366 cppType="Config",
367 storageName="PickleStorage",
368 locationList=[location],
369 dataId=actualId,
370 mapper=self,
371 storage=Storage.makeFromURI(self.getLinearizerDir())
372 )
374 @classmethod
375 def getCrosstalkDir(cls):
376 """Directory containing crosstalk tables.
377 """
378 packageName = cls.getPackageName()
379 packageDir = getPackageDir(packageName)
380 return os.path.join(packageDir, "decam", "crosstalk")