Coverage for python/lsst/obs/decam/decamMapper.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2012-2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import re
24import traceback
26import numpy as np
28from lsst.utils import getPackageDir
29from astro_metadata_translator import fix_header, DecamTranslator
30import lsst.afw.image as afwImage
31from lsst.afw.fits import readMetadata
32from lsst.afw.geom import makeSkyWcs
33from lsst.obs.base import CameraMapper
34from lsst.obs.base.utils import createInitialSkyWcs, InitialSkyWcsError
35from lsst.daf.persistence import ButlerLocation, Storage, Policy
36from .makeDecamRawVisitInfo import MakeDecamRawVisitInfo
37from .decamFilters import DECAM_FILTER_DEFINITIONS
39np.seterr(divide="ignore")
41__all__ = ["DecamMapper"]
44class DecamMapper(CameraMapper):
45 packageName = 'obs_decam'
47 MakeRawVisitInfoClass = MakeDecamRawVisitInfo
49 detectorNames = {
50 1: 'S29', 2: 'S30', 3: 'S31', 4: 'S25', 5: 'S26', 6: 'S27', 7: 'S28', 8: 'S20', 9: 'S21',
51 10: 'S22', 11: 'S23', 12: 'S24', 13: 'S14', 14: 'S15', 15: 'S16', 16: 'S17', 17: 'S18',
52 18: 'S19', 19: 'S8', 20: 'S9', 21: 'S10', 22: 'S11', 23: 'S12', 24: 'S13', 25: 'S1', 26: 'S2',
53 27: 'S3', 28: 'S4', 29: 'S5', 30: 'S6', 31: 'S7', 32: 'N1', 33: 'N2', 34: 'N3', 35: 'N4',
54 36: 'N5', 37: 'N6', 38: 'N7', 39: 'N8', 40: 'N9', 41: 'N10', 42: 'N11', 43: 'N12', 44: 'N13',
55 45: 'N14', 46: 'N15', 47: 'N16', 48: 'N17', 49: 'N18', 50: 'N19', 51: 'N20', 52: 'N21',
56 53: 'N22', 54: 'N23', 55: 'N24', 56: 'N25', 57: 'N26', 58: 'N27', 59: 'N28', 60: 'N29',
57 62: 'N31'}
59 def __init__(self, inputPolicy=None, **kwargs):
60 policyFile = Policy.defaultPolicyFile(self.packageName, "DecamMapper.yaml", "policy")
61 policy = Policy(policyFile)
63 super(DecamMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
65 DECAM_FILTER_DEFINITIONS.defineFilters()
67 # The data ID key ccdnum is not directly used in the current policy
68 # template of the raw and instcal et al. datasets, so is not in its
69 # keyDict automatically. Add it so the butler know about the data ID key
70 # ccdnum.
71 # Similarly, add "object" for raws.
72 for datasetType in ("raw", "instcal", "dqmask", "wtmap", "cpIllumcor"):
73 self.mappings[datasetType].keyDict.update({'ccdnum': int})
74 self.mappings["raw"].keyDict.update({'object': str})
76 # The number of bits allocated for fields in object IDs
77 # TODO: This needs to be updated; also see Trac #2797
78 DecamMapper._nbit_tract = 10
79 DecamMapper._nbit_patch = 10
80 DecamMapper._nbit_filter = 4
81 DecamMapper._nbit_id = 64 - (DecamMapper._nbit_tract
82 + 2*DecamMapper._nbit_patch
83 + DecamMapper._nbit_filter)
85 def _extractDetectorName(self, dataId):
86 copyId = self._transformId(dataId)
87 try:
88 return DecamMapper.detectorNames[copyId['ccdnum']]
89 except KeyError:
90 raise RuntimeError("No name found for dataId: %s"%(dataId))
92 def _transformId(self, dataId):
93 copyId = CameraMapper._transformId(self, dataId)
94 if "ccd" in copyId:
95 copyId.setdefault("ccdnum", copyId["ccd"])
96 return copyId
98 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
99 return self._computeCcdExposureId(dataId)
101 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
102 return 32 # not really, but this leaves plenty of space for sources
104 def _computeCcdExposureId(self, dataId):
105 """Compute the 64-bit (long) identifier for a CCD exposure.
107 Parameters
108 ----------
109 dataId : `dict`
110 Data identifier with visit, ccd.
112 Returns
113 -------
114 result : `int`
115 Integer identifier for a CCD exposure.
116 """
117 copyId = self._transformId(dataId)
118 visit = copyId['visit']
119 ccdnum = copyId['ccdnum']
120 return int("%07d%02d" % (visit, ccdnum))
122 def _computeCoaddExposureId(self, dataId, singleFilter):
123 """Compute the 64-bit (long) identifier for a coadd.
125 Parameters
126 ----------
127 dataId : `dict`
128 Data identifier with tract and patch.
129 singleFilter : `bool`
130 True means the desired ID is for a single-filter coadd,
131 in which case the dataId must contain filter.
133 Returns
134 -------
135 oid : `int`
136 Unique integer identifier.
137 """
138 tract = int(dataId['tract'])
139 if tract < 0 or tract >= 2**DecamMapper._nbit_tract:
140 raise RuntimeError('tract not in range [0,%d)' % (2**DecamMapper._nbit_tract))
141 patchX, patchY = [int(x) for x in dataId['patch'].split(',')]
142 for p in (patchX, patchY):
143 if p < 0 or p >= 2**DecamMapper._nbit_patch:
144 raise RuntimeError('patch component not in range [0, %d)' % 2**DecamMapper._nbit_patch)
145 oid = (((tract << DecamMapper._nbit_patch) + patchX) << DecamMapper._nbit_patch) + patchY
146 if singleFilter:
147 return (oid << DecamMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
148 return oid
150 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
151 return self._computeCoaddExposureId(dataId, True)
153 def bypass_deepCoaddId_bits(self, *args, **kwargs):
154 return 64 - DecamMapper._nbit_id
156 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
157 return self._computeCoaddExposureId(dataId, False)
159 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
160 return 64 - DecamMapper._nbit_id
162 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
163 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
165 def bypass_dcrCoaddId_bits(self, *args, **kwargs):
166 return self.bypass_deepCoaddId_bits(*args, **kwargs)
168 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
169 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
171 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
172 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
174 def translate_dqmask(self, dqmask):
175 # TODO: make a class member variable that knows the mappings
176 # below instead of hard-coding them
177 dqmArr = dqmask.getArray()
178 mask = afwImage.Mask(dqmask.getDimensions())
179 mArr = mask.getArray()
180 idxBad = np.where(dqmArr & 1)
181 idxSat = np.where(dqmArr & 2)
182 idxIntrp = np.where(dqmArr & 4)
183 idxCr = np.where(dqmArr & 16)
184 idxBleed = np.where(dqmArr & 64)
185 idxEdge = np.where(dqmArr & 512)
186 mArr[idxBad] |= mask.getPlaneBitMask("BAD")
187 mArr[idxSat] |= mask.getPlaneBitMask("SAT")
188 mArr[idxIntrp] |= mask.getPlaneBitMask("INTRP")
189 mArr[idxCr] |= mask.getPlaneBitMask("CR")
190 mArr[idxBleed] |= mask.getPlaneBitMask("SAT")
191 mArr[idxEdge] |= mask.getPlaneBitMask("EDGE")
192 return mask
194 def translate_wtmap(self, wtmap):
195 wtmArr = wtmap.getArray()
196 idxUndefWeight = np.where(wtmArr <= 0)
197 # Reassign weights to be finite but small:
198 wtmArr[idxUndefWeight] = min(1e-14, np.min(wtmArr[np.where(wtmArr > 0)]))
199 var = 1.0 / wtmArr
200 varim = afwImage.ImageF(var)
201 return varim
203 def bypass_instcal(self, datasetType, pythonType, butlerLocation, dataId):
204 # Workaround until I can access the butler
205 instcalMap = self.map_instcal(dataId)
206 dqmaskMap = self.map_dqmask(dataId)
207 wtmapMap = self.map_wtmap(dataId)
208 instcalType = getattr(afwImage, instcalMap.getPythonType().split(".")[-1])
209 dqmaskType = getattr(afwImage, dqmaskMap.getPythonType().split(".")[-1])
210 wtmapType = getattr(afwImage, wtmapMap.getPythonType().split(".")[-1])
211 instcal = instcalType(instcalMap.getLocationsWithRoot()[0])
212 dqmask = dqmaskType(dqmaskMap.getLocationsWithRoot()[0])
213 wtmap = wtmapType(wtmapMap.getLocationsWithRoot()[0])
215 mask = self.translate_dqmask(dqmask)
216 variance = self.translate_wtmap(wtmap)
218 mi = afwImage.MaskedImageF(afwImage.ImageF(instcal.getImage()), mask, variance)
219 md = readMetadata(instcalMap.getLocationsWithRoot()[0])
220 fix_header(md, translator_class=DecamTranslator)
221 wcs = makeSkyWcs(md, strip=True)
222 exp = afwImage.ExposureF(mi, wcs)
224 exp.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(10**(0.4 * md.getScalar("MAGZERO")), 0))
225 visitInfo = self.makeRawVisitInfo(md=md)
226 exp.getInfo().setVisitInfo(visitInfo)
228 for kw in ('LTV1', 'LTV2'):
229 md.remove(kw)
231 exp.setMetadata(md)
232 return exp
234 def std_raw(self, item, dataId):
235 """Standardize a raw dataset by converting it to an Exposure.
237 Raw images are MEF files with one HDU for each detector.
239 Parameters
240 ----------
241 item : `lsst.afw.image.DecoratedImage`
242 The image read by the butler.
243 dataId : data ID
244 Data identifier.
246 Returns
247 -------
248 result : `lsst.afw.image.Exposure`
249 The standardized Exposure.
250 """
251 return self._standardizeExposure(self.exposures['raw'], item, dataId,
252 trimmed=False)
254 def _createInitialSkyWcs(self, exposure):
255 # DECam has a coordinate system flipped on X with respect to our
256 # VisitInfo definition of the field angle orientation.
257 # We have to override this method until RFC-605 is implemented, to pass
258 # `flipX=True` to createInitialSkyWcs below.
259 self._createSkyWcsFromMetadata(exposure)
261 if exposure.getInfo().getVisitInfo() is None:
262 msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs."
263 self.log.warn(msg)
264 return
265 try:
266 newSkyWcs = createInitialSkyWcs(exposure.getInfo().getVisitInfo(), exposure.getDetector(),
267 flipX=True)
268 exposure.setWcs(newSkyWcs)
269 except InitialSkyWcsError as e:
270 msg = "Cannot create SkyWcs using VisitInfo and Detector, using metadata-based SkyWcs: %s"
271 self.log.warn(msg, e)
272 self.log.debug("Exception was: %s", traceback.TracebackException.from_exception(e))
273 if e.__context__ is not None:
274 self.log.debug("Root-cause Exception was: %s",
275 traceback.TracebackException.from_exception(e.__context__))
277 def std_dark(self, item, dataId):
278 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
279 rawPath = self.map_raw(dataId).getLocations()[0]
280 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", rawPath)
281 md0 = readMetadata(headerPath)
282 fix_header(md0, translator_class=DecamTranslator)
283 visitInfo = self.makeRawVisitInfo(md0)
284 exp.getInfo().setVisitInfo(visitInfo)
285 return self._standardizeExposure(self.calibrations["dark"], exp, dataId, filter=False)
287 def std_bias(self, item, dataId):
288 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
289 return self._standardizeExposure(self.calibrations["bias"], exp, dataId, filter=False)
291 def std_flat(self, item, dataId):
292 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
293 return self._standardizeExposure(self.calibrations["flat"], exp, dataId, filter=True)
295 def std_illumcor(self, item, dataId):
296 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
297 return self._standardizeExposure(self.calibrations["illumcor"], exp, dataId, filter=True)
299 def _standardizeCpMasterCal(self, datasetType, item, dataId, setFilter=False):
300 """Standardize a MasterCal image obtained from NOAO archive into Exposure
302 These MasterCal images are MEF files with one HDU for each detector.
303 Some WCS header, eg CTYPE1, exists only in the zeroth extensionr,
304 so info in the zeroth header need to be copied over to metadata.
306 Parameters
307 ----------
308 datasetType : `str`
309 Dataset type ("bias", "flat", or "illumcor").
310 item : `lsst.afw.image.DecoratedImage`
311 The image read by the butler.
312 dataId : data ID
313 Data identifier.
314 setFilter : `bool`
315 Whether to set the filter in the Exposure.
317 Returns
318 -------
319 result : `lsst.afw.image.Exposure`
320 The standardized Exposure.
321 """
322 mi = afwImage.makeMaskedImage(item.getImage())
323 md = item.getMetadata()
324 masterCalMap = getattr(self, "map_" + datasetType)
325 masterCalPath = masterCalMap(dataId).getLocationsWithRoot()[0]
326 headerPath = re.sub(r'[\[](\d+)[\]]$', "[0]", masterCalPath)
327 md0 = readMetadata(headerPath)
328 fix_header(md0, translator_class=DecamTranslator)
329 for kw in ('CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CUNIT1', 'CUNIT2',
330 'CD1_1', 'CD1_2', 'CD2_1', 'CD2_2'):
331 if kw in md0.paramNames() and kw not in md.paramNames():
332 md.add(kw, md0.getScalar(kw))
333 wcs = makeSkyWcs(md, strip=True)
334 exp = afwImage.makeExposure(mi, wcs)
335 exp.setMetadata(md)
336 return self._standardizeExposure(self.calibrations[datasetType], exp, dataId, filter=setFilter)
338 def std_cpBias(self, item, dataId):
339 return self._standardizeCpMasterCal("cpBias", item, dataId, setFilter=False)
341 def std_cpFlat(self, item, dataId):
342 return self._standardizeCpMasterCal("cpFlat", item, dataId, setFilter=True)
344 def std_fringe(self, item, dataId):
345 exp = afwImage.makeExposure(afwImage.makeMaskedImage(item))
346 return self._standardizeExposure(self.calibrations["fringe"], exp, dataId)
348 def std_cpIllumcor(self, item, dataId):
349 return self._standardizeCpMasterCal("cpIllumcor", item, dataId, setFilter=True)
351 @classmethod
352 def getLinearizerDir(cls):
353 """Directory containing linearizers"""
354 packageName = cls.getPackageName()
355 packageDir = getPackageDir(packageName)
356 return os.path.join(packageDir, "decam", "linearizer")
358 def map_linearizer(self, dataId, write=False):
359 """Map a linearizer"""
360 actualId = self._transformId(dataId)
361 location = "%02d.fits" % (dataId["ccdnum"])
362 return ButlerLocation(
363 pythonType="lsst.ip.isr.LinearizeSquared",
364 cppType="Config",
365 storageName="PickleStorage",
366 locationList=[location],
367 dataId=actualId,
368 mapper=self,
369 storage=Storage.makeFromURI(self.getLinearizerDir())
370 )