Coverage for python/lsst/obs/lsstSim/lsstSimMapper.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012, 2013 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23__all__ = ["LsstSimMapper"]
25import os
26import re
27from astropy.io import fits
29import lsst.daf.base as dafBase
30import lsst.afw.image.utils as afwImageUtils
31import lsst.geom as geom
32import lsst.daf.persistence as dafPersist
33from lsst.meas.algorithms import Defects
34from .makeLsstSimRawVisitInfo import MakeLsstSimRawVisitInfo
35from lsst.utils import getPackageDir
37from lsst.obs.base import CameraMapper
39# Solely to get boost serialization registrations for Measurement subclasses
42class LsstSimMapper(CameraMapper):
43 packageName = 'obs_lsstSim'
45 MakeRawVisitInfoClass = MakeLsstSimRawVisitInfo
47 _CcdNameRe = re.compile(r"R:(\d,\d) S:(\d,\d(?:,[AB])?)$")
49 def __init__(self, inputPolicy=None, **kwargs):
50 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, "LsstSimMapper.yaml", "policy")
51 policy = dafPersist.Policy(policyFile)
52 repositoryDir = os.path.join(getPackageDir(self.packageName), 'policy')
53 self.defectRegistry = None
54 if 'defects' in policy:
55 self.defectPath = os.path.join(repositoryDir, policy['defects'])
56 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
57 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
59 self.doFootprints = False
60 if inputPolicy is not None:
61 for kw in inputPolicy.paramNames(True):
62 if kw == "doFootprints":
63 self.doFootprints = True
64 else:
65 kwargs[kw] = inputPolicy.get(kw)
67 super(LsstSimMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
68 self.filterIdMap = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5, 'i2': 5}
70 # The LSST Filters from L. Jones 04/07/10
71 afwImageUtils.resetFilters()
72 afwImageUtils.defineFilter('u', lambdaEff=364.59, lambdaMin=324.0, lambdaMax=395.0)
73 afwImageUtils.defineFilter('g', lambdaEff=476.31, lambdaMin=405.0, lambdaMax=552.0)
74 afwImageUtils.defineFilter('r', lambdaEff=619.42, lambdaMin=552.0, lambdaMax=691.0)
75 afwImageUtils.defineFilter('i', lambdaEff=752.06, lambdaMin=818.0, lambdaMax=921.0)
76 afwImageUtils.defineFilter('z', lambdaEff=866.85, lambdaMin=922.0, lambdaMax=997.0)
77 # official y filter
78 afwImageUtils.defineFilter('y', lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4'])
79 # If/when y3 sim data becomes available, uncomment this and
80 # modify the schema appropriately
81 # afwImageUtils.defineFilter('y3', 1002.44) # candidate y-band
83 def _transformId(self, dataId):
84 """Transform an ID dict into standard form for LSST
86 Standard keys are as follows:
87 - raft: in the form <x>,<y>
88 - sensor: in the form <x>,<y>,<c> where <c> = A or B
89 - channel: in the form <x>,<y>
90 - snap: exposure number
92 Other supported keys, which are used to set the above, if not already set:
93 - ccd: an alias for sensor (hence NOT the full ccd name)
94 - ccdName or sensorName: full ccd name in the form R:<x>,<y> S:<x>,<y>[,<c>]
95 if found, used to set raft and sensor, if not already set
96 - channelName, ampName: an alternate way to specify channel, in the form: IDxx
97 - amp: an alias for channel
98 - exposure: an alias for snap
100 @param dataId[in] (dict) Dataset identifier; this must not be modified
101 @return (dict) Transformed dataset identifier
102 @raise RuntimeError if a value is not valid
103 """
104 actualId = dataId.copy()
105 for ccdAlias in ("ccdName", "sensorName"):
106 if ccdAlias in actualId:
107 ccdName = actualId[ccdAlias].upper()
108 m = self._CcdNameRe.match(ccdName)
109 if m is None:
110 raise RuntimeError("Invalid value for %s: %r" % (ccdAlias, ccdName))
111 actualId.setdefault("raft", m.group(1))
112 actualId.setdefault("sensor", m.group(2))
113 break
114 if "ccd" in actualId:
115 actualId.setdefault("sensor", actualId["ccd"])
116 if "amp" in actualId:
117 actualId.setdefault("channel", actualId["amp"])
118 elif "channel" not in actualId:
119 for ampName in ("ampName", "channelName"):
120 if ampName in actualId:
121 m = re.match(r'ID(\d+)$', actualId[ampName])
122 channelNumber = int(m.group(1))
123 channelX = channelNumber % 8
124 channelY = channelNumber // 8
125 actualId['channel'] = str(channelX) + "," + str(channelY)
126 break
127 if "exposure" in actualId:
128 actualId.setdefault("snap", actualId["exposure"])
130 # why strip out the commas after carefully adding them?
131 if "raft" in actualId:
132 actualId['raft'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['raft'])
133 if "sensor" in actualId:
134 actualId['sensor'] = actualId['sensor'].replace(",", "")
135 if "channel" in actualId:
136 actualId['channel'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['channel'])
137 return actualId
139 def validate(self, dataId):
140 for component in ("raft", "sensor", "channel"):
141 if component not in dataId:
142 continue
143 val = dataId[component]
144 if not isinstance(val, str):
145 raise RuntimeError(
146 "%s identifier should be type str, not %s: %r" % (component.title(), type(val), val))
147 if component == "sensor":
148 if not re.search(r'^\d,\d(,[AB])?$', val):
149 raise RuntimeError("Invalid %s identifier: %r" % (component, val))
150 else:
151 if not re.search(r'^(\d),(\d)$', val):
152 raise RuntimeError("Invalid %s identifier: %r" % (component, val))
153 return dataId
155 def _extractDetectorName(self, dataId):
156 return "R:%(raft)s S:%(sensor)s" % dataId
158 def getDataId(self, visit, ccdId):
159 """get dataId dict from visit and ccd identifier
161 @param visit 32 or 64-bit depending on camera
162 @param ccdId detector name: same as detector.getName()
163 """
164 dataId = {'visit': int(visit)}
165 m = self._CcdNameRe.match(ccdId)
166 if m is None:
167 raise RuntimeError("Cannot parse ccdId=%r" % (ccdId,))
168 dataId['raft'] = m.group(0)
169 dataId['sensor'] = m.group(1)
170 return dataId
172 def _computeAmpExposureId(self, dataId):
173 # visit, snap, raft, sensor, channel):
174 """Compute the 64-bit (long) identifier for an amp exposure.
176 @param dataId (dict) Data identifier with visit, snap, raft, sensor, channel
177 """
179 pathId = self._transformId(dataId)
180 visit = pathId['visit']
181 snap = pathId['snap']
182 raft = pathId['raft'] # "xy" e.g. "20"
183 sensor = pathId['sensor'] # "xy" e.g. "11"
184 channel = pathId['channel'] # "yx" e.g. "05" (NB: yx, not xy, in original comment)
186 r1, r2 = raft
187 s1, s2 = sensor
188 c1, c2 = channel
189 return (visit << 13) + (snap << 12) + \
190 (int(r1) * 5 + int(r2)) * 160 + \
191 (int(s1) * 3 + int(s2)) * 16 + \
192 (int(c1) * 8 + int(c2))
194 def _computeCcdExposureId(self, dataId):
195 """Compute the 64-bit (long) identifier for a CCD exposure.
197 @param dataId (dict) Data identifier with visit, raft, sensor
198 """
200 pathId = self._transformId(dataId)
201 visit = pathId['visit']
202 raft = pathId['raft'] # "xy" e.g. "20"
203 sensor = pathId['sensor'] # "xy" e.g. "11"
205 r1, r2 = raft
206 s1, s2 = sensor
207 return (visit << 9) + \
208 (int(r1) * 5 + int(r2)) * 10 + \
209 (int(s1) * 3 + int(s2))
211 def _computeCoaddExposureId(self, dataId, singleFilter):
212 """Compute the 64-bit (long) identifier for a coadd.
214 @param dataId (dict) Data identifier with tract and patch.
215 @param singleFilter (bool) True means the desired ID is for a single-
216 filter coadd, in which case dataId
217 must contain filter.
218 """
219 tract = int(dataId['tract'])
220 if tract < 0 or tract >= 128:
221 raise RuntimeError('tract not in range [0,128)')
222 patchX, patchY = list(map(int, dataId['patch'].split(',')))
223 for p in (patchX, patchY):
224 if p < 0 or p >= 2**13:
225 raise RuntimeError('patch component not in range [0, 8192)')
226 id = (tract * 2**13 + patchX) * 2**13 + patchY
227 if singleFilter:
228 return id * 8 + self.filterIdMap[dataId['filter']]
229 return id
231 def _defectLookup(self, dataId, dateKey='taiObs'):
232 """Find the defects for a given CCD.
234 Parameters
235 ----------
236 dataId : `dict`
237 Dataset identifier
239 Returns
240 -------
241 `str`
242 Path to the defects file or None if not available.
243 """
244 if self.defectRegistry is None:
245 return None
246 if self.registry is None:
247 raise RuntimeError("No registry for defect lookup")
249 ccdKey, ccdVal = self._getCcdKeyVal(dataId)
251 dataIdForLookup = {'visit': dataId['visit']}
252 # .lookup will fail in a posix registry because there is no template to provide.
253 rows = self.registry.lookup((dateKey), ('raw_visit'), dataIdForLookup)
254 if len(rows) == 0:
255 return None
256 assert len(rows) == 1
257 dayObs = rows[0][0]
259 # Lookup the defects for this CCD serial number that are valid at the exposure midpoint.
260 rows = self.defectRegistry.executeQuery(("path",), ("defect",),
261 [(ccdKey, "?")],
262 ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"),
263 (ccdVal, dayObs))
264 if not rows or len(rows) == 0:
265 return None
266 if len(rows) == 1:
267 return os.path.join(self.defectPath, rows[0][0])
268 else:
269 raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" %
270 (ccdVal, dayObs, len(rows), ", ".join([_[0] for _ in rows])))
272 def map_defects(self, dataId, write=False):
273 """Map defects dataset.
275 Returns
276 -------
277 `lsst.daf.butler.ButlerLocation`
278 Minimal ButlerLocation containing just the locationList field
279 (just enough information that bypass_defects can use it).
280 """
281 defectFitsPath = self._defectLookup(dataId=dataId)
282 if defectFitsPath is None:
283 raise RuntimeError("No defects available for dataId=%s" % (dataId,))
285 return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
286 dataId, self,
287 storage=self.rootStorage)
289 def map_linearizer(self, dataId, write=False):
290 return None
292 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId):
293 """Return a defect based on the butler location returned by map_defects
295 Parameters
296 ----------
297 butlerLocation : `lsst.daf.persistence.ButlerLocation`
298 locationList = path to defects FITS file
299 dataId : `dict`
300 Butler data ID; "ccd" must be set.
302 Note: the name "bypass_XXX" means the butler makes no attempt to
303 convert the ButlerLocation into an object, which is what we want for
304 now, since that conversion is a bit tricky.
305 """
306 detectorName = self._extractDetectorName(dataId)
307 defectsFitsPath = butlerLocation.locationList[0]
309 with fits.open(defectsFitsPath) as hduList:
310 for hdu in hduList[1:]:
311 if hdu.header["name"] != detectorName:
312 continue
314 defectList = Defects()
315 for data in hdu.data:
316 bbox = geom.Box2I(
317 geom.Point2I(int(data['x0']), int(data['y0'])),
318 geom.Extent2I(int(data['width']), int(data['height'])),
319 )
320 defectList.append(bbox)
321 return defectList
323 raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
325 _nbit_id = 30
327 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
328 """The number of bits used up for patch ID bits"""
329 return 64 - self._nbit_id
331 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
332 return self._computeCoaddExposureId(dataId, False)
334 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
335 """The number of bits used up for patch ID bits"""
336 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
338 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
339 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
341 @staticmethod
342 def getShortCcdName(ccdId):
343 """Convert a CCD name to a form useful as a filename
345 This LSST version converts spaces to underscores and elides colons and commas.
346 """
347 return re.sub("[:,]", "", ccdId.replace(" ", "_"))
349 def _setAmpExposureId(self, propertyList, dataId):
350 propertyList.set("Computed_ampExposureId", self._computeAmpExposureId(dataId))
351 return propertyList
353 def _setCcdExposureId(self, propertyList, dataId):
354 propertyList.set("Computed_ccdExposureId", self._computeCcdExposureId(dataId))
355 return propertyList
357###############################################################################
359 def std_raw(self, item, dataId):
360 md = item.getMetadata()
361 if md.exists("VERSION") and md.getInt("VERSION") < 16952:
362 # CRVAL is FK5 at date of observation
363 dateObsTaiMjd = md.getScalar("TAI")
364 dateObs = dafBase.DateTime(dateObsTaiMjd,
365 system=dafBase.DateTime.MJD,
366 scale=dafBase.DateTime.TAI)
367 correctedEquinox = dateObs.get(system=dafBase.DateTime.EPOCH,
368 scale=dafBase.DateTime.TAI)
369 md.set("EQUINOX", correctedEquinox)
370 md.set("RADESYS", "FK5")
371 print("****** changing equinox to", correctedEquinox)
372 return super(LsstSimMapper, self).std_raw(item, dataId)
374 def std_eimage(self, item, dataId):
375 """Standardize a eimage dataset by converting it to an Exposure instead of an Image"""
376 return self._standardizeExposure(self.exposures['eimage'], item, dataId, trimmed=True)
378 def _createInitialSkyWcs(self, exposure):
379 """Create a SkyWcs from the header metadata.
381 PhoSim data may not have self-consistent boresight and crval/crpix
382 values, and/or may have been written in FK5, so we just use the
383 metadata here, and ignore VisitInfo/CameraGeom.
385 Parameters
386 ----------
387 exposure : `lsst.afw.image.Exposure`
388 The exposure to get data from, and attach the SkyWcs to.
389 """
390 self._createSkyWcsFromMetadata(exposure)
392###############################################################################
394 def _getCcdKeyVal(self, dataId):
395 """Return CCD key and value used to look a defect in the defect
396 registry
398 The default implementation simply returns ("ccd", full detector name)
399 """
400 return ("ccd", self._extractDetectorName(dataId))
402 def bypass_ampExposureId(self, datasetType, pythonType, location, dataId):
403 return self._computeAmpExposureId(dataId)
405 def bypass_ampExposureId_bits(self, datasetType, pythonType, location, dataId):
406 return 45
408 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
409 return self._computeCcdExposureId(dataId)
411 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
412 return 41
414 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
415 return self._computeCoaddExposureId(dataId, True)
417 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
418 return 1 + 7 + 13*2 + 3
420 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
421 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
423 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId):
424 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId)
426###############################################################################
428 def add_sdqaAmp(self, dataId):
429 ampExposureId = self._computeAmpExposureId(dataId)
430 return {"ampExposureId": ampExposureId, "sdqaRatingScope": "AMP"}
432 def add_sdqaCcd(self, dataId):
433 ccdExposureId = self._computeCcdExposureId(dataId)
434 return {"ccdExposureId": ccdExposureId, "sdqaRatingScope": "CCD"}
436###############################################################################
439for dsType in ("raw", "postISR"):
440 setattr(LsstSimMapper, "std_" + dsType + "_md", 440 ↛ exitline 440 didn't jump to the function exit
441 lambda self, item, dataId: self._setAmpExposureId(item, dataId))
442for dsType in ("eimage", "postISRCCD", "visitim", "calexp", "calsnap"):
443 setattr(LsstSimMapper, "std_" + dsType + "_md", 443 ↛ exitline 443 didn't jump to the function exit
444 lambda self, item, dataId: self._setCcdExposureId(item, dataId))