Coverage for python/lsst/obs/lsstSim/lsstSimMapper.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012, 2013 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23__all__ = ["LsstSimMapper"]
25import os
26import re
27from astropy.io import fits
29import lsst.daf.base as dafBase
30import lsst.afw.image.utils as afwImageUtils
31import lsst.geom as geom
32import lsst.daf.persistence as dafPersist
33from lsst.meas.algorithms import Defects
34from .makeLsstSimRawVisitInfo import MakeLsstSimRawVisitInfo
35from lsst.utils import getPackageDir
37from lsst.obs.base import CameraMapper
39# Solely to get boost serialization registrations for Measurement subclasses
42class LsstSimMapper(CameraMapper):
43 packageName = 'obs_lsstSim'
45 MakeRawVisitInfoClass = MakeLsstSimRawVisitInfo
47 _CcdNameRe = re.compile(r"R:(\d,\d) S:(\d,\d(?:,[AB])?)$")
49 def __init__(self, inputPolicy=None, **kwargs):
50 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, "LsstSimMapper.yaml", "policy")
51 policy = dafPersist.Policy(policyFile)
52 repositoryDir = os.path.join(getPackageDir(self.packageName), 'policy')
53 self.defectRegistry = None
54 if 'defects' in policy:
55 self.defectPath = os.path.join(repositoryDir, policy['defects'])
56 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
57 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
59 self.doFootprints = False
60 if inputPolicy is not None:
61 for kw in inputPolicy.paramNames(True):
62 if kw == "doFootprints":
63 self.doFootprints = True
64 else:
65 kwargs[kw] = inputPolicy.get(kw)
67 super(LsstSimMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
68 self.filterIdMap = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5, 'i2': 5}
70 # The LSST Filters from L. Jones 04/07/10
71 afwImageUtils.resetFilters()
72 afwImageUtils.defineFilter('u', lambdaEff=364.59, lambdaMin=324.0, lambdaMax=395.0)
73 afwImageUtils.defineFilter('g', lambdaEff=476.31, lambdaMin=405.0, lambdaMax=552.0)
74 afwImageUtils.defineFilter('r', lambdaEff=619.42, lambdaMin=552.0, lambdaMax=691.0)
75 afwImageUtils.defineFilter('i', lambdaEff=752.06, lambdaMin=818.0, lambdaMax=921.0)
76 afwImageUtils.defineFilter('z', lambdaEff=866.85, lambdaMin=922.0, lambdaMax=997.0)
77 # official y filter
78 afwImageUtils.defineFilter('y', lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4'])
79 # If/when y3 sim data becomes available, uncomment this and
80 # modify the schema appropriately
81 # afwImageUtils.defineFilter('y3', 1002.44) # candidate y-band
83 def _transformId(self, dataId):
84 """Transform an ID dict into standard form for LSST
86 Standard keys are as follows:
87 - raft: in the form <x>,<y>
88 - sensor: in the form <x>,<y>,<c> where <c> = A or B
89 - channel: in the form <x>,<y>
90 - snap: exposure number
92 Other supported keys, which are used to set the above, if not already set:
93 - ccd: an alias for sensor (hence NOT the full ccd name)
94 - ccdName or sensorName: full ccd name in the form R:<x>,<y> S:<x>,<y>[,<c>]
95 if found, used to set raft and sensor, if not already set
96 - channelName, ampName: an alternate way to specify channel, in the form: IDxx
97 - amp: an alias for channel
98 - exposure: an alias for snap
100 @param dataId[in] (dict) Dataset identifier; this must not be modified
101 @return (dict) Transformed dataset identifier
102 @raise RuntimeError if a value is not valid
103 """
104 actualId = dataId.copy()
105 for ccdAlias in ("ccdName", "sensorName"):
106 if ccdAlias in actualId:
107 ccdName = actualId[ccdAlias].upper()
108 m = self._CcdNameRe.match(ccdName)
109 if m is None:
110 raise RuntimeError("Invalid value for %s: %r" % (ccdAlias, ccdName))
111 actualId.setdefault("raft", m.group(1))
112 actualId.setdefault("sensor", m.group(2))
113 break
114 if "ccd" in actualId:
115 actualId.setdefault("sensor", actualId["ccd"])
116 if "amp" in actualId:
117 actualId.setdefault("channel", actualId["amp"])
118 elif "channel" not in actualId:
119 for ampName in ("ampName", "channelName"):
120 if ampName in actualId:
121 m = re.match(r'ID(\d+)$', actualId[ampName])
122 channelNumber = int(m.group(1))
123 channelX = channelNumber % 8
124 channelY = channelNumber // 8
125 actualId['channel'] = str(channelX) + "," + str(channelY)
126 break
127 if "exposure" in actualId:
128 actualId.setdefault("snap", actualId["exposure"])
130 # why strip out the commas after carefully adding them?
131 if "raft" in actualId:
132 actualId['raft'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['raft'])
133 if "sensor" in actualId:
134 actualId['sensor'] = actualId['sensor'].replace(",", "")
135 if "channel" in actualId:
136 actualId['channel'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['channel'])
137 return actualId
139 def validate(self, dataId):
140 for component in ("raft", "sensor", "channel"):
141 if component not in dataId:
142 continue
143 val = dataId[component]
144 if not isinstance(val, str):
145 raise RuntimeError(
146 "%s identifier should be type str, not %s: %r" % (component.title(), type(val), val))
147 if component == "sensor":
148 if not re.search(r'^\d,\d(,[AB])?$', val):
149 raise RuntimeError("Invalid %s identifier: %r" % (component, val))
150 else:
151 if not re.search(r'^(\d),(\d)$', val):
152 raise RuntimeError("Invalid %s identifier: %r" % (component, val))
153 return dataId
155 def _extractDetectorName(self, dataId):
156 return "R:%(raft)s S:%(sensor)s" % dataId
158 def getDataId(self, visit, ccdId):
159 """get dataId dict from visit and ccd identifier
161 @param visit 32 or 64-bit depending on camera
162 @param ccdId detector name: same as detector.getName()
163 """
164 dataId = {'visit': int(visit)}
165 m = self._CcdNameRe.match(ccdId)
166 if m is None:
167 raise RuntimeError("Cannot parse ccdId=%r" % (ccdId,))
168 dataId['raft'] = m.group(0)
169 dataId['sensor'] = m.group(1)
170 return dataId
172 def _extractAmpId(self, dataId):
173 m = re.match(r'(\d),(\d)', dataId['channel'])
174 # Note that indices are swapped in the camera geometry vs. official
175 # channel specification.
176 return (self._extractDetectorName(dataId),
177 int(m.group(1)), int(m.group(2)))
179 def _computeAmpExposureId(self, dataId):
180 # visit, snap, raft, sensor, channel):
181 """Compute the 64-bit (long) identifier for an amp exposure.
183 @param dataId (dict) Data identifier with visit, snap, raft, sensor, channel
184 """
186 pathId = self._transformId(dataId)
187 visit = pathId['visit']
188 snap = pathId['snap']
189 raft = pathId['raft'] # "xy" e.g. "20"
190 sensor = pathId['sensor'] # "xy" e.g. "11"
191 channel = pathId['channel'] # "yx" e.g. "05" (NB: yx, not xy, in original comment)
193 r1, r2 = raft
194 s1, s2 = sensor
195 c1, c2 = channel
196 return (visit << 13) + (snap << 12) + \
197 (int(r1) * 5 + int(r2)) * 160 + \
198 (int(s1) * 3 + int(s2)) * 16 + \
199 (int(c1) * 8 + int(c2))
201 def _computeCcdExposureId(self, dataId):
202 """Compute the 64-bit (long) identifier for a CCD exposure.
204 @param dataId (dict) Data identifier with visit, raft, sensor
205 """
207 pathId = self._transformId(dataId)
208 visit = pathId['visit']
209 raft = pathId['raft'] # "xy" e.g. "20"
210 sensor = pathId['sensor'] # "xy" e.g. "11"
212 r1, r2 = raft
213 s1, s2 = sensor
214 return (visit << 9) + \
215 (int(r1) * 5 + int(r2)) * 10 + \
216 (int(s1) * 3 + int(s2))
218 def _computeCoaddExposureId(self, dataId, singleFilter):
219 """Compute the 64-bit (long) identifier for a coadd.
221 @param dataId (dict) Data identifier with tract and patch.
222 @param singleFilter (bool) True means the desired ID is for a single-
223 filter coadd, in which case dataId
224 must contain filter.
225 """
226 tract = int(dataId['tract'])
227 if tract < 0 or tract >= 128:
228 raise RuntimeError('tract not in range [0,128)')
229 patchX, patchY = list(map(int, dataId['patch'].split(',')))
230 for p in (patchX, patchY):
231 if p < 0 or p >= 2**13:
232 raise RuntimeError('patch component not in range [0, 8192)')
233 id = (tract * 2**13 + patchX) * 2**13 + patchY
234 if singleFilter:
235 return id * 8 + self.filterIdMap[dataId['filter']]
236 return id
238 def _defectLookup(self, dataId, dateKey='taiObs'):
239 """Find the defects for a given CCD.
241 Parameters
242 ----------
243 dataId : `dict`
244 Dataset identifier
246 Returns
247 -------
248 `str`
249 Path to the defects file or None if not available.
250 """
251 if self.defectRegistry is None:
252 return None
253 if self.registry is None:
254 raise RuntimeError("No registry for defect lookup")
256 ccdKey, ccdVal = self._getCcdKeyVal(dataId)
258 dataIdForLookup = {'visit': dataId['visit']}
259 # .lookup will fail in a posix registry because there is no template to provide.
260 rows = self.registry.lookup((dateKey), ('raw_visit'), dataIdForLookup)
261 if len(rows) == 0:
262 return None
263 assert len(rows) == 1
264 dayObs = rows[0][0]
266 # Lookup the defects for this CCD serial number that are valid at the exposure midpoint.
267 rows = self.defectRegistry.executeQuery(("path",), ("defect",),
268 [(ccdKey, "?")],
269 ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"),
270 (ccdVal, dayObs))
271 if not rows or len(rows) == 0:
272 return None
273 if len(rows) == 1:
274 return os.path.join(self.defectPath, rows[0][0])
275 else:
276 raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" %
277 (ccdVal, dayObs, len(rows), ", ".join([_[0] for _ in rows])))
279 def map_defects(self, dataId, write=False):
280 """Map defects dataset.
282 Returns
283 -------
284 `lsst.daf.butler.ButlerLocation`
285 Minimal ButlerLocation containing just the locationList field
286 (just enough information that bypass_defects can use it).
287 """
288 defectFitsPath = self._defectLookup(dataId=dataId)
289 if defectFitsPath is None:
290 raise RuntimeError("No defects available for dataId=%s" % (dataId,))
292 return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
293 dataId, self,
294 storage=self.rootStorage)
296 def map_linearizer(self, dataId, write=False):
297 return None
299 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId):
300 """Return a defect based on the butler location returned by map_defects
302 Parameters
303 ----------
304 butlerLocation : `lsst.daf.persistence.ButlerLocation`
305 locationList = path to defects FITS file
306 dataId : `dict`
307 Butler data ID; "ccd" must be set.
309 Note: the name "bypass_XXX" means the butler makes no attempt to
310 convert the ButlerLocation into an object, which is what we want for
311 now, since that conversion is a bit tricky.
312 """
313 detectorName = self._extractDetectorName(dataId)
314 defectsFitsPath = butlerLocation.locationList[0]
316 with fits.open(defectsFitsPath) as hduList:
317 for hdu in hduList[1:]:
318 if hdu.header["name"] != detectorName:
319 continue
321 defectList = Defects()
322 for data in hdu.data:
323 bbox = geom.Box2I(
324 geom.Point2I(int(data['x0']), int(data['y0'])),
325 geom.Extent2I(int(data['width']), int(data['height'])),
326 )
327 defectList.append(bbox)
328 return defectList
330 raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
332 _nbit_id = 30
334 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
335 """The number of bits used up for patch ID bits"""
336 return 64 - self._nbit_id
338 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
339 return self._computeCoaddExposureId(dataId, False)
341 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs):
342 """The number of bits used up for patch ID bits"""
343 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs)
345 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId):
346 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId)
348 @staticmethod
349 def getShortCcdName(ccdId):
350 """Convert a CCD name to a form useful as a filename
352 This LSST version converts spaces to underscores and elides colons and commas.
353 """
354 return re.sub("[:,]", "", ccdId.replace(" ", "_"))
356 def _setAmpExposureId(self, propertyList, dataId):
357 propertyList.set("Computed_ampExposureId", self._computeAmpExposureId(dataId))
358 return propertyList
360 def _setCcdExposureId(self, propertyList, dataId):
361 propertyList.set("Computed_ccdExposureId", self._computeCcdExposureId(dataId))
362 return propertyList
364###############################################################################
366 def std_raw(self, item, dataId):
367 md = item.getMetadata()
368 if md.exists("VERSION") and md.getInt("VERSION") < 16952:
369 # CRVAL is FK5 at date of observation
370 dateObsTaiMjd = md.getScalar("TAI")
371 dateObs = dafBase.DateTime(dateObsTaiMjd,
372 system=dafBase.DateTime.MJD,
373 scale=dafBase.DateTime.TAI)
374 correctedEquinox = dateObs.get(system=dafBase.DateTime.EPOCH,
375 scale=dafBase.DateTime.TAI)
376 md.set("EQUINOX", correctedEquinox)
377 md.set("RADESYS", "FK5")
378 print("****** changing equinox to", correctedEquinox)
379 return super(LsstSimMapper, self).std_raw(item, dataId)
381 def std_eimage(self, item, dataId):
382 """Standardize a eimage dataset by converting it to an Exposure instead of an Image"""
383 return self._standardizeExposure(self.exposures['eimage'], item, dataId, trimmed=True)
385 def _createInitialSkyWcs(self, exposure):
386 """Create a SkyWcs from the header metadata.
388 PhoSim data may not have self-consistent boresight and crval/crpix
389 values, and/or may have been written in FK5, so we just use the
390 metadata here, and ignore VisitInfo/CameraGeom.
392 Parameters
393 ----------
394 exposure : `lsst.afw.image.Exposure`
395 The exposure to get data from, and attach the SkyWcs to.
396 """
397 self._createSkyWcsFromMetadata(exposure)
399###############################################################################
401 def _getCcdKeyVal(self, dataId):
402 """Return CCD key and value used to look a defect in the defect
403 registry
405 The default implementation simply returns ("ccd", full detector name)
406 """
407 return ("ccd", self._extractDetectorName(dataId))
409 def bypass_ampExposureId(self, datasetType, pythonType, location, dataId):
410 return self._computeAmpExposureId(dataId)
412 def bypass_ampExposureId_bits(self, datasetType, pythonType, location, dataId):
413 return 45
415 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
416 return self._computeCcdExposureId(dataId)
418 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
419 return 41
421 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
422 return self._computeCoaddExposureId(dataId, True)
424 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
425 return 1 + 7 + 13*2 + 3
427 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId):
428 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId)
430 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId):
431 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId)
433###############################################################################
435 def add_sdqaAmp(self, dataId):
436 ampExposureId = self._computeAmpExposureId(dataId)
437 return {"ampExposureId": ampExposureId, "sdqaRatingScope": "AMP"}
439 def add_sdqaCcd(self, dataId):
440 ccdExposureId = self._computeCcdExposureId(dataId)
441 return {"ccdExposureId": ccdExposureId, "sdqaRatingScope": "CCD"}
443###############################################################################
446for dsType in ("raw", "postISR"):
447 setattr(LsstSimMapper, "std_" + dsType + "_md", 447 ↛ exitline 447 didn't jump to the function exit
448 lambda self, item, dataId: self._setAmpExposureId(item, dataId))
449for dsType in ("eimage", "postISRCCD", "visitim", "calexp", "calsnap"):
450 setattr(LsstSimMapper, "std_" + dsType + "_md", 450 ↛ exitline 450 didn't jump to the function exit
451 lambda self, item, dataId: self._setCcdExposureId(item, dataId))