Coverage for python/lsst/obs/hsc/hscMapper.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
3import lsst.log
4from lsst.obs.base import CameraMapper
5from lsst.daf.persistence import ButlerLocation, Policy
6import lsst.afw.image as afwImage
7import lsst.afw.math as afwMath
8import lsst.afw.geom as afwGeom
9import lsst.geom as geom
10from lsst.ip.isr import Linearizer
11import lsst.pex.exceptions
12from .makeHscRawVisitInfo import MakeHscRawVisitInfo
13from .hscPupil import HscPupilFactory
14from .hscFilters import HSC_FILTER_DEFINITIONS
17class HscMapper(CameraMapper):
18 """Provides abstract-physical mapping for HSC data"""
19 packageName = "obs_subaru"
21 MakeRawVisitInfoClass = MakeHscRawVisitInfo
23 PupilFactoryClass = HscPupilFactory
25 # Use the full instrument class name to prevent import errors
26 # between hsc/ and subaru/ packages.
27 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam"
28 _cameraCache = None # Camera object, cached to speed up instantiation time
30 @classmethod
31 def addFilters(cls):
32 HSC_FILTER_DEFINITIONS.defineFilters()
34 def __init__(self, **kwargs):
35 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy")
36 policy = Policy(policyFile)
37 if not kwargs.get('root', None):
38 try:
39 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC')
40 except Exception:
41 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified")
42 if not kwargs.get('calibRoot', None):
43 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
44 if "repositoryCfg" in kwargs:
45 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
46 hasattr(cfg, "root")]
47 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
48 for calibRoot in calibSearch:
49 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
50 kwargs['calibRoot'] = calibRoot
51 break
52 if not kwargs.get('calibRoot', None):
53 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory")
55 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
57 # Ensure each dataset type of interest knows about the full range of keys available from the registry
58 keys = {'field': str,
59 'visit': int,
60 'filter': str,
61 'ccd': int,
62 'dateObs': str,
63 'taiObs': str,
64 'expTime': float,
65 'pointing': int,
66 }
67 for name in ("raw",
68 # processCcd outputs
69 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch",
70 "srcMatch",
71 # mosaic outputs
72 "wcs", "fcr",
73 # processCcd QA
74 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough",
75 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap",
76 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid",
77 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid",
78 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid",
79 # forcedPhot outputs
80 "forced_src",
81 ):
82 self.mappings[name].keyDict.update(keys)
84 self.addFilters()
86 self.filters = {}
87 for filt in HSC_FILTER_DEFINITIONS:
88 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName()
89 self.defaultFilterName = "UNRECOGNISED"
91 #
92 # The number of bits allocated for fields in object IDs, appropriate for
93 # the default-configured Rings skymap.
94 #
95 # This shouldn't be the mapper's job at all; see #2797.
97 HscMapper._nbit_tract = 16
98 HscMapper._nbit_patch = 5
99 HscMapper._nbit_filter = 6
101 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter)
103 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter:
104 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %
105 HscMapper._nbit_filter)
107 def _makeCamera(self, *args, **kwargs):
108 """Make the camera object
110 This implementation layers a cache over the parent class'
111 implementation. Caching the camera improves the instantiation
112 time for the HscMapper because parsing the camera's Config
113 involves a lot of 'stat' calls (through the tracebacks).
114 """
115 if not self._cameraCache:
116 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs)
117 return self._cameraCache
119 @classmethod
120 def clearCache(cls):
121 """Clear the camera cache
123 This is principally intended to help memory leak tests pass.
124 """
125 cls._cameraCache = None
127 def map(self, datasetType, dataId, write=False):
128 """Need to strip 'flags' argument from map
130 We want the 'flags' argument passed to the butler to work (it's
131 used to change how the reading/writing is done), but want it
132 removed from the mapper (because it doesn't correspond to a
133 registry column).
134 """
135 copyId = dataId.copy()
136 copyId.pop("flags", None)
137 location = super(HscMapper, self).map(datasetType, copyId, write=write)
139 if 'flags' in dataId:
140 location.getAdditionalData().set('flags', dataId['flags'])
142 return location
144 @staticmethod
145 def _flipChipsLR(exp, wcs, detectorId, dims=None):
146 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs
148 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B
149 """
150 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False)
151 if exp:
152 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB))
153 if wcs:
154 ampDimensions = exp.getDimensions() if dims is None else dims
155 ampCenter = geom.Point2D(ampDimensions/2.0)
156 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter)
158 return exp, wcs
160 def std_raw_md(self, md, dataId):
161 """We need to flip the WCS defined by the metadata in case anyone ever
162 constructs a Wcs from it.
163 """
164 wcs = afwGeom.makeSkyWcs(md)
165 wcs = self._flipChipsLR(None, wcs, dataId['ccd'],
166 dims=afwImage.bboxFromMetadata(md).getDimensions())[1]
167 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper.
168 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(),
169 crval=wcs.getSkyOrigin(),
170 cdMatrix=wcs.getCdMatrix()*0.992)
171 wcsMd = wcsR.getFitsMetadata()
173 for k in wcsMd.names():
174 md.set(k, wcsMd.getScalar(k))
176 return md
178 def _createSkyWcsFromMetadata(self, exposure):
179 # Overridden to flip chips as necessary to get sensible SkyWcs.
180 metadata = exposure.getMetadata()
181 try:
182 wcs = afwGeom.makeSkyWcs(metadata, strip=True)
183 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId())
184 exposure.setWcs(wcs)
185 except lsst.pex.exceptions.TypeError as e:
186 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata).
187 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
188 " %s", e.args[0])
190 # ensure any WCS values stripped from the metadata are removed in the exposure
191 exposure.setMetadata(metadata)
193 def std_dark(self, item, dataId):
194 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False)
195 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
196 exposure.getInfo().setVisitInfo(visitInfo)
197 return exposure
199 def _extractDetectorName(self, dataId):
200 return int("%(ccd)d" % dataId)
202 def _computeCcdExposureId(self, dataId):
203 """Compute the 64-bit (long) identifier for a CCD exposure.
205 @param dataId (dict) Data identifier with visit, ccd
206 """
207 pathId = self._transformId(dataId)
208 visit = pathId['visit']
209 ccd = pathId['ccd']
210 return visit*200 + ccd
212 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
213 return self._computeCcdExposureId(dataId)
215 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
216 """How many bits are required for the maximum exposure ID"""
217 return 32 # just a guess, but this leaves plenty of space for sources
219 def map_linearizer(self, dataId, write=False):
220 """Map a linearizer."""
221 actualId = self._transformId(dataId)
222 return ButlerLocation(
223 pythonType="lsst.ip.isr.LinearizeSquared",
224 cppType="Config",
225 storageName="PickleStorage",
226 locationList="ignored",
227 dataId=actualId,
228 mapper=self,
229 storage=self.rootStorage)
231 def map_crosstalk(self, dataId, write=False):
232 """Fake the mapping for crosstalk.
234 Crosstalk is constructed from config parameters, but we need
235 Gen2 butlers to be able to respond to requests for it.
236 Returning None provides a response that can be used with the
237 config parameters to generate the appropriate calibration.
238 """
239 return None
241 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
242 """Return a linearizer for the given detector.
244 On each call, a fresh instance of `Linearizer` is returned; the caller is responsible for
245 initializing it appropriately for the detector.
247 Parameters
248 ----------
249 datasetType : `str``
250 The dataset type.
251 pythonType : `str` or `type`
252 Type of python object.
253 butlerLocation : `lsst.daf.persistence.ButlerLocation`
254 Struct-like class that holds information needed to persist and retrieve an object using
255 the LSST Persistence Framework.
256 dataId : `dict`
257 dataId passed to map location.
259 Returns
260 -------
261 Linearizer : `lsst.ip.isr.Linearizer`
262 Linearizer object for the given detector.
264 Notes
265 -----
266 Linearizers are not saved to persistent storage; rather, they are managed entirely in memory.
267 On each call, this function will return a new instance of `Linearizer`, which must be managed
268 (including setting it up for use with a particular detector) by the caller. Calling
269 `bypass_linearizer` twice for the same detector will return _different_ instances of `Linearizer`,
270 which share no state.
271 """
272 return Linearizer()
274 def _computeCoaddExposureId(self, dataId, singleFilter):
275 """Compute the 64-bit (long) identifier for a coadd.
277 @param dataId (dict) Data identifier with tract and patch.
278 @param singleFilter (bool) True means the desired ID is for a single-
279 filter coadd, in which case dataId
280 must contain filter.
281 """
283 tract = int(dataId['tract'])
284 if tract < 0 or tract >= 2**HscMapper._nbit_tract:
285 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract))
286 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
287 for p in (patchX, patchY):
288 if p < 0 or p >= 2**HscMapper._nbit_patch:
289 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch)
290 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY
291 if singleFilter:
292 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
293 return oid
295 def bypass_deepCoaddId_bits(self, *args, **kwargs):
296 """The number of bits used up for patch ID bits"""
297 return 64 - HscMapper._nbit_id
299 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
300 return self._computeCoaddExposureId(dataId, True)
302 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
303 """The number of bits used up for patch ID bits"""
304 return 64 - HscMapper._nbit_id
306 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
307 return self._computeCoaddExposureId(dataId, False)
309 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp
310 def map_psf(self, dataId, write=False):
311 if write:
312 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp")
313 copyId = dataId.copy()
314 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1))
315 return self.map_calexp_sub(copyId)
317 def std_psf(self, calexp, dataId):
318 return calexp.getPsf()
320 @classmethod
321 def getCameraName(cls):
322 return "hsc"