Coverage for python/lsst/obs/hsc/hscMapper.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
3import lsst.log
4from lsst.obs.base import CameraMapper
5from lsst.daf.persistence import ButlerLocation, Policy
6import lsst.afw.image as afwImage
7import lsst.afw.math as afwMath
8import lsst.afw.geom as afwGeom
9import lsst.geom as geom
10from lsst.ip.isr import Linearizer
11import lsst.pex.exceptions
12from .makeHscRawVisitInfo import MakeHscRawVisitInfo
13from .hscPupil import HscPupilFactory
14from .hscFilters import HSC_FILTER_DEFINITIONS
17class HscMapper(CameraMapper):
18 """Provides abstract-physical mapping for HSC data"""
19 packageName = "obs_subaru"
21 MakeRawVisitInfoClass = MakeHscRawVisitInfo
23 PupilFactoryClass = HscPupilFactory
25 # Use the full instrument class name to prevent import errors
26 # between hsc/ and subaru/ packages.
27 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam"
28 _cameraCache = None # Camera object, cached to speed up instantiation time
30 @classmethod
31 def addFilters(cls):
32 HSC_FILTER_DEFINITIONS.defineFilters()
34 def __init__(self, **kwargs):
35 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy")
36 policy = Policy(policyFile)
37 if not kwargs.get('root', None):
38 try:
39 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC')
40 except Exception:
41 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified")
42 if not kwargs.get('calibRoot', None):
43 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
44 if "repositoryCfg" in kwargs:
45 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
46 hasattr(cfg, "root")]
47 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
48 for calibRoot in calibSearch:
49 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
50 kwargs['calibRoot'] = calibRoot
51 break
52 if not kwargs.get('calibRoot', None):
53 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory")
55 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
57 # Ensure each dataset type of interest knows about the full range of
58 # keys available from the registry
59 keys = {'field': str,
60 'visit': int,
61 'filter': str,
62 'ccd': int,
63 'dateObs': str,
64 'taiObs': str,
65 'expTime': float,
66 'pointing': int,
67 }
68 for name in ("raw",
69 # processCcd outputs
70 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch",
71 "srcMatch",
72 # mosaic outputs
73 "wcs", "fcr",
74 # processCcd QA
75 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough",
76 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap",
77 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid",
78 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid",
79 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid",
80 # forcedPhot outputs
81 "forced_src",
82 ):
83 self.mappings[name].keyDict.update(keys)
85 self.addFilters()
87 self.filters = {}
88 for filt in HSC_FILTER_DEFINITIONS:
89 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName()
90 self.defaultFilterName = "UNRECOGNISED"
92 #
93 # The number of bits allocated for fields in object IDs, appropriate
94 # for the default-configured Rings skymap.
95 #
96 # This shouldn't be the mapper's job at all; see #2797.
98 HscMapper._nbit_tract = 16
99 HscMapper._nbit_patch = 5
100 HscMapper._nbit_filter = 6
102 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter)
104 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter:
105 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %
106 HscMapper._nbit_filter)
108 def _makeCamera(self, *args, **kwargs):
109 """Make the camera object
111 This implementation layers a cache over the parent class'
112 implementation. Caching the camera improves the instantiation
113 time for the HscMapper because parsing the camera's Config
114 involves a lot of 'stat' calls (through the tracebacks).
115 """
116 if not self._cameraCache:
117 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs)
118 return self._cameraCache
120 @classmethod
121 def clearCache(cls):
122 """Clear the camera cache
124 This is principally intended to help memory leak tests pass.
125 """
126 cls._cameraCache = None
128 def map(self, datasetType, dataId, write=False):
129 """Need to strip 'flags' argument from map
131 We want the 'flags' argument passed to the butler to work (it's
132 used to change how the reading/writing is done), but want it
133 removed from the mapper (because it doesn't correspond to a
134 registry column).
135 """
136 copyId = dataId.copy()
137 copyId.pop("flags", None)
138 location = super(HscMapper, self).map(datasetType, copyId, write=write)
140 if 'flags' in dataId:
141 location.getAdditionalData().set('flags', dataId['flags'])
143 return location
145 @staticmethod
146 def _flipChipsLR(exp, wcs, detectorId, dims=None):
147 """Flip the chip left/right or top/bottom. Process either/and the
148 pixels and wcs
150 Most chips are flipped L/R, but the rotated ones (100..103) are
151 flipped T/B.
152 """
153 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False)
154 if exp:
155 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB))
156 if wcs:
157 ampDimensions = exp.getDimensions() if dims is None else dims
158 ampCenter = geom.Point2D(ampDimensions/2.0)
159 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter)
161 return exp, wcs
163 def std_raw_md(self, md, dataId):
164 """We need to flip the WCS defined by the metadata in case anyone ever
165 constructs a Wcs from it.
166 """
167 wcs = afwGeom.makeSkyWcs(md)
168 wcs = self._flipChipsLR(None, wcs, dataId['ccd'],
169 dims=afwImage.bboxFromMetadata(md).getDimensions())[1]
170 # NOTE: we don't know where the 0.992 magic constant came from.
171 # It was copied over from hscSimMapper.
172 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(),
173 crval=wcs.getSkyOrigin(),
174 cdMatrix=wcs.getCdMatrix()*0.992)
175 wcsMd = wcsR.getFitsMetadata()
177 for k in wcsMd.names():
178 md.set(k, wcsMd.getScalar(k))
180 return md
182 def _createSkyWcsFromMetadata(self, exposure):
183 # Overridden to flip chips as necessary to get sensible SkyWcs.
184 metadata = exposure.getMetadata()
185 try:
186 wcs = afwGeom.makeSkyWcs(metadata, strip=True)
187 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId())
188 exposure.setWcs(wcs)
189 except lsst.pex.exceptions.TypeError as e:
190 # See DM-14372 for why this is debug and not warn (e.g. calib
191 # files without wcs metadata).
192 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
193 " %s", e.args[0])
195 # ensure any WCS values stripped from the metadata are removed in
196 # the exposure
197 exposure.setMetadata(metadata)
199 def std_dark(self, item, dataId):
200 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False)
201 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
202 exposure.getInfo().setVisitInfo(visitInfo)
203 return exposure
205 def _extractDetectorName(self, dataId):
206 return int("%(ccd)d" % dataId)
208 def _computeCcdExposureId(self, dataId):
209 """Compute the 64-bit (long) identifier for a CCD exposure.
211 @param dataId (dict) Data identifier with visit, ccd
212 """
213 pathId = self._transformId(dataId)
214 visit = pathId['visit']
215 ccd = pathId['ccd']
216 return visit*200 + ccd
218 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
219 return self._computeCcdExposureId(dataId)
221 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
222 """How many bits are required for the maximum exposure ID"""
223 return 32 # just a guess, but this leaves plenty of space for sources
225 def map_linearizer(self, dataId, write=False):
226 """Map a linearizer."""
227 actualId = self._transformId(dataId)
228 return ButlerLocation(
229 pythonType="lsst.ip.isr.LinearizeSquared",
230 cppType="Config",
231 storageName="PickleStorage",
232 locationList="ignored",
233 dataId=actualId,
234 mapper=self,
235 storage=self.rootStorage)
237 def map_crosstalk(self, dataId, write=False):
238 """Fake the mapping for crosstalk.
240 Crosstalk is constructed from config parameters, but we need
241 Gen2 butlers to be able to respond to requests for it.
242 Returning None provides a response that can be used with the
243 config parameters to generate the appropriate calibration.
244 """
245 return None
247 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
248 """Return a linearizer for the given detector.
250 On each call, a fresh instance of `Linearizer` is returned; the caller
251 is responsible for initializing it appropriately for the detector.
253 Parameters
254 ----------
255 datasetType : `str``
256 The dataset type.
257 pythonType : `str` or `type`
258 Type of python object.
259 butlerLocation : `lsst.daf.persistence.ButlerLocation`
260 Struct-like class that holds information needed to persist and
261 retrieve an object using the LSST Persistence Framework.
262 dataId : `dict`
263 dataId passed to map location.
265 Returns
266 -------
267 Linearizer : `lsst.ip.isr.Linearizer`
268 Linearizer object for the given detector.
270 Notes
271 -----
272 Linearizers are not saved to persistent storage; rather, they are
273 managed entirely in memory. On each call, this function will return a
274 new instance of `Linearizer`, which must be managed (including setting
275 it up for use with a particular detector) by the caller. Calling
276 `bypass_linearizer` twice for the same detector will return
277 _different_ instances of `Linearizer`, which share no state.
278 """
279 return Linearizer(detectorId=dataId.get('ccd', None))
281 def _computeCoaddExposureId(self, dataId, singleFilter):
282 """Compute the 64-bit (long) identifier for a coadd.
284 @param dataId (dict) Data identifier with tract and patch.
285 @param singleFilter (bool) True means the desired ID is for a single-
286 filter coadd, in which case dataId
287 must contain filter.
288 """
290 tract = int(dataId['tract'])
291 if tract < 0 or tract >= 2**HscMapper._nbit_tract:
292 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract))
293 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
294 for p in (patchX, patchY):
295 if p < 0 or p >= 2**HscMapper._nbit_patch:
296 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch)
297 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY
298 if singleFilter:
299 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
300 return oid
302 def bypass_deepCoaddId_bits(self, *args, **kwargs):
303 """The number of bits used up for patch ID bits"""
304 return 64 - HscMapper._nbit_id
306 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
307 return self._computeCoaddExposureId(dataId, True)
309 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
310 """The number of bits used up for patch ID bits"""
311 return 64 - HscMapper._nbit_id
313 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
314 return self._computeCoaddExposureId(dataId, False)
316 # The following allow grabbing a 'psf' from the butler directly, without
317 # having to get it from a calexp
318 def map_psf(self, dataId, write=False):
319 if write:
320 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp")
321 copyId = dataId.copy()
322 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1))
323 return self.map_calexp_sub(copyId)
325 def std_psf(self, calexp, dataId):
326 return calexp.getPsf()
328 @classmethod
329 def getCameraName(cls):
330 return "hsc"