Coverage for python/lsst/obs/hsc/hscMapper.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
3import lsst.log
4from lsst.obs.base import CameraMapper
5from lsst.daf.persistence import ButlerLocation, Policy
6import lsst.afw.image as afwImage
7import lsst.afw.math as afwMath
8import lsst.afw.geom as afwGeom
9import lsst.geom as geom
10from lsst.ip.isr import Linearizer
11import lsst.pex.exceptions
12from .makeHscRawVisitInfo import MakeHscRawVisitInfo
13from .hscPupil import HscPupilFactory
14from .hscFilters import HSC_FILTER_DEFINITIONS
17class HscMapper(CameraMapper):
18 """Provides abstract-physical mapping for HSC data"""
19 packageName = "obs_subaru"
21 MakeRawVisitInfoClass = MakeHscRawVisitInfo
23 PupilFactoryClass = HscPupilFactory
25 _cameraCache = None # Camera object, cached to speed up instantiation time
27 @classmethod
28 def addFilters(cls):
29 HSC_FILTER_DEFINITIONS.defineFilters()
31 def __init__(self, **kwargs):
32 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy")
33 policy = Policy(policyFile)
34 if not kwargs.get('root', None):
35 try:
36 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC')
37 except Exception:
38 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified")
39 if not kwargs.get('calibRoot', None):
40 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
41 if "repositoryCfg" in kwargs:
42 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
43 hasattr(cfg, "root")]
44 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
45 for calibRoot in calibSearch:
46 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
47 kwargs['calibRoot'] = calibRoot
48 break
49 if not kwargs.get('calibRoot', None):
50 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory")
52 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
54 # Ensure each dataset type of interest knows about the full range of keys available from the registry
55 keys = {'field': str,
56 'visit': int,
57 'filter': str,
58 'ccd': int,
59 'dateObs': str,
60 'taiObs': str,
61 'expTime': float,
62 'pointing': int,
63 }
64 for name in ("raw",
65 # processCcd outputs
66 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch",
67 "srcMatch",
68 # mosaic outputs
69 "wcs", "fcr",
70 # processCcd QA
71 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough",
72 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap",
73 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid",
74 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid",
75 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid",
76 # forcedPhot outputs
77 "forced_src",
78 ):
79 self.mappings[name].keyDict.update(keys)
81 self.addFilters()
83 self.filters = {}
84 for filt in HSC_FILTER_DEFINITIONS:
85 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName()
86 self.defaultFilterName = "UNRECOGNISED"
88 #
89 # The number of bits allocated for fields in object IDs, appropriate for
90 # the default-configured Rings skymap.
91 #
92 # This shouldn't be the mapper's job at all; see #2797.
94 HscMapper._nbit_tract = 16
95 HscMapper._nbit_patch = 5
96 HscMapper._nbit_filter = 6
98 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter)
100 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter:
101 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %
102 HscMapper._nbit_filter)
104 def _makeCamera(self, *args, **kwargs):
105 """Make the camera object
107 This implementation layers a cache over the parent class'
108 implementation. Caching the camera improves the instantiation
109 time for the HscMapper because parsing the camera's Config
110 involves a lot of 'stat' calls (through the tracebacks).
111 """
112 if not self._cameraCache:
113 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs)
114 return self._cameraCache
116 @classmethod
117 def clearCache(cls):
118 """Clear the camera cache
120 This is principally intended to help memory leak tests pass.
121 """
122 cls._cameraCache = None
124 def map(self, datasetType, dataId, write=False):
125 """Need to strip 'flags' argument from map
127 We want the 'flags' argument passed to the butler to work (it's
128 used to change how the reading/writing is done), but want it
129 removed from the mapper (because it doesn't correspond to a
130 registry column).
131 """
132 copyId = dataId.copy()
133 copyId.pop("flags", None)
134 location = super(HscMapper, self).map(datasetType, copyId, write=write)
136 if 'flags' in dataId:
137 location.getAdditionalData().set('flags', dataId['flags'])
139 return location
141 @staticmethod
142 def _flipChipsLR(exp, wcs, detectorId, dims=None):
143 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs
145 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B
146 """
147 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False)
148 if exp:
149 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB))
150 if wcs:
151 ampDimensions = exp.getDimensions() if dims is None else dims
152 ampCenter = geom.Point2D(ampDimensions/2.0)
153 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter)
155 return exp, wcs
157 def std_raw_md(self, md, dataId):
158 """We need to flip the WCS defined by the metadata in case anyone ever
159 constructs a Wcs from it.
160 """
161 wcs = afwGeom.makeSkyWcs(md)
162 wcs = self._flipChipsLR(None, wcs, dataId['ccd'],
163 dims=afwImage.bboxFromMetadata(md).getDimensions())[1]
164 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper.
165 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(),
166 crval=wcs.getSkyOrigin(),
167 cdMatrix=wcs.getCdMatrix()*0.992)
168 wcsMd = wcsR.getFitsMetadata()
170 for k in wcsMd.names():
171 md.set(k, wcsMd.getScalar(k))
173 return md
175 def _createSkyWcsFromMetadata(self, exposure):
176 # Overridden to flip chips as necessary to get sensible SkyWcs.
177 metadata = exposure.getMetadata()
178 try:
179 wcs = afwGeom.makeSkyWcs(metadata, strip=True)
180 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId())
181 exposure.setWcs(wcs)
182 except lsst.pex.exceptions.TypeError as e:
183 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata).
184 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
185 " %s", e.args[0])
187 # ensure any WCS values stripped from the metadata are removed in the exposure
188 exposure.setMetadata(metadata)
190 def std_dark(self, item, dataId):
191 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False)
192 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
193 exposure.getInfo().setVisitInfo(visitInfo)
194 return exposure
196 def _extractDetectorName(self, dataId):
197 return int("%(ccd)d" % dataId)
199 def _computeCcdExposureId(self, dataId):
200 """Compute the 64-bit (long) identifier for a CCD exposure.
202 @param dataId (dict) Data identifier with visit, ccd
203 """
204 pathId = self._transformId(dataId)
205 visit = pathId['visit']
206 ccd = pathId['ccd']
207 return visit*200 + ccd
209 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
210 return self._computeCcdExposureId(dataId)
212 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
213 """How many bits are required for the maximum exposure ID"""
214 return 32 # just a guess, but this leaves plenty of space for sources
216 def map_linearizer(self, dataId, write=False):
217 """Map a linearizer."""
218 actualId = self._transformId(dataId)
219 return ButlerLocation(
220 pythonType="lsst.ip.isr.LinearizeSquared",
221 cppType="Config",
222 storageName="PickleStorage",
223 locationList="ignored",
224 dataId=actualId,
225 mapper=self,
226 storage=self.rootStorage)
228 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
229 """Return a linearizer for the given detector.
231 On each call, a fresh instance of `Linearizer` is returned; the caller is responsible for
232 initializing it appropriately for the detector.
234 Parameters
235 ----------
236 datasetType : `str``
237 The dataset type.
238 pythonType : `str` or `type`
239 Type of python object.
240 butlerLocation : `lsst.daf.persistence.ButlerLocation`
241 Struct-like class that holds information needed to persist and retrieve an object using
242 the LSST Persistence Framework.
243 dataId : `dict`
244 dataId passed to map location.
246 Returns
247 -------
248 Linearizer : `lsst.ip.isr.Linearizer`
249 Linearizer object for the given detector.
251 Notes
252 -----
253 Linearizers are not saved to persistent storage; rather, they are managed entirely in memory.
254 On each call, this function will return a new instance of `Linearizer`, which must be managed
255 (including setting it up for use with a particular detector) by the caller. Calling
256 `bypass_linearizer` twice for the same detector will return _different_ instances of `Linearizer`,
257 which share no state.
258 """
259 return Linearizer()
261 def _computeCoaddExposureId(self, dataId, singleFilter):
262 """Compute the 64-bit (long) identifier for a coadd.
264 @param dataId (dict) Data identifier with tract and patch.
265 @param singleFilter (bool) True means the desired ID is for a single-
266 filter coadd, in which case dataId
267 must contain filter.
268 """
270 tract = int(dataId['tract'])
271 if tract < 0 or tract >= 2**HscMapper._nbit_tract:
272 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract))
273 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
274 for p in (patchX, patchY):
275 if p < 0 or p >= 2**HscMapper._nbit_patch:
276 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch)
277 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY
278 if singleFilter:
279 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
280 return oid
282 def bypass_deepCoaddId_bits(self, *args, **kwargs):
283 """The number of bits used up for patch ID bits"""
284 return 64 - HscMapper._nbit_id
286 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
287 return self._computeCoaddExposureId(dataId, True)
289 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
290 """The number of bits used up for patch ID bits"""
291 return 64 - HscMapper._nbit_id
293 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
294 return self._computeCoaddExposureId(dataId, False)
296 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp
297 def map_psf(self, dataId, write=False):
298 if write:
299 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp")
300 copyId = dataId.copy()
301 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1))
302 return self.map_calexp_sub(copyId)
304 def std_psf(self, calexp, dataId):
305 return calexp.getPsf()
307 @classmethod
308 def getCameraName(cls):
309 return "hsc"