Coverage for python/lsst/obs/hsc/hscMapper.py: 24%
156 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:55 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:55 -0800
1import os
2import warnings
4import lsst.log
5from lsst.obs.base import CameraMapper
6from lsst.daf.persistence import ButlerLocation, Policy
7import lsst.afw.image as afwImage
8import lsst.afw.math as afwMath
9import lsst.afw.geom as afwGeom
10import lsst.geom as geom
11from lsst.ip.isr import Linearizer
12import lsst.pex.exceptions
13from .makeHscRawVisitInfo import MakeHscRawVisitInfo
14from .hscPupil import HscPupilFactory
15from .hscFilters import HSC_FILTER_DEFINITIONS
18class HscMapper(CameraMapper):
19 """Provides abstract-physical mapping for HSC data"""
20 packageName = "obs_subaru"
22 MakeRawVisitInfoClass = MakeHscRawVisitInfo
24 PupilFactoryClass = HscPupilFactory
26 # Use the full instrument class name to prevent import errors
27 # between hsc/ and subaru/ packages.
28 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam"
29 _cameraCache = None # Camera object, cached to speed up instantiation time
31 @classmethod
32 def addFilters(cls):
33 HSC_FILTER_DEFINITIONS.defineFilters()
35 def __init__(self, **kwargs):
36 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy")
37 policy = Policy(policyFile)
38 if not kwargs.get('root', None):
39 try:
40 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC')
41 except Exception:
42 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified")
43 if not kwargs.get('calibRoot', None):
44 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
45 if "repositoryCfg" in kwargs:
46 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
47 hasattr(cfg, "root")]
48 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
49 for calibRoot in calibSearch:
50 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
51 kwargs['calibRoot'] = calibRoot
52 break
53 if not kwargs.get('calibRoot', None):
54 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory")
56 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
58 # Ensure each dataset type of interest knows about the full range of
59 # keys available from the registry
60 keys = {'field': str,
61 'visit': int,
62 'filter': str,
63 'ccd': int,
64 'dateObs': str,
65 'taiObs': str,
66 'expTime': float,
67 'pointing': int,
68 }
69 for name in ("raw",
70 # processCcd outputs
71 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch",
72 "srcMatch",
73 # mosaic outputs
74 "wcs", "fcr",
75 # processCcd QA
76 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough",
77 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap",
78 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid",
79 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid",
80 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid",
81 # forcedPhot outputs
82 "forced_src",
83 ):
84 self.mappings[name].keyDict.update(keys)
86 self.addFilters()
88 self.filters = {}
89 with warnings.catch_warnings():
90 # surpress Filter warnings; we already know this is deprecated
91 warnings.simplefilter('ignore', category=FutureWarning)
92 for filt in HSC_FILTER_DEFINITIONS:
93 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName()
94 self.defaultFilterName = "unknown"
96 #
97 # The number of bits allocated for fields in object IDs, appropriate
98 # for the default-configured Rings skymap.
99 #
100 # This shouldn't be the mapper's job at all; see #2797.
102 HscMapper._nbit_tract = 16
103 HscMapper._nbit_patch = 5
104 HscMapper._nbit_filter = 6
106 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter)
108 with warnings.catch_warnings():
109 # surpress Filter warnings; we already know this is deprecated
110 warnings.simplefilter('ignore', category=FutureWarning)
111 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter:
112 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %
113 HscMapper._nbit_filter)
115 def _makeCamera(self, *args, **kwargs):
116 """Make the camera object
118 This implementation layers a cache over the parent class'
119 implementation. Caching the camera improves the instantiation
120 time for the HscMapper because parsing the camera's Config
121 involves a lot of 'stat' calls (through the tracebacks).
122 """
123 if not self._cameraCache:
124 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs)
125 return self._cameraCache
127 @classmethod
128 def clearCache(cls):
129 """Clear the camera cache
131 This is principally intended to help memory leak tests pass.
132 """
133 cls._cameraCache = None
135 def map(self, datasetType, dataId, write=False):
136 """Need to strip 'flags' argument from map
138 We want the 'flags' argument passed to the butler to work (it's
139 used to change how the reading/writing is done), but want it
140 removed from the mapper (because it doesn't correspond to a
141 registry column).
142 """
143 copyId = dataId.copy()
144 copyId.pop("flags", None)
145 location = super(HscMapper, self).map(datasetType, copyId, write=write)
147 if 'flags' in dataId:
148 location.getAdditionalData().set('flags', dataId['flags'])
150 return location
152 @staticmethod
153 def _flipChipsLR(exp, wcs, detectorId, dims=None):
154 """Flip the chip left/right or top/bottom. Process either/and the
155 pixels and wcs
157 Most chips are flipped L/R, but the rotated ones (100..103) are
158 flipped T/B.
159 """
160 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False)
161 if exp:
162 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB))
163 if wcs:
164 ampDimensions = exp.getDimensions() if dims is None else dims
165 ampCenter = geom.Point2D(ampDimensions/2.0)
166 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter)
168 return exp, wcs
170 def std_raw_md(self, md, dataId):
171 """We need to flip the WCS defined by the metadata in case anyone ever
172 constructs a Wcs from it.
173 """
174 wcs = afwGeom.makeSkyWcs(md)
175 wcs = self._flipChipsLR(None, wcs, dataId['ccd'],
176 dims=afwImage.bboxFromMetadata(md).getDimensions())[1]
177 # NOTE: we don't know where the 0.992 magic constant came from.
178 # It was copied over from hscSimMapper.
179 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(),
180 crval=wcs.getSkyOrigin(),
181 cdMatrix=wcs.getCdMatrix()*0.992)
182 wcsMd = wcsR.getFitsMetadata()
184 for k in wcsMd.names():
185 md.set(k, wcsMd.getScalar(k))
187 return md
189 def _createSkyWcsFromMetadata(self, exposure):
190 # Overridden to flip chips as necessary to get sensible SkyWcs.
191 metadata = exposure.getMetadata()
192 try:
193 wcs = afwGeom.makeSkyWcs(metadata, strip=True)
194 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId())
195 exposure.setWcs(wcs)
196 except lsst.pex.exceptions.TypeError as e:
197 # See DM-14372 for why this is debug and not warn (e.g. calib
198 # files without wcs metadata).
199 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
200 " %s", e.args[0])
202 # ensure any WCS values stripped from the metadata are removed in
203 # the exposure
204 exposure.setMetadata(metadata)
206 def std_dark(self, item, dataId):
207 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False)
208 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
209 exposure.getInfo().setVisitInfo(visitInfo)
210 return exposure
212 def _extractDetectorName(self, dataId):
213 return int("%(ccd)d" % dataId)
215 def _computeCcdExposureId(self, dataId):
216 """Compute the 64-bit (long) identifier for a CCD exposure.
218 @param dataId (dict) Data identifier with visit, ccd
219 """
220 pathId = self._transformId(dataId)
221 visit = pathId['visit']
222 ccd = pathId['ccd']
223 return visit*200 + ccd
225 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
226 return self._computeCcdExposureId(dataId)
228 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
229 """How many bits are required for the maximum exposure ID"""
230 return 32 # just a guess, but this leaves plenty of space for sources
232 def map_linearizer(self, dataId, write=False):
233 """Map a linearizer."""
234 actualId = self._transformId(dataId)
235 return ButlerLocation(
236 pythonType="lsst.ip.isr.LinearizeSquared",
237 cppType="Config",
238 storageName="PickleStorage",
239 locationList="ignored",
240 dataId=actualId,
241 mapper=self,
242 storage=self.rootStorage)
244 def map_crosstalk(self, dataId, write=False):
245 """Fake the mapping for crosstalk.
247 Crosstalk is constructed from config parameters, but we need
248 Gen2 butlers to be able to respond to requests for it.
249 Returning None provides a response that can be used with the
250 config parameters to generate the appropriate calibration.
251 """
252 return None
254 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
255 """Return a linearizer for the given detector.
257 On each call, a fresh instance of `Linearizer` is returned; the caller
258 is responsible for initializing it appropriately for the detector.
260 Parameters
261 ----------
262 datasetType : `str``
263 The dataset type.
264 pythonType : `str` or `type`
265 Type of python object.
266 butlerLocation : `lsst.daf.persistence.ButlerLocation`
267 Struct-like class that holds information needed to persist and
268 retrieve an object using the LSST Persistence Framework.
269 dataId : `dict`
270 dataId passed to map location.
272 Returns
273 -------
274 Linearizer : `lsst.ip.isr.Linearizer`
275 Linearizer object for the given detector.
277 Notes
278 -----
279 Linearizers are not saved to persistent storage; rather, they are
280 managed entirely in memory. On each call, this function will return a
281 new instance of `Linearizer`, which must be managed (including setting
282 it up for use with a particular detector) by the caller. Calling
283 `bypass_linearizer` twice for the same detector will return
284 _different_ instances of `Linearizer`, which share no state.
285 """
286 return Linearizer(detectorId=dataId.get('ccd', None))
288 def _computeCoaddExposureId(self, dataId, singleFilter):
289 """Compute the 64-bit (long) identifier for a coadd.
291 @param dataId (dict) Data identifier with tract and patch.
292 @param singleFilter (bool) True means the desired ID is for a single-
293 filter coadd, in which case dataId
294 must contain filter.
295 """
297 tract = int(dataId['tract'])
298 if tract < 0 or tract >= 2**HscMapper._nbit_tract:
299 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract))
300 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
301 for p in (patchX, patchY):
302 if p < 0 or p >= 2**HscMapper._nbit_patch:
303 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch)
304 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY
305 if singleFilter:
306 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
307 return oid
309 def bypass_deepCoaddId_bits(self, *args, **kwargs):
310 """The number of bits used up for patch ID bits"""
311 return 64 - HscMapper._nbit_id
313 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
314 return self._computeCoaddExposureId(dataId, True)
316 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
317 """The number of bits used up for patch ID bits"""
318 return 64 - HscMapper._nbit_id
320 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
321 return self._computeCoaddExposureId(dataId, False)
323 # The following allow grabbing a 'psf' from the butler directly, without
324 # having to get it from a calexp
325 def map_psf(self, dataId, write=False):
326 if write:
327 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp")
328 copyId = dataId.copy()
329 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1))
330 return self.map_calexp_sub(copyId)
332 def std_psf(self, calexp, dataId):
333 return calexp.getPsf()
335 @classmethod
336 def getCameraName(cls):
337 return "hsc"