Coverage for python/lsst/obs/hsc/hscMapper.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
3import lsst.log
4from lsst.obs.base import CameraMapper
5from lsst.daf.persistence import ButlerLocation, Policy
6import lsst.afw.image as afwImage
7import lsst.afw.math as afwMath
8import lsst.afw.geom as afwGeom
9import lsst.geom as geom
10from lsst.ip.isr import Linearizer
11import lsst.pex.exceptions
12from .makeHscRawVisitInfo import MakeHscRawVisitInfo
13from .hscPupil import HscPupilFactory
14from .hscFilters import HSC_FILTER_DEFINITIONS
17class HscMapper(CameraMapper):
18 """Provides abstract-physical mapping for HSC data"""
19 packageName = "obs_subaru"
21 MakeRawVisitInfoClass = MakeHscRawVisitInfo
23 PupilFactoryClass = HscPupilFactory
25 _cameraCache = None # Camera object, cached to speed up instantiation time
27 @classmethod
28 def addFilters(cls):
29 HSC_FILTER_DEFINITIONS.defineFilters()
31 def __init__(self, **kwargs):
32 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy")
33 policy = Policy(policyFile)
34 if not kwargs.get('root', None):
35 try:
36 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC')
37 except Exception:
38 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified")
39 if not kwargs.get('calibRoot', None):
40 calibSearch = [os.path.join(kwargs['root'], 'CALIB')]
41 if "repositoryCfg" in kwargs:
42 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if
43 hasattr(cfg, "root")]
44 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")]
45 for calibRoot in calibSearch:
46 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")):
47 kwargs['calibRoot'] = calibRoot
48 break
49 if not kwargs.get('calibRoot', None):
50 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory")
52 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs)
54 self._linearize = Linearizer()
56 # Ensure each dataset type of interest knows about the full range of keys available from the registry
57 keys = {'field': str,
58 'visit': int,
59 'filter': str,
60 'ccd': int,
61 'dateObs': str,
62 'taiObs': str,
63 'expTime': float,
64 'pointing': int,
65 }
66 for name in ("raw",
67 # processCcd outputs
68 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch",
69 "srcMatch",
70 # mosaic outputs
71 "wcs", "fcr",
72 # processCcd QA
73 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough",
74 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap",
75 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid",
76 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid",
77 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid",
78 # forcedPhot outputs
79 "forced_src",
80 ):
81 self.mappings[name].keyDict.update(keys)
83 self.addFilters()
85 self.filters = {}
86 for filt in HSC_FILTER_DEFINITIONS:
87 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName()
88 self.defaultFilterName = "UNRECOGNISED"
90 #
91 # The number of bits allocated for fields in object IDs, appropriate for
92 # the default-configured Rings skymap.
93 #
94 # This shouldn't be the mapper's job at all; see #2797.
96 HscMapper._nbit_tract = 16
97 HscMapper._nbit_patch = 5
98 HscMapper._nbit_filter = 6
100 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter)
102 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter:
103 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" %
104 HscMapper._nbit_filter)
106 def _makeCamera(self, *args, **kwargs):
107 """Make the camera object
109 This implementation layers a cache over the parent class'
110 implementation. Caching the camera improves the instantiation
111 time for the HscMapper because parsing the camera's Config
112 involves a lot of 'stat' calls (through the tracebacks).
113 """
114 if not self._cameraCache:
115 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs)
116 return self._cameraCache
118 @classmethod
119 def clearCache(cls):
120 """Clear the camera cache
122 This is principally intended to help memory leak tests pass.
123 """
124 cls._cameraCache = None
126 def map(self, datasetType, dataId, write=False):
127 """Need to strip 'flags' argument from map
129 We want the 'flags' argument passed to the butler to work (it's
130 used to change how the reading/writing is done), but want it
131 removed from the mapper (because it doesn't correspond to a
132 registry column).
133 """
134 copyId = dataId.copy()
135 copyId.pop("flags", None)
136 location = super(HscMapper, self).map(datasetType, copyId, write=write)
138 if 'flags' in dataId:
139 location.getAdditionalData().set('flags', dataId['flags'])
141 return location
143 @staticmethod
144 def _flipChipsLR(exp, wcs, detectorId, dims=None):
145 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs
147 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B
148 """
149 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False)
150 if exp:
151 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB))
152 if wcs:
153 ampDimensions = exp.getDimensions() if dims is None else dims
154 ampCenter = geom.Point2D(ampDimensions/2.0)
155 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter)
157 return exp, wcs
159 def std_raw_md(self, md, dataId):
160 """We need to flip the WCS defined by the metadata in case anyone ever
161 constructs a Wcs from it.
162 """
163 wcs = afwGeom.makeSkyWcs(md)
164 wcs = self._flipChipsLR(None, wcs, dataId['ccd'],
165 dims=afwImage.bboxFromMetadata(md).getDimensions())[1]
166 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper.
167 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(),
168 crval=wcs.getSkyOrigin(),
169 cdMatrix=wcs.getCdMatrix()*0.992)
170 wcsMd = wcsR.getFitsMetadata()
172 for k in wcsMd.names():
173 md.set(k, wcsMd.getScalar(k))
175 return md
177 def _createSkyWcsFromMetadata(self, exposure):
178 # Overridden to flip chips as necessary to get sensible SkyWcs.
179 metadata = exposure.getMetadata()
180 try:
181 wcs = afwGeom.makeSkyWcs(metadata, strip=True)
182 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId())
183 exposure.setWcs(wcs)
184 except lsst.pex.exceptions.TypeError as e:
185 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata).
186 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:"
187 " %s", e.args[0])
189 # ensure any WCS values stripped from the metadata are removed in the exposure
190 exposure.setMetadata(metadata)
192 def std_dark(self, item, dataId):
193 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False)
194 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
195 exposure.getInfo().setVisitInfo(visitInfo)
196 return exposure
198 def _extractAmpId(self, dataId):
199 return (self._extractDetectorName(dataId), 0, 0)
201 def _extractDetectorName(self, dataId):
202 return int("%(ccd)d" % dataId)
204 def _computeCcdExposureId(self, dataId):
205 """Compute the 64-bit (long) identifier for a CCD exposure.
207 @param dataId (dict) Data identifier with visit, ccd
208 """
209 pathId = self._transformId(dataId)
210 visit = pathId['visit']
211 ccd = pathId['ccd']
212 return visit*200 + ccd
214 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
215 return self._computeCcdExposureId(dataId)
217 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
218 """How many bits are required for the maximum exposure ID"""
219 return 32 # just a guess, but this leaves plenty of space for sources
221 def map_linearizer(self, dataId, write=False):
222 """Map a linearizer."""
223 if self._linearize is None:
224 raise RuntimeError("No linearizer available.")
225 actualId = self._transformId(dataId)
226 return ButlerLocation(
227 pythonType="lsst.ip.isr.LinearizeSquared",
228 cppType="Config",
229 storageName="PickleStorage",
230 locationList="ignored",
231 dataId=actualId,
232 mapper=self,
233 storage=self.rootStorage)
235 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
236 """Return the linearizer.
237 """
238 if self._linearize is None:
239 raise RuntimeError("No linearizer available.")
240 return self._linearize
242 def _computeCoaddExposureId(self, dataId, singleFilter):
243 """Compute the 64-bit (long) identifier for a coadd.
245 @param dataId (dict) Data identifier with tract and patch.
246 @param singleFilter (bool) True means the desired ID is for a single-
247 filter coadd, in which case dataId
248 must contain filter.
249 """
251 tract = int(dataId['tract'])
252 if tract < 0 or tract >= 2**HscMapper._nbit_tract:
253 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract))
254 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')]
255 for p in (patchX, patchY):
256 if p < 0 or p >= 2**HscMapper._nbit_patch:
257 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch)
258 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY
259 if singleFilter:
260 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId()
261 return oid
263 def bypass_deepCoaddId_bits(self, *args, **kwargs):
264 """The number of bits used up for patch ID bits"""
265 return 64 - HscMapper._nbit_id
267 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
268 return self._computeCoaddExposureId(dataId, True)
270 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs):
271 """The number of bits used up for patch ID bits"""
272 return 64 - HscMapper._nbit_id
274 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
275 return self._computeCoaddExposureId(dataId, False)
277 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp
278 def map_psf(self, dataId, write=False):
279 if write:
280 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp")
281 copyId = dataId.copy()
282 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1))
283 return self.map_calexp_sub(copyId)
285 def std_psf(self, calexp, dataId):
286 return calexp.getPsf()
288 @classmethod
289 def getCameraName(cls):
290 return "hsc"