Coverage for python/lsst/obs/hsc/hscMapper.py: 25%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

159 statements  

1import os 

2 

3import lsst.log 

4from lsst.obs.base import CameraMapper 

5from lsst.daf.persistence import ButlerLocation, Policy 

6import lsst.afw.image as afwImage 

7import lsst.afw.math as afwMath 

8import lsst.afw.geom as afwGeom 

9import lsst.geom as geom 

10from lsst.ip.isr import Linearizer 

11import lsst.pex.exceptions 

12from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

13from .hscPupil import HscPupilFactory 

14from .hscFilters import HSC_FILTER_DEFINITIONS 

15 

16 

17class HscMapper(CameraMapper): 

18 """Provides abstract-physical mapping for HSC data""" 

19 packageName = "obs_subaru" 

20 

21 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

22 

23 PupilFactoryClass = HscPupilFactory 

24 filterDefinitions = HSC_FILTER_DEFINITIONS 

25 

26 # Use the full instrument class name to prevent import errors 

27 # between hsc/ and subaru/ packages. 

28 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam" 

29 _cameraCache = None # Camera object, cached to speed up instantiation time 

30 

31 @classmethod 

32 def addFilters(cls): 

33 HSC_FILTER_DEFINITIONS.defineFilters() 

34 

35 def __init__(self, **kwargs): 

36 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

37 policy = Policy(policyFile) 

38 if not kwargs.get('root', None): 

39 try: 

40 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

41 except Exception: 

42 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

43 if not kwargs.get('calibRoot', None): 

44 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

45 if "repositoryCfg" in kwargs: 

46 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

47 hasattr(cfg, "root")] 

48 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

49 for calibRoot in calibSearch: 

50 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

51 kwargs['calibRoot'] = calibRoot 

52 break 

53 if not kwargs.get('calibRoot', None): 

54 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

55 

56 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

57 

58 # Ensure each dataset type of interest knows about the full range of 

59 # keys available from the registry 

60 keys = {'field': str, 

61 'visit': int, 

62 'filter': str, 

63 'ccd': int, 

64 'dateObs': str, 

65 'taiObs': str, 

66 'expTime': float, 

67 'pointing': int, 

68 } 

69 for name in ("raw", 

70 # processCcd outputs 

71 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

72 "srcMatch", 

73 # mosaic outputs 

74 "wcs", "fcr", 

75 # processCcd QA 

76 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

77 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

78 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

79 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

80 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

81 # forcedPhot outputs 

82 "forced_src", 

83 ): 

84 self.mappings[name].keyDict.update(keys) 

85 

86 self.filters = {} 

87 self.bandToIdNumDict = {} # this is to get rid of afwFilter.getId calls 

88 filterNum = -1 

89 for filterDef in self.filterDefinitions: 

90 physical_filter = filterDef.physical_filter 

91 band = filterDef.band 

92 self.filters[physical_filter] = band 

93 if band not in self.bandToIdNumDict: 

94 filterNum += 1 

95 self.bandToIdNumDict[band] = filterNum 

96 # 

97 # The number of bits allocated for fields in object IDs, appropriate 

98 # for the default-configured Rings skymap. 

99 # 

100 # This shouldn't be the mapper's job at all; see #2797. 

101 

102 HscMapper._nbit_tract = 17 # Changes to mimic effective Gen3 behavior 

103 HscMapper._nbit_patch = 8 

104 HscMapper._nbit_filter = 5 

105 

106 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

107 if len(self.bandToIdNumDict) >= 2**HscMapper._nbit_filter: 

108 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

109 HscMapper._nbit_filter) 

110 

111 def _makeCamera(self, *args, **kwargs): 

112 """Make the camera object 

113 

114 This implementation layers a cache over the parent class' 

115 implementation. Caching the camera improves the instantiation 

116 time for the HscMapper because parsing the camera's Config 

117 involves a lot of 'stat' calls (through the tracebacks). 

118 """ 

119 if not self._cameraCache: 

120 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

121 return self._cameraCache 

122 

123 @classmethod 

124 def clearCache(cls): 

125 """Clear the camera cache 

126 

127 This is principally intended to help memory leak tests pass. 

128 """ 

129 cls._cameraCache = None 

130 

131 def map(self, datasetType, dataId, write=False): 

132 """Need to strip 'flags' argument from map 

133 

134 We want the 'flags' argument passed to the butler to work (it's 

135 used to change how the reading/writing is done), but want it 

136 removed from the mapper (because it doesn't correspond to a 

137 registry column). 

138 """ 

139 copyId = dataId.copy() 

140 copyId.pop("flags", None) 

141 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

142 

143 if 'flags' in dataId: 

144 location.getAdditionalData().set('flags', dataId['flags']) 

145 

146 return location 

147 

148 @staticmethod 

149 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

150 """Flip the chip left/right or top/bottom. Process either/and the 

151 pixels and wcs 

152 

153 Most chips are flipped L/R, but the rotated ones (100..103) are 

154 flipped T/B. 

155 """ 

156 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

157 if exp: 

158 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

159 if wcs: 

160 ampDimensions = exp.getDimensions() if dims is None else dims 

161 ampCenter = geom.Point2D(ampDimensions/2.0) 

162 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

163 

164 return exp, wcs 

165 

166 def std_raw_md(self, md, dataId): 

167 """We need to flip the WCS defined by the metadata in case anyone ever 

168 constructs a Wcs from it. 

169 """ 

170 wcs = afwGeom.makeSkyWcs(md) 

171 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

172 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

173 # NOTE: we don't know where the 0.992 magic constant came from. 

174 # It was copied over from hscSimMapper. 

175 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

176 crval=wcs.getSkyOrigin(), 

177 cdMatrix=wcs.getCdMatrix()*0.992) 

178 wcsMd = wcsR.getFitsMetadata() 

179 

180 for k in wcsMd.names(): 

181 md.set(k, wcsMd.getScalar(k)) 

182 

183 return md 

184 

185 def _createSkyWcsFromMetadata(self, exposure): 

186 # Overridden to flip chips as necessary to get sensible SkyWcs. 

187 metadata = exposure.getMetadata() 

188 try: 

189 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

190 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

191 exposure.setWcs(wcs) 

192 except lsst.pex.exceptions.TypeError as e: 

193 # See DM-14372 for why this is debug and not warn (e.g. calib 

194 # files without wcs metadata). 

195 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

196 " %s", e.args[0]) 

197 

198 # ensure any WCS values stripped from the metadata are removed in 

199 # the exposure 

200 exposure.setMetadata(metadata) 

201 

202 def std_dark(self, item, dataId): 

203 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

204 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

205 exposure.getInfo().setVisitInfo(visitInfo) 

206 return exposure 

207 

208 def _extractDetectorName(self, dataId): 

209 return int("%(ccd)d" % dataId) 

210 

211 def _computeCcdExposureId(self, dataId): 

212 """Compute the 64-bit (long) identifier for a CCD exposure. 

213 

214 @param dataId (dict) Data identifier with visit, ccd 

215 """ 

216 pathId = self._transformId(dataId) 

217 visit = pathId['visit'] 

218 ccd = pathId['ccd'] 

219 return visit*200 + ccd 

220 

221 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

222 return self._computeCcdExposureId(dataId) 

223 

224 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

225 """How many bits are required for the maximum exposure ID""" 

226 return 32 # just a guess, but this leaves plenty of space for sources 

227 

228 def map_linearizer(self, dataId, write=False): 

229 """Map a linearizer.""" 

230 actualId = self._transformId(dataId) 

231 return ButlerLocation( 

232 pythonType="lsst.ip.isr.LinearizeSquared", 

233 cppType="Config", 

234 storageName="PickleStorage", 

235 locationList="ignored", 

236 dataId=actualId, 

237 mapper=self, 

238 storage=self.rootStorage) 

239 

240 def map_crosstalk(self, dataId, write=False): 

241 """Fake the mapping for crosstalk. 

242 

243 Crosstalk is constructed from config parameters, but we need 

244 Gen2 butlers to be able to respond to requests for it. 

245 Returning None provides a response that can be used with the 

246 config parameters to generate the appropriate calibration. 

247 """ 

248 return None 

249 

250 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

251 """Return a linearizer for the given detector. 

252 

253 On each call, a fresh instance of `Linearizer` is returned; the caller 

254 is responsible for initializing it appropriately for the detector. 

255 

256 Parameters 

257 ---------- 

258 datasetType : `str`` 

259 The dataset type. 

260 pythonType : `str` or `type` 

261 Type of python object. 

262 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

263 Struct-like class that holds information needed to persist and 

264 retrieve an object using the LSST Persistence Framework. 

265 dataId : `dict` 

266 dataId passed to map location. 

267 

268 Returns 

269 ------- 

270 Linearizer : `lsst.ip.isr.Linearizer` 

271 Linearizer object for the given detector. 

272 

273 Notes 

274 ----- 

275 Linearizers are not saved to persistent storage; rather, they are 

276 managed entirely in memory. On each call, this function will return a 

277 new instance of `Linearizer`, which must be managed (including setting 

278 it up for use with a particular detector) by the caller. Calling 

279 `bypass_linearizer` twice for the same detector will return 

280 _different_ instances of `Linearizer`, which share no state. 

281 """ 

282 return Linearizer(detectorId=dataId.get('ccd', None)) 

283 

284 def _computeCoaddExposureId(self, dataId, singleFilter): 

285 """Compute the 64-bit (long) identifier for a coadd. 

286 

287 @param dataId (dict) Data identifier with tract and patch. 

288 @param singleFilter (bool) True means the desired ID is for a single- 

289 filter coadd, in which case dataId 

290 must contain filter. 

291 """ 

292 

293 tract = int(dataId['tract']) 

294 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

295 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

296 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

297 for p in (patchX, patchY): 

298 if p < 0 or p >= 2**HscMapper._nbit_patch: 

299 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

300 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

301 if singleFilter: 

302 return (oid << HscMapper._nbit_filter) + self.bandToIdNumDict[self.filters[dataId['filter']]] 

303 return oid 

304 

305 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId): 

306 """Return the canonical/generic band name associated with the filter in 

307 the dataId. 

308 """ 

309 return(self.filters[dataId["filter"]]) 

310 

311 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

312 """The number of bits used up for patch ID bits""" 

313 return HscMapper._nbit_id 

314 

315 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

316 return self._computeCoaddExposureId(dataId, True) 

317 

318 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

319 """The number of bits used up for patch ID bits""" 

320 return HscMapper._nbit_id - HscMapper._nbit_filter 

321 

322 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

323 return self._computeCoaddExposureId(dataId, False) 

324 

325 # The following allow grabbing a 'psf' from the butler directly, without 

326 # having to get it from a calexp 

327 def map_psf(self, dataId, write=False): 

328 if write: 

329 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

330 copyId = dataId.copy() 

331 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

332 return self.map_calexp_sub(copyId) 

333 

334 def std_psf(self, calexp, dataId): 

335 return calexp.getPsf() 

336 

337 @classmethod 

338 def getCameraName(cls): 

339 return "hsc"