Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2 

3import lsst.log 

4from lsst.obs.base import CameraMapper 

5from lsst.daf.persistence import ButlerLocation, Policy 

6import lsst.afw.image as afwImage 

7import lsst.afw.math as afwMath 

8import lsst.afw.geom as afwGeom 

9import lsst.geom as geom 

10from lsst.ip.isr import Linearizer 

11import lsst.pex.exceptions 

12from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

13from .hscPupil import HscPupilFactory 

14from .hscFilters import HSC_FILTER_DEFINITIONS 

15 

16 

17class HscMapper(CameraMapper): 

18 """Provides abstract-physical mapping for HSC data""" 

19 packageName = "obs_subaru" 

20 

21 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

22 

23 PupilFactoryClass = HscPupilFactory 

24 

25 # Use the full instrument class name to prevent import errors 

26 # between hsc/ and subaru/ packages. 

27 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam" 

28 _cameraCache = None # Camera object, cached to speed up instantiation time 

29 

30 @classmethod 

31 def addFilters(cls): 

32 HSC_FILTER_DEFINITIONS.defineFilters() 

33 

34 def __init__(self, **kwargs): 

35 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

36 policy = Policy(policyFile) 

37 if not kwargs.get('root', None): 

38 try: 

39 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

40 except Exception: 

41 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

42 if not kwargs.get('calibRoot', None): 

43 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

44 if "repositoryCfg" in kwargs: 

45 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

46 hasattr(cfg, "root")] 

47 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

48 for calibRoot in calibSearch: 

49 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

50 kwargs['calibRoot'] = calibRoot 

51 break 

52 if not kwargs.get('calibRoot', None): 

53 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

54 

55 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

56 

57 # Ensure each dataset type of interest knows about the full range of 

58 # keys available from the registry 

59 keys = {'field': str, 

60 'visit': int, 

61 'filter': str, 

62 'ccd': int, 

63 'dateObs': str, 

64 'taiObs': str, 

65 'expTime': float, 

66 'pointing': int, 

67 } 

68 for name in ("raw", 

69 # processCcd outputs 

70 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

71 "srcMatch", 

72 # mosaic outputs 

73 "wcs", "fcr", 

74 # processCcd QA 

75 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

76 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

77 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

78 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

79 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

80 # forcedPhot outputs 

81 "forced_src", 

82 ): 

83 self.mappings[name].keyDict.update(keys) 

84 

85 self.addFilters() 

86 

87 self.filters = {} 

88 for filt in HSC_FILTER_DEFINITIONS: 

89 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() 

90 self.defaultFilterName = "UNRECOGNISED" 

91 

92 # 

93 # The number of bits allocated for fields in object IDs, appropriate 

94 # for the default-configured Rings skymap. 

95 # 

96 # This shouldn't be the mapper's job at all; see #2797. 

97 

98 HscMapper._nbit_tract = 16 

99 HscMapper._nbit_patch = 5 

100 HscMapper._nbit_filter = 6 

101 

102 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

103 

104 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: 

105 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

106 HscMapper._nbit_filter) 

107 

108 def _makeCamera(self, *args, **kwargs): 

109 """Make the camera object 

110 

111 This implementation layers a cache over the parent class' 

112 implementation. Caching the camera improves the instantiation 

113 time for the HscMapper because parsing the camera's Config 

114 involves a lot of 'stat' calls (through the tracebacks). 

115 """ 

116 if not self._cameraCache: 

117 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

118 return self._cameraCache 

119 

120 @classmethod 

121 def clearCache(cls): 

122 """Clear the camera cache 

123 

124 This is principally intended to help memory leak tests pass. 

125 """ 

126 cls._cameraCache = None 

127 

128 def map(self, datasetType, dataId, write=False): 

129 """Need to strip 'flags' argument from map 

130 

131 We want the 'flags' argument passed to the butler to work (it's 

132 used to change how the reading/writing is done), but want it 

133 removed from the mapper (because it doesn't correspond to a 

134 registry column). 

135 """ 

136 copyId = dataId.copy() 

137 copyId.pop("flags", None) 

138 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

139 

140 if 'flags' in dataId: 

141 location.getAdditionalData().set('flags', dataId['flags']) 

142 

143 return location 

144 

145 @staticmethod 

146 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

147 """Flip the chip left/right or top/bottom. Process either/and the 

148 pixels and wcs 

149 

150 Most chips are flipped L/R, but the rotated ones (100..103) are 

151 flipped T/B. 

152 """ 

153 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

154 if exp: 

155 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

156 if wcs: 

157 ampDimensions = exp.getDimensions() if dims is None else dims 

158 ampCenter = geom.Point2D(ampDimensions/2.0) 

159 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

160 

161 return exp, wcs 

162 

163 def std_raw_md(self, md, dataId): 

164 """We need to flip the WCS defined by the metadata in case anyone ever 

165 constructs a Wcs from it. 

166 """ 

167 wcs = afwGeom.makeSkyWcs(md) 

168 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

169 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

170 # NOTE: we don't know where the 0.992 magic constant came from. 

171 # It was copied over from hscSimMapper. 

172 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

173 crval=wcs.getSkyOrigin(), 

174 cdMatrix=wcs.getCdMatrix()*0.992) 

175 wcsMd = wcsR.getFitsMetadata() 

176 

177 for k in wcsMd.names(): 

178 md.set(k, wcsMd.getScalar(k)) 

179 

180 return md 

181 

182 def _createSkyWcsFromMetadata(self, exposure): 

183 # Overridden to flip chips as necessary to get sensible SkyWcs. 

184 metadata = exposure.getMetadata() 

185 try: 

186 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

187 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

188 exposure.setWcs(wcs) 

189 except lsst.pex.exceptions.TypeError as e: 

190 # See DM-14372 for why this is debug and not warn (e.g. calib 

191 # files without wcs metadata). 

192 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

193 " %s", e.args[0]) 

194 

195 # ensure any WCS values stripped from the metadata are removed in 

196 # the exposure 

197 exposure.setMetadata(metadata) 

198 

199 def std_dark(self, item, dataId): 

200 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

201 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

202 exposure.getInfo().setVisitInfo(visitInfo) 

203 return exposure 

204 

205 def _extractDetectorName(self, dataId): 

206 return int("%(ccd)d" % dataId) 

207 

208 def _computeCcdExposureId(self, dataId): 

209 """Compute the 64-bit (long) identifier for a CCD exposure. 

210 

211 @param dataId (dict) Data identifier with visit, ccd 

212 """ 

213 pathId = self._transformId(dataId) 

214 visit = pathId['visit'] 

215 ccd = pathId['ccd'] 

216 return visit*200 + ccd 

217 

218 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

219 return self._computeCcdExposureId(dataId) 

220 

221 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

222 """How many bits are required for the maximum exposure ID""" 

223 return 32 # just a guess, but this leaves plenty of space for sources 

224 

225 def map_linearizer(self, dataId, write=False): 

226 """Map a linearizer.""" 

227 actualId = self._transformId(dataId) 

228 return ButlerLocation( 

229 pythonType="lsst.ip.isr.LinearizeSquared", 

230 cppType="Config", 

231 storageName="PickleStorage", 

232 locationList="ignored", 

233 dataId=actualId, 

234 mapper=self, 

235 storage=self.rootStorage) 

236 

237 def map_crosstalk(self, dataId, write=False): 

238 """Fake the mapping for crosstalk. 

239 

240 Crosstalk is constructed from config parameters, but we need 

241 Gen2 butlers to be able to respond to requests for it. 

242 Returning None provides a response that can be used with the 

243 config parameters to generate the appropriate calibration. 

244 """ 

245 return None 

246 

247 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

248 """Return a linearizer for the given detector. 

249 

250 On each call, a fresh instance of `Linearizer` is returned; the caller 

251 is responsible for initializing it appropriately for the detector. 

252 

253 Parameters 

254 ---------- 

255 datasetType : `str`` 

256 The dataset type. 

257 pythonType : `str` or `type` 

258 Type of python object. 

259 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

260 Struct-like class that holds information needed to persist and 

261 retrieve an object using the LSST Persistence Framework. 

262 dataId : `dict` 

263 dataId passed to map location. 

264 

265 Returns 

266 ------- 

267 Linearizer : `lsst.ip.isr.Linearizer` 

268 Linearizer object for the given detector. 

269 

270 Notes 

271 ----- 

272 Linearizers are not saved to persistent storage; rather, they are 

273 managed entirely in memory. On each call, this function will return a 

274 new instance of `Linearizer`, which must be managed (including setting 

275 it up for use with a particular detector) by the caller. Calling 

276 `bypass_linearizer` twice for the same detector will return 

277 _different_ instances of `Linearizer`, which share no state. 

278 """ 

279 return Linearizer(detectorId=dataId.get('ccd', None)) 

280 

281 def _computeCoaddExposureId(self, dataId, singleFilter): 

282 """Compute the 64-bit (long) identifier for a coadd. 

283 

284 @param dataId (dict) Data identifier with tract and patch. 

285 @param singleFilter (bool) True means the desired ID is for a single- 

286 filter coadd, in which case dataId 

287 must contain filter. 

288 """ 

289 

290 tract = int(dataId['tract']) 

291 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

292 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

293 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

294 for p in (patchX, patchY): 

295 if p < 0 or p >= 2**HscMapper._nbit_patch: 

296 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

297 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

298 if singleFilter: 

299 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

300 return oid 

301 

302 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

303 """The number of bits used up for patch ID bits""" 

304 return 64 - HscMapper._nbit_id 

305 

306 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

307 return self._computeCoaddExposureId(dataId, True) 

308 

309 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

310 """The number of bits used up for patch ID bits""" 

311 return 64 - HscMapper._nbit_id 

312 

313 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

314 return self._computeCoaddExposureId(dataId, False) 

315 

316 # The following allow grabbing a 'psf' from the butler directly, without 

317 # having to get it from a calexp 

318 def map_psf(self, dataId, write=False): 

319 if write: 

320 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

321 copyId = dataId.copy() 

322 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

323 return self.map_calexp_sub(copyId) 

324 

325 def std_psf(self, calexp, dataId): 

326 return calexp.getPsf() 

327 

328 @classmethod 

329 def getCameraName(cls): 

330 return "hsc"