Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2 

3import lsst.log 

4from lsst.obs.base import CameraMapper 

5from lsst.daf.persistence import ButlerLocation, Policy 

6import lsst.afw.image as afwImage 

7import lsst.afw.math as afwMath 

8import lsst.afw.geom as afwGeom 

9import lsst.geom as geom 

10from lsst.ip.isr import Linearizer 

11import lsst.pex.exceptions 

12from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

13from .hscPupil import HscPupilFactory 

14from .hscFilters import HSC_FILTER_DEFINITIONS 

15 

16 

17class HscMapper(CameraMapper): 

18 """Provides abstract-physical mapping for HSC data""" 

19 packageName = "obs_subaru" 

20 

21 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

22 

23 PupilFactoryClass = HscPupilFactory 

24 

25 # Use the full instrument class name to prevent import errors 

26 # between hsc/ and subaru/ packages. 

27 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam" 

28 _cameraCache = None # Camera object, cached to speed up instantiation time 

29 

30 @classmethod 

31 def addFilters(cls): 

32 HSC_FILTER_DEFINITIONS.defineFilters() 

33 

34 def __init__(self, **kwargs): 

35 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

36 policy = Policy(policyFile) 

37 if not kwargs.get('root', None): 

38 try: 

39 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

40 except Exception: 

41 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

42 if not kwargs.get('calibRoot', None): 

43 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

44 if "repositoryCfg" in kwargs: 

45 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

46 hasattr(cfg, "root")] 

47 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

48 for calibRoot in calibSearch: 

49 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

50 kwargs['calibRoot'] = calibRoot 

51 break 

52 if not kwargs.get('calibRoot', None): 

53 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

54 

55 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

56 

57 # Ensure each dataset type of interest knows about the full range of keys available from the registry 

58 keys = {'field': str, 

59 'visit': int, 

60 'filter': str, 

61 'ccd': int, 

62 'dateObs': str, 

63 'taiObs': str, 

64 'expTime': float, 

65 'pointing': int, 

66 } 

67 for name in ("raw", 

68 # processCcd outputs 

69 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

70 "srcMatch", 

71 # mosaic outputs 

72 "wcs", "fcr", 

73 # processCcd QA 

74 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

75 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

76 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

77 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

78 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

79 # forcedPhot outputs 

80 "forced_src", 

81 ): 

82 self.mappings[name].keyDict.update(keys) 

83 

84 self.addFilters() 

85 

86 self.filters = {} 

87 for filt in HSC_FILTER_DEFINITIONS: 

88 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() 

89 self.defaultFilterName = "UNRECOGNISED" 

90 

91 # 

92 # The number of bits allocated for fields in object IDs, appropriate for 

93 # the default-configured Rings skymap. 

94 # 

95 # This shouldn't be the mapper's job at all; see #2797. 

96 

97 HscMapper._nbit_tract = 16 

98 HscMapper._nbit_patch = 5 

99 HscMapper._nbit_filter = 6 

100 

101 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

102 

103 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: 

104 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

105 HscMapper._nbit_filter) 

106 

107 def _makeCamera(self, *args, **kwargs): 

108 """Make the camera object 

109 

110 This implementation layers a cache over the parent class' 

111 implementation. Caching the camera improves the instantiation 

112 time for the HscMapper because parsing the camera's Config 

113 involves a lot of 'stat' calls (through the tracebacks). 

114 """ 

115 if not self._cameraCache: 

116 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

117 return self._cameraCache 

118 

119 @classmethod 

120 def clearCache(cls): 

121 """Clear the camera cache 

122 

123 This is principally intended to help memory leak tests pass. 

124 """ 

125 cls._cameraCache = None 

126 

127 def map(self, datasetType, dataId, write=False): 

128 """Need to strip 'flags' argument from map 

129 

130 We want the 'flags' argument passed to the butler to work (it's 

131 used to change how the reading/writing is done), but want it 

132 removed from the mapper (because it doesn't correspond to a 

133 registry column). 

134 """ 

135 copyId = dataId.copy() 

136 copyId.pop("flags", None) 

137 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

138 

139 if 'flags' in dataId: 

140 location.getAdditionalData().set('flags', dataId['flags']) 

141 

142 return location 

143 

144 @staticmethod 

145 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

146 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs 

147 

148 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B 

149 """ 

150 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

151 if exp: 

152 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

153 if wcs: 

154 ampDimensions = exp.getDimensions() if dims is None else dims 

155 ampCenter = geom.Point2D(ampDimensions/2.0) 

156 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

157 

158 return exp, wcs 

159 

160 def std_raw_md(self, md, dataId): 

161 """We need to flip the WCS defined by the metadata in case anyone ever 

162 constructs a Wcs from it. 

163 """ 

164 wcs = afwGeom.makeSkyWcs(md) 

165 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

166 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

167 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper. 

168 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

169 crval=wcs.getSkyOrigin(), 

170 cdMatrix=wcs.getCdMatrix()*0.992) 

171 wcsMd = wcsR.getFitsMetadata() 

172 

173 for k in wcsMd.names(): 

174 md.set(k, wcsMd.getScalar(k)) 

175 

176 return md 

177 

178 def _createSkyWcsFromMetadata(self, exposure): 

179 # Overridden to flip chips as necessary to get sensible SkyWcs. 

180 metadata = exposure.getMetadata() 

181 try: 

182 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

183 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

184 exposure.setWcs(wcs) 

185 except lsst.pex.exceptions.TypeError as e: 

186 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata). 

187 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

188 " %s", e.args[0]) 

189 

190 # ensure any WCS values stripped from the metadata are removed in the exposure 

191 exposure.setMetadata(metadata) 

192 

193 def std_dark(self, item, dataId): 

194 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

195 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

196 exposure.getInfo().setVisitInfo(visitInfo) 

197 return exposure 

198 

199 def _extractDetectorName(self, dataId): 

200 return int("%(ccd)d" % dataId) 

201 

202 def _computeCcdExposureId(self, dataId): 

203 """Compute the 64-bit (long) identifier for a CCD exposure. 

204 

205 @param dataId (dict) Data identifier with visit, ccd 

206 """ 

207 pathId = self._transformId(dataId) 

208 visit = pathId['visit'] 

209 ccd = pathId['ccd'] 

210 return visit*200 + ccd 

211 

212 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

213 return self._computeCcdExposureId(dataId) 

214 

215 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

216 """How many bits are required for the maximum exposure ID""" 

217 return 32 # just a guess, but this leaves plenty of space for sources 

218 

219 def map_linearizer(self, dataId, write=False): 

220 """Map a linearizer.""" 

221 actualId = self._transformId(dataId) 

222 return ButlerLocation( 

223 pythonType="lsst.ip.isr.LinearizeSquared", 

224 cppType="Config", 

225 storageName="PickleStorage", 

226 locationList="ignored", 

227 dataId=actualId, 

228 mapper=self, 

229 storage=self.rootStorage) 

230 

231 def map_crosstalk(self, dataId, write=False): 

232 """Fake the mapping for crosstalk. 

233 

234 Crosstalk is constructed from config parameters, but we need 

235 Gen2 butlers to be able to respond to requests for it. 

236 Returning None provides a response that can be used with the 

237 config parameters to generate the appropriate calibration. 

238 """ 

239 return None 

240 

241 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

242 """Return a linearizer for the given detector. 

243 

244 On each call, a fresh instance of `Linearizer` is returned; the caller is responsible for 

245 initializing it appropriately for the detector. 

246 

247 Parameters 

248 ---------- 

249 datasetType : `str`` 

250 The dataset type. 

251 pythonType : `str` or `type` 

252 Type of python object. 

253 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

254 Struct-like class that holds information needed to persist and retrieve an object using 

255 the LSST Persistence Framework. 

256 dataId : `dict` 

257 dataId passed to map location. 

258 

259 Returns 

260 ------- 

261 Linearizer : `lsst.ip.isr.Linearizer` 

262 Linearizer object for the given detector. 

263 

264 Notes 

265 ----- 

266 Linearizers are not saved to persistent storage; rather, they are managed entirely in memory. 

267 On each call, this function will return a new instance of `Linearizer`, which must be managed 

268 (including setting it up for use with a particular detector) by the caller. Calling 

269 `bypass_linearizer` twice for the same detector will return _different_ instances of `Linearizer`, 

270 which share no state. 

271 """ 

272 return Linearizer(detectorId=dataId.get('ccd', None)) 

273 

274 def _computeCoaddExposureId(self, dataId, singleFilter): 

275 """Compute the 64-bit (long) identifier for a coadd. 

276 

277 @param dataId (dict) Data identifier with tract and patch. 

278 @param singleFilter (bool) True means the desired ID is for a single- 

279 filter coadd, in which case dataId 

280 must contain filter. 

281 """ 

282 

283 tract = int(dataId['tract']) 

284 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

285 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

286 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

287 for p in (patchX, patchY): 

288 if p < 0 or p >= 2**HscMapper._nbit_patch: 

289 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

290 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

291 if singleFilter: 

292 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

293 return oid 

294 

295 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

296 """The number of bits used up for patch ID bits""" 

297 return 64 - HscMapper._nbit_id 

298 

299 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

300 return self._computeCoaddExposureId(dataId, True) 

301 

302 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

303 """The number of bits used up for patch ID bits""" 

304 return 64 - HscMapper._nbit_id 

305 

306 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

307 return self._computeCoaddExposureId(dataId, False) 

308 

309 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp 

310 def map_psf(self, dataId, write=False): 

311 if write: 

312 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

313 copyId = dataId.copy() 

314 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

315 return self.map_calexp_sub(copyId) 

316 

317 def std_psf(self, calexp, dataId): 

318 return calexp.getPsf() 

319 

320 @classmethod 

321 def getCameraName(cls): 

322 return "hsc"