Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2 

3import lsst.log 

4from lsst.obs.base import CameraMapper 

5from lsst.daf.persistence import ButlerLocation, Policy 

6import lsst.afw.image as afwImage 

7import lsst.afw.math as afwMath 

8import lsst.afw.geom as afwGeom 

9import lsst.geom as geom 

10from lsst.ip.isr import Linearizer 

11import lsst.pex.exceptions 

12from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

13from .hscPupil import HscPupilFactory 

14from .hscFilters import HSC_FILTER_DEFINITIONS 

15 

16 

17class HscMapper(CameraMapper): 

18 """Provides abstract-physical mapping for HSC data""" 

19 packageName = "obs_subaru" 

20 

21 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

22 

23 PupilFactoryClass = HscPupilFactory 

24 

25 _cameraCache = None # Camera object, cached to speed up instantiation time 

26 

27 @classmethod 

28 def addFilters(cls): 

29 HSC_FILTER_DEFINITIONS.defineFilters() 

30 

31 def __init__(self, **kwargs): 

32 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

33 policy = Policy(policyFile) 

34 if not kwargs.get('root', None): 

35 try: 

36 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

37 except Exception: 

38 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

39 if not kwargs.get('calibRoot', None): 

40 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

41 if "repositoryCfg" in kwargs: 

42 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

43 hasattr(cfg, "root")] 

44 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

45 for calibRoot in calibSearch: 

46 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

47 kwargs['calibRoot'] = calibRoot 

48 break 

49 if not kwargs.get('calibRoot', None): 

50 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

51 

52 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

53 

54 # Ensure each dataset type of interest knows about the full range of keys available from the registry 

55 keys = {'field': str, 

56 'visit': int, 

57 'filter': str, 

58 'ccd': int, 

59 'dateObs': str, 

60 'taiObs': str, 

61 'expTime': float, 

62 'pointing': int, 

63 } 

64 for name in ("raw", 

65 # processCcd outputs 

66 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

67 "srcMatch", 

68 # mosaic outputs 

69 "wcs", "fcr", 

70 # processCcd QA 

71 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

72 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

73 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

74 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

75 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

76 # forcedPhot outputs 

77 "forced_src", 

78 ): 

79 self.mappings[name].keyDict.update(keys) 

80 

81 self.addFilters() 

82 

83 self.filters = {} 

84 for filt in HSC_FILTER_DEFINITIONS: 

85 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() 

86 self.defaultFilterName = "UNRECOGNISED" 

87 

88 # 

89 # The number of bits allocated for fields in object IDs, appropriate for 

90 # the default-configured Rings skymap. 

91 # 

92 # This shouldn't be the mapper's job at all; see #2797. 

93 

94 HscMapper._nbit_tract = 16 

95 HscMapper._nbit_patch = 5 

96 HscMapper._nbit_filter = 6 

97 

98 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

99 

100 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: 

101 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

102 HscMapper._nbit_filter) 

103 

104 def _makeCamera(self, *args, **kwargs): 

105 """Make the camera object 

106 

107 This implementation layers a cache over the parent class' 

108 implementation. Caching the camera improves the instantiation 

109 time for the HscMapper because parsing the camera's Config 

110 involves a lot of 'stat' calls (through the tracebacks). 

111 """ 

112 if not self._cameraCache: 

113 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

114 return self._cameraCache 

115 

116 @classmethod 

117 def clearCache(cls): 

118 """Clear the camera cache 

119 

120 This is principally intended to help memory leak tests pass. 

121 """ 

122 cls._cameraCache = None 

123 

124 def map(self, datasetType, dataId, write=False): 

125 """Need to strip 'flags' argument from map 

126 

127 We want the 'flags' argument passed to the butler to work (it's 

128 used to change how the reading/writing is done), but want it 

129 removed from the mapper (because it doesn't correspond to a 

130 registry column). 

131 """ 

132 copyId = dataId.copy() 

133 copyId.pop("flags", None) 

134 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

135 

136 if 'flags' in dataId: 

137 location.getAdditionalData().set('flags', dataId['flags']) 

138 

139 return location 

140 

141 @staticmethod 

142 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

143 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs 

144 

145 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B 

146 """ 

147 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

148 if exp: 

149 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

150 if wcs: 

151 ampDimensions = exp.getDimensions() if dims is None else dims 

152 ampCenter = geom.Point2D(ampDimensions/2.0) 

153 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

154 

155 return exp, wcs 

156 

157 def std_raw_md(self, md, dataId): 

158 """We need to flip the WCS defined by the metadata in case anyone ever 

159 constructs a Wcs from it. 

160 """ 

161 wcs = afwGeom.makeSkyWcs(md) 

162 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

163 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

164 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper. 

165 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

166 crval=wcs.getSkyOrigin(), 

167 cdMatrix=wcs.getCdMatrix()*0.992) 

168 wcsMd = wcsR.getFitsMetadata() 

169 

170 for k in wcsMd.names(): 

171 md.set(k, wcsMd.getScalar(k)) 

172 

173 return md 

174 

175 def _createSkyWcsFromMetadata(self, exposure): 

176 # Overridden to flip chips as necessary to get sensible SkyWcs. 

177 metadata = exposure.getMetadata() 

178 try: 

179 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

180 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

181 exposure.setWcs(wcs) 

182 except lsst.pex.exceptions.TypeError as e: 

183 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata). 

184 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

185 " %s", e.args[0]) 

186 

187 # ensure any WCS values stripped from the metadata are removed in the exposure 

188 exposure.setMetadata(metadata) 

189 

190 def std_dark(self, item, dataId): 

191 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

192 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

193 exposure.getInfo().setVisitInfo(visitInfo) 

194 return exposure 

195 

196 def _extractDetectorName(self, dataId): 

197 return int("%(ccd)d" % dataId) 

198 

199 def _computeCcdExposureId(self, dataId): 

200 """Compute the 64-bit (long) identifier for a CCD exposure. 

201 

202 @param dataId (dict) Data identifier with visit, ccd 

203 """ 

204 pathId = self._transformId(dataId) 

205 visit = pathId['visit'] 

206 ccd = pathId['ccd'] 

207 return visit*200 + ccd 

208 

209 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

210 return self._computeCcdExposureId(dataId) 

211 

212 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

213 """How many bits are required for the maximum exposure ID""" 

214 return 32 # just a guess, but this leaves plenty of space for sources 

215 

216 def map_linearizer(self, dataId, write=False): 

217 """Map a linearizer.""" 

218 actualId = self._transformId(dataId) 

219 return ButlerLocation( 

220 pythonType="lsst.ip.isr.LinearizeSquared", 

221 cppType="Config", 

222 storageName="PickleStorage", 

223 locationList="ignored", 

224 dataId=actualId, 

225 mapper=self, 

226 storage=self.rootStorage) 

227 

228 def map_crosstalk(self, dataId, write=False): 

229 """Fake the mapping for crosstalk. 

230 

231 Crosstalk is constructed from config parameters, but we need 

232 Gen2 butlers to be able to respond to requests for it. 

233 Returning None provides a response that can be used with the 

234 config parameters to generate the appropriate calibration. 

235 """ 

236 return None 

237 

238 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

239 """Return a linearizer for the given detector. 

240 

241 On each call, a fresh instance of `Linearizer` is returned; the caller is responsible for 

242 initializing it appropriately for the detector. 

243 

244 Parameters 

245 ---------- 

246 datasetType : `str`` 

247 The dataset type. 

248 pythonType : `str` or `type` 

249 Type of python object. 

250 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

251 Struct-like class that holds information needed to persist and retrieve an object using 

252 the LSST Persistence Framework. 

253 dataId : `dict` 

254 dataId passed to map location. 

255 

256 Returns 

257 ------- 

258 Linearizer : `lsst.ip.isr.Linearizer` 

259 Linearizer object for the given detector. 

260 

261 Notes 

262 ----- 

263 Linearizers are not saved to persistent storage; rather, they are managed entirely in memory. 

264 On each call, this function will return a new instance of `Linearizer`, which must be managed 

265 (including setting it up for use with a particular detector) by the caller. Calling 

266 `bypass_linearizer` twice for the same detector will return _different_ instances of `Linearizer`, 

267 which share no state. 

268 """ 

269 return Linearizer() 

270 

271 def _computeCoaddExposureId(self, dataId, singleFilter): 

272 """Compute the 64-bit (long) identifier for a coadd. 

273 

274 @param dataId (dict) Data identifier with tract and patch. 

275 @param singleFilter (bool) True means the desired ID is for a single- 

276 filter coadd, in which case dataId 

277 must contain filter. 

278 """ 

279 

280 tract = int(dataId['tract']) 

281 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

282 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

283 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

284 for p in (patchX, patchY): 

285 if p < 0 or p >= 2**HscMapper._nbit_patch: 

286 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

287 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

288 if singleFilter: 

289 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

290 return oid 

291 

292 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

293 """The number of bits used up for patch ID bits""" 

294 return 64 - HscMapper._nbit_id 

295 

296 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

297 return self._computeCoaddExposureId(dataId, True) 

298 

299 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

300 """The number of bits used up for patch ID bits""" 

301 return 64 - HscMapper._nbit_id 

302 

303 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

304 return self._computeCoaddExposureId(dataId, False) 

305 

306 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp 

307 def map_psf(self, dataId, write=False): 

308 if write: 

309 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

310 copyId = dataId.copy() 

311 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

312 return self.map_calexp_sub(copyId) 

313 

314 def std_psf(self, calexp, dataId): 

315 return calexp.getPsf() 

316 

317 @classmethod 

318 def getCameraName(cls): 

319 return "hsc"