Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2import warnings 

3 

4import lsst.log 

5from lsst.obs.base import CameraMapper 

6from lsst.daf.persistence import ButlerLocation, Policy 

7import lsst.afw.image as afwImage 

8import lsst.afw.math as afwMath 

9import lsst.afw.geom as afwGeom 

10import lsst.geom as geom 

11from lsst.ip.isr import Linearizer 

12import lsst.pex.exceptions 

13from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

14from .hscPupil import HscPupilFactory 

15from .hscFilters import HSC_FILTER_DEFINITIONS 

16 

17 

18class HscMapper(CameraMapper): 

19 """Provides abstract-physical mapping for HSC data""" 

20 packageName = "obs_subaru" 

21 

22 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

23 

24 PupilFactoryClass = HscPupilFactory 

25 

26 # Use the full instrument class name to prevent import errors 

27 # between hsc/ and subaru/ packages. 

28 _gen3instrument = "lsst.obs.subaru.HyperSuprimeCam" 

29 _cameraCache = None # Camera object, cached to speed up instantiation time 

30 

31 @classmethod 

32 def addFilters(cls): 

33 HSC_FILTER_DEFINITIONS.defineFilters() 

34 

35 def __init__(self, **kwargs): 

36 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

37 policy = Policy(policyFile) 

38 if not kwargs.get('root', None): 

39 try: 

40 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

41 except Exception: 

42 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

43 if not kwargs.get('calibRoot', None): 

44 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

45 if "repositoryCfg" in kwargs: 

46 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

47 hasattr(cfg, "root")] 

48 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

49 for calibRoot in calibSearch: 

50 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

51 kwargs['calibRoot'] = calibRoot 

52 break 

53 if not kwargs.get('calibRoot', None): 

54 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

55 

56 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

57 

58 # Ensure each dataset type of interest knows about the full range of 

59 # keys available from the registry 

60 keys = {'field': str, 

61 'visit': int, 

62 'filter': str, 

63 'ccd': int, 

64 'dateObs': str, 

65 'taiObs': str, 

66 'expTime': float, 

67 'pointing': int, 

68 } 

69 for name in ("raw", 

70 # processCcd outputs 

71 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

72 "srcMatch", 

73 # mosaic outputs 

74 "wcs", "fcr", 

75 # processCcd QA 

76 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

77 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

78 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

79 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

80 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

81 # forcedPhot outputs 

82 "forced_src", 

83 ): 

84 self.mappings[name].keyDict.update(keys) 

85 

86 self.addFilters() 

87 

88 self.filters = {} 

89 with warnings.catch_warnings(): 

90 # surpress Filter warnings; we already know this is deprecated 

91 warnings.simplefilter('ignore', category=FutureWarning) 

92 for filt in HSC_FILTER_DEFINITIONS: 

93 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() 

94 self.defaultFilterName = "UNRECOGNISED" 

95 

96 # 

97 # The number of bits allocated for fields in object IDs, appropriate 

98 # for the default-configured Rings skymap. 

99 # 

100 # This shouldn't be the mapper's job at all; see #2797. 

101 

102 HscMapper._nbit_tract = 16 

103 HscMapper._nbit_patch = 5 

104 HscMapper._nbit_filter = 6 

105 

106 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

107 

108 with warnings.catch_warnings(): 

109 # surpress Filter warnings; we already know this is deprecated 

110 warnings.simplefilter('ignore', category=FutureWarning) 

111 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: 

112 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

113 HscMapper._nbit_filter) 

114 

115 def _makeCamera(self, *args, **kwargs): 

116 """Make the camera object 

117 

118 This implementation layers a cache over the parent class' 

119 implementation. Caching the camera improves the instantiation 

120 time for the HscMapper because parsing the camera's Config 

121 involves a lot of 'stat' calls (through the tracebacks). 

122 """ 

123 if not self._cameraCache: 

124 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

125 return self._cameraCache 

126 

127 @classmethod 

128 def clearCache(cls): 

129 """Clear the camera cache 

130 

131 This is principally intended to help memory leak tests pass. 

132 """ 

133 cls._cameraCache = None 

134 

135 def map(self, datasetType, dataId, write=False): 

136 """Need to strip 'flags' argument from map 

137 

138 We want the 'flags' argument passed to the butler to work (it's 

139 used to change how the reading/writing is done), but want it 

140 removed from the mapper (because it doesn't correspond to a 

141 registry column). 

142 """ 

143 copyId = dataId.copy() 

144 copyId.pop("flags", None) 

145 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

146 

147 if 'flags' in dataId: 

148 location.getAdditionalData().set('flags', dataId['flags']) 

149 

150 return location 

151 

152 @staticmethod 

153 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

154 """Flip the chip left/right or top/bottom. Process either/and the 

155 pixels and wcs 

156 

157 Most chips are flipped L/R, but the rotated ones (100..103) are 

158 flipped T/B. 

159 """ 

160 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

161 if exp: 

162 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

163 if wcs: 

164 ampDimensions = exp.getDimensions() if dims is None else dims 

165 ampCenter = geom.Point2D(ampDimensions/2.0) 

166 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

167 

168 return exp, wcs 

169 

170 def std_raw_md(self, md, dataId): 

171 """We need to flip the WCS defined by the metadata in case anyone ever 

172 constructs a Wcs from it. 

173 """ 

174 wcs = afwGeom.makeSkyWcs(md) 

175 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

176 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

177 # NOTE: we don't know where the 0.992 magic constant came from. 

178 # It was copied over from hscSimMapper. 

179 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

180 crval=wcs.getSkyOrigin(), 

181 cdMatrix=wcs.getCdMatrix()*0.992) 

182 wcsMd = wcsR.getFitsMetadata() 

183 

184 for k in wcsMd.names(): 

185 md.set(k, wcsMd.getScalar(k)) 

186 

187 return md 

188 

189 def _createSkyWcsFromMetadata(self, exposure): 

190 # Overridden to flip chips as necessary to get sensible SkyWcs. 

191 metadata = exposure.getMetadata() 

192 try: 

193 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

194 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

195 exposure.setWcs(wcs) 

196 except lsst.pex.exceptions.TypeError as e: 

197 # See DM-14372 for why this is debug and not warn (e.g. calib 

198 # files without wcs metadata). 

199 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

200 " %s", e.args[0]) 

201 

202 # ensure any WCS values stripped from the metadata are removed in 

203 # the exposure 

204 exposure.setMetadata(metadata) 

205 

206 def std_dark(self, item, dataId): 

207 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

208 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

209 exposure.getInfo().setVisitInfo(visitInfo) 

210 return exposure 

211 

212 def _extractDetectorName(self, dataId): 

213 return int("%(ccd)d" % dataId) 

214 

215 def _computeCcdExposureId(self, dataId): 

216 """Compute the 64-bit (long) identifier for a CCD exposure. 

217 

218 @param dataId (dict) Data identifier with visit, ccd 

219 """ 

220 pathId = self._transformId(dataId) 

221 visit = pathId['visit'] 

222 ccd = pathId['ccd'] 

223 return visit*200 + ccd 

224 

225 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

226 return self._computeCcdExposureId(dataId) 

227 

228 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

229 """How many bits are required for the maximum exposure ID""" 

230 return 32 # just a guess, but this leaves plenty of space for sources 

231 

232 def map_linearizer(self, dataId, write=False): 

233 """Map a linearizer.""" 

234 actualId = self._transformId(dataId) 

235 return ButlerLocation( 

236 pythonType="lsst.ip.isr.LinearizeSquared", 

237 cppType="Config", 

238 storageName="PickleStorage", 

239 locationList="ignored", 

240 dataId=actualId, 

241 mapper=self, 

242 storage=self.rootStorage) 

243 

244 def map_crosstalk(self, dataId, write=False): 

245 """Fake the mapping for crosstalk. 

246 

247 Crosstalk is constructed from config parameters, but we need 

248 Gen2 butlers to be able to respond to requests for it. 

249 Returning None provides a response that can be used with the 

250 config parameters to generate the appropriate calibration. 

251 """ 

252 return None 

253 

254 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

255 """Return a linearizer for the given detector. 

256 

257 On each call, a fresh instance of `Linearizer` is returned; the caller 

258 is responsible for initializing it appropriately for the detector. 

259 

260 Parameters 

261 ---------- 

262 datasetType : `str`` 

263 The dataset type. 

264 pythonType : `str` or `type` 

265 Type of python object. 

266 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

267 Struct-like class that holds information needed to persist and 

268 retrieve an object using the LSST Persistence Framework. 

269 dataId : `dict` 

270 dataId passed to map location. 

271 

272 Returns 

273 ------- 

274 Linearizer : `lsst.ip.isr.Linearizer` 

275 Linearizer object for the given detector. 

276 

277 Notes 

278 ----- 

279 Linearizers are not saved to persistent storage; rather, they are 

280 managed entirely in memory. On each call, this function will return a 

281 new instance of `Linearizer`, which must be managed (including setting 

282 it up for use with a particular detector) by the caller. Calling 

283 `bypass_linearizer` twice for the same detector will return 

284 _different_ instances of `Linearizer`, which share no state. 

285 """ 

286 return Linearizer(detectorId=dataId.get('ccd', None)) 

287 

288 def _computeCoaddExposureId(self, dataId, singleFilter): 

289 """Compute the 64-bit (long) identifier for a coadd. 

290 

291 @param dataId (dict) Data identifier with tract and patch. 

292 @param singleFilter (bool) True means the desired ID is for a single- 

293 filter coadd, in which case dataId 

294 must contain filter. 

295 """ 

296 

297 tract = int(dataId['tract']) 

298 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

299 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

300 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

301 for p in (patchX, patchY): 

302 if p < 0 or p >= 2**HscMapper._nbit_patch: 

303 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

304 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

305 if singleFilter: 

306 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

307 return oid 

308 

309 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

310 """The number of bits used up for patch ID bits""" 

311 return 64 - HscMapper._nbit_id 

312 

313 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

314 return self._computeCoaddExposureId(dataId, True) 

315 

316 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

317 """The number of bits used up for patch ID bits""" 

318 return 64 - HscMapper._nbit_id 

319 

320 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

321 return self._computeCoaddExposureId(dataId, False) 

322 

323 # The following allow grabbing a 'psf' from the butler directly, without 

324 # having to get it from a calexp 

325 def map_psf(self, dataId, write=False): 

326 if write: 

327 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

328 copyId = dataId.copy() 

329 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

330 return self.map_calexp_sub(copyId) 

331 

332 def std_psf(self, calexp, dataId): 

333 return calexp.getPsf() 

334 

335 @classmethod 

336 def getCameraName(cls): 

337 return "hsc"