Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2 

3import lsst.log 

4from lsst.obs.base import CameraMapper 

5from lsst.daf.persistence import ButlerLocation, Policy 

6import lsst.afw.image as afwImage 

7import lsst.afw.math as afwMath 

8import lsst.afw.geom as afwGeom 

9import lsst.geom as geom 

10from lsst.ip.isr import Linearizer 

11import lsst.pex.exceptions 

12from .makeHscRawVisitInfo import MakeHscRawVisitInfo 

13from .hscPupil import HscPupilFactory 

14from .hscFilters import HSC_FILTER_DEFINITIONS 

15 

16 

17class HscMapper(CameraMapper): 

18 """Provides abstract-physical mapping for HSC data""" 

19 packageName = "obs_subaru" 

20 

21 MakeRawVisitInfoClass = MakeHscRawVisitInfo 

22 

23 PupilFactoryClass = HscPupilFactory 

24 

25 _cameraCache = None # Camera object, cached to speed up instantiation time 

26 

27 @classmethod 

28 def addFilters(cls): 

29 HSC_FILTER_DEFINITIONS.defineFilters() 

30 

31 def __init__(self, **kwargs): 

32 policyFile = Policy.defaultPolicyFile("obs_subaru", "HscMapper.yaml", "policy") 

33 policy = Policy(policyFile) 

34 if not kwargs.get('root', None): 

35 try: 

36 kwargs['root'] = os.path.join(os.environ.get('SUPRIME_DATA_DIR'), 'HSC') 

37 except Exception: 

38 raise RuntimeError("Either $SUPRIME_DATA_DIR or root= must be specified") 

39 if not kwargs.get('calibRoot', None): 

40 calibSearch = [os.path.join(kwargs['root'], 'CALIB')] 

41 if "repositoryCfg" in kwargs: 

42 calibSearch += [os.path.join(cfg.root, 'CALIB') for cfg in kwargs["repositoryCfg"].parents if 

43 hasattr(cfg, "root")] 

44 calibSearch += [cfg.root for cfg in kwargs["repositoryCfg"].parents if hasattr(cfg, "root")] 

45 for calibRoot in calibSearch: 

46 if os.path.exists(os.path.join(calibRoot, "calibRegistry.sqlite3")): 

47 kwargs['calibRoot'] = calibRoot 

48 break 

49 if not kwargs.get('calibRoot', None): 

50 lsst.log.Log.getLogger("HscMapper").warn("Unable to find calib root directory") 

51 

52 super(HscMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

53 

54 self._linearize = Linearizer() 

55 

56 # Ensure each dataset type of interest knows about the full range of keys available from the registry 

57 keys = {'field': str, 

58 'visit': int, 

59 'filter': str, 

60 'ccd': int, 

61 'dateObs': str, 

62 'taiObs': str, 

63 'expTime': float, 

64 'pointing': int, 

65 } 

66 for name in ("raw", 

67 # processCcd outputs 

68 "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "icMatch", 

69 "srcMatch", 

70 # mosaic outputs 

71 "wcs", "fcr", 

72 # processCcd QA 

73 "ossThumb", "flattenedThumb", "calexpThumb", "plotMagHist", "plotSeeingRough", 

74 "plotSeeingRobust", "plotSeeingMap", "plotEllipseMap", "plotEllipticityMap", 

75 "plotFwhmGrid", "plotEllipseGrid", "plotEllipticityGrid", "plotPsfSrcGrid", 

76 "plotPsfModelGrid", "fitsFwhmGrid", "fitsEllipticityGrid", "fitsEllPaGrid", 

77 "fitsPsfSrcGrid", "fitsPsfModelGrid", "tableSeeingMap", "tableSeeingGrid", 

78 # forcedPhot outputs 

79 "forced_src", 

80 ): 

81 self.mappings[name].keyDict.update(keys) 

82 

83 self.addFilters() 

84 

85 self.filters = {} 

86 for filt in HSC_FILTER_DEFINITIONS: 

87 self.filters[filt.physical_filter] = afwImage.Filter(filt.physical_filter).getCanonicalName() 

88 self.defaultFilterName = "UNRECOGNISED" 

89 

90 # 

91 # The number of bits allocated for fields in object IDs, appropriate for 

92 # the default-configured Rings skymap. 

93 # 

94 # This shouldn't be the mapper's job at all; see #2797. 

95 

96 HscMapper._nbit_tract = 16 

97 HscMapper._nbit_patch = 5 

98 HscMapper._nbit_filter = 6 

99 

100 HscMapper._nbit_id = 64 - (HscMapper._nbit_tract + 2*HscMapper._nbit_patch + HscMapper._nbit_filter) 

101 

102 if len(afwImage.Filter.getNames()) >= 2**HscMapper._nbit_filter: 

103 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

104 HscMapper._nbit_filter) 

105 

106 def _makeCamera(self, *args, **kwargs): 

107 """Make the camera object 

108 

109 This implementation layers a cache over the parent class' 

110 implementation. Caching the camera improves the instantiation 

111 time for the HscMapper because parsing the camera's Config 

112 involves a lot of 'stat' calls (through the tracebacks). 

113 """ 

114 if not self._cameraCache: 

115 self._cameraCache = CameraMapper._makeCamera(self, *args, **kwargs) 

116 return self._cameraCache 

117 

118 @classmethod 

119 def clearCache(cls): 

120 """Clear the camera cache 

121 

122 This is principally intended to help memory leak tests pass. 

123 """ 

124 cls._cameraCache = None 

125 

126 def map(self, datasetType, dataId, write=False): 

127 """Need to strip 'flags' argument from map 

128 

129 We want the 'flags' argument passed to the butler to work (it's 

130 used to change how the reading/writing is done), but want it 

131 removed from the mapper (because it doesn't correspond to a 

132 registry column). 

133 """ 

134 copyId = dataId.copy() 

135 copyId.pop("flags", None) 

136 location = super(HscMapper, self).map(datasetType, copyId, write=write) 

137 

138 if 'flags' in dataId: 

139 location.getAdditionalData().set('flags', dataId['flags']) 

140 

141 return location 

142 

143 @staticmethod 

144 def _flipChipsLR(exp, wcs, detectorId, dims=None): 

145 """Flip the chip left/right or top/bottom. Process either/and the pixels and wcs 

146 

147 Most chips are flipped L/R, but the rotated ones (100..103) are flipped T/B 

148 """ 

149 flipLR, flipTB = (False, True) if detectorId in (100, 101, 102, 103) else (True, False) 

150 if exp: 

151 exp.setMaskedImage(afwMath.flipImage(exp.getMaskedImage(), flipLR, flipTB)) 

152 if wcs: 

153 ampDimensions = exp.getDimensions() if dims is None else dims 

154 ampCenter = geom.Point2D(ampDimensions/2.0) 

155 wcs = afwGeom.makeFlippedWcs(wcs, flipLR, flipTB, ampCenter) 

156 

157 return exp, wcs 

158 

159 def std_raw_md(self, md, dataId): 

160 """We need to flip the WCS defined by the metadata in case anyone ever 

161 constructs a Wcs from it. 

162 """ 

163 wcs = afwGeom.makeSkyWcs(md) 

164 wcs = self._flipChipsLR(None, wcs, dataId['ccd'], 

165 dims=afwImage.bboxFromMetadata(md).getDimensions())[1] 

166 # NOTE: we don't know where the 0.992 magic constant came from. It was copied over from hscSimMapper. 

167 wcsR = afwGeom.makeSkyWcs(crpix=wcs.getPixelOrigin(), 

168 crval=wcs.getSkyOrigin(), 

169 cdMatrix=wcs.getCdMatrix()*0.992) 

170 wcsMd = wcsR.getFitsMetadata() 

171 

172 for k in wcsMd.names(): 

173 md.set(k, wcsMd.getScalar(k)) 

174 

175 return md 

176 

177 def _createSkyWcsFromMetadata(self, exposure): 

178 # Overridden to flip chips as necessary to get sensible SkyWcs. 

179 metadata = exposure.getMetadata() 

180 try: 

181 wcs = afwGeom.makeSkyWcs(metadata, strip=True) 

182 exposure, wcs = self._flipChipsLR(exposure, wcs, exposure.getDetector().getId()) 

183 exposure.setWcs(wcs) 

184 except lsst.pex.exceptions.TypeError as e: 

185 # See DM-14372 for why this is debug and not warn (e.g. calib files without wcs metadata). 

186 self.log.debug("wcs set to None; missing information found in metadata to create a valid wcs:" 

187 " %s", e.args[0]) 

188 

189 # ensure any WCS values stripped from the metadata are removed in the exposure 

190 exposure.setMetadata(metadata) 

191 

192 def std_dark(self, item, dataId): 

193 exposure = self._standardizeExposure(self.calibrations['dark'], item, dataId, trimmed=False) 

194 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0) 

195 exposure.getInfo().setVisitInfo(visitInfo) 

196 return exposure 

197 

198 def _extractDetectorName(self, dataId): 

199 return int("%(ccd)d" % dataId) 

200 

201 def _computeCcdExposureId(self, dataId): 

202 """Compute the 64-bit (long) identifier for a CCD exposure. 

203 

204 @param dataId (dict) Data identifier with visit, ccd 

205 """ 

206 pathId = self._transformId(dataId) 

207 visit = pathId['visit'] 

208 ccd = pathId['ccd'] 

209 return visit*200 + ccd 

210 

211 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

212 return self._computeCcdExposureId(dataId) 

213 

214 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

215 """How many bits are required for the maximum exposure ID""" 

216 return 32 # just a guess, but this leaves plenty of space for sources 

217 

218 def map_linearizer(self, dataId, write=False): 

219 """Map a linearizer.""" 

220 if self._linearize is None: 

221 raise RuntimeError("No linearizer available.") 

222 actualId = self._transformId(dataId) 

223 return ButlerLocation( 

224 pythonType="lsst.ip.isr.LinearizeSquared", 

225 cppType="Config", 

226 storageName="PickleStorage", 

227 locationList="ignored", 

228 dataId=actualId, 

229 mapper=self, 

230 storage=self.rootStorage) 

231 

232 def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId): 

233 """Return the linearizer. 

234 """ 

235 if self._linearize is None: 

236 raise RuntimeError("No linearizer available.") 

237 return self._linearize 

238 

239 def _computeCoaddExposureId(self, dataId, singleFilter): 

240 """Compute the 64-bit (long) identifier for a coadd. 

241 

242 @param dataId (dict) Data identifier with tract and patch. 

243 @param singleFilter (bool) True means the desired ID is for a single- 

244 filter coadd, in which case dataId 

245 must contain filter. 

246 """ 

247 

248 tract = int(dataId['tract']) 

249 if tract < 0 or tract >= 2**HscMapper._nbit_tract: 

250 raise RuntimeError('tract not in range [0,%d)' % (2**HscMapper._nbit_tract)) 

251 patchX, patchY = [int(patch) for patch in dataId['patch'].split(',')] 

252 for p in (patchX, patchY): 

253 if p < 0 or p >= 2**HscMapper._nbit_patch: 

254 raise RuntimeError('patch component not in range [0, %d)' % 2**HscMapper._nbit_patch) 

255 oid = (((tract << HscMapper._nbit_patch) + patchX) << HscMapper._nbit_patch) + patchY 

256 if singleFilter: 

257 return (oid << HscMapper._nbit_filter) + afwImage.Filter(dataId['filter']).getId() 

258 return oid 

259 

260 def bypass_deepCoaddId_bits(self, *args, **kwargs): 

261 """The number of bits used up for patch ID bits""" 

262 return 64 - HscMapper._nbit_id 

263 

264 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

265 return self._computeCoaddExposureId(dataId, True) 

266 

267 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

268 """The number of bits used up for patch ID bits""" 

269 return 64 - HscMapper._nbit_id 

270 

271 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

272 return self._computeCoaddExposureId(dataId, False) 

273 

274 # The following allow grabbing a 'psf' from the butler directly, without having to get it from a calexp 

275 def map_psf(self, dataId, write=False): 

276 if write: 

277 raise RuntimeError("Writing a psf directly is no longer permitted: write as part of a calexp") 

278 copyId = dataId.copy() 

279 copyId['bbox'] = geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1, 1)) 

280 return self.map_calexp_sub(copyId) 

281 

282 def std_psf(self, calexp, dataId): 

283 return calexp.getPsf() 

284 

285 @classmethod 

286 def getCameraName(cls): 

287 return "hsc"