Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["MegacamMapper"] 

24 

25import os 

26 

27from astropy.io import fits 

28 

29import lsst.geom as geom 

30import lsst.afw.image.utils as afwImageUtils 

31import lsst.meas.algorithms as measAlg 

32import lsst.daf.persistence as dafPersist 

33 

34from lsst.daf.persistence import Policy 

35from lsst.obs.base import CameraMapper, exposureFromImage 

36from .makeMegacamRawVisitInfo import MakeMegacamRawVisitInfo 

37 

38 

39class MegacamMapper(CameraMapper): 

40 """Camera Mapper for CFHT MegaCam.""" 

41 packageName = "obs_cfht" 

42 

43 MakeRawVisitInfoClass = MakeMegacamRawVisitInfo 

44 

45 def __init__(self, **kwargs): 

46 policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") 

47 policy = Policy(policyFile) 

48 repositoryDir = os.path.dirname(policyFile) 

49 super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs) 

50 

51 # Defect registry and root. Defects are stored with the camera and the registry is loaded from the 

52 # camera package, which is on the local filesystem. 

53 self.defectRegistry = None 

54 if 'defects' in policy: 

55 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

56 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

57 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

58 

59 # The "ccd" provided by the user is translated through the registry 

60 # into an extension name for the "raw" template. The template 

61 # therefore doesn't include "ccd", so we need to ensure it's 

62 # explicitly included so the ArgumentParser can recognise and accept 

63 # it. 

64 

65 self.exposures['raw'].keyDict['ccd'] = int 

66 

67 afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") 

68 afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") 

69 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") 

70 afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") 

71 afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") 

72 afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") 

73 afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") 

74 afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") 

75 afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") 

76 afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") 

77 afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") 

78 

79 # define filters? 

80 self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) 

81 

82 # Ensure each dataset type of interest knows about the full range of 

83 # keys available from the registry 

84 keys = {'runId': str, 

85 'object': str, 

86 'visit': int, 

87 'ccd': int, 

88 'extension': int, 

89 'state': str, 

90 'filter': str, 

91 'date': str, 

92 'taiObs': str, 

93 'expTime': float, 

94 } 

95 for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): 

96 self.mappings[name].keyDict.update(keys) 

97 

98 # 

99 # The number of bits allocated for fields in object IDs, appropriate for 

100 # the default-configured Rings skymap. 

101 # 

102 

103 MegacamMapper._nbit_tract = 16 

104 MegacamMapper._nbit_patch = 5 

105 MegacamMapper._nbit_filter = 6 

106 

107 MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch + 

108 MegacamMapper._nbit_filter) 

109 

110 if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter: 

111 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

112 MegacamMapper._nbit_filter) 

113 

114 def map_defects(self, dataId, write=False): 

115 """Map defects dataset. 

116 

117 Returns 

118 ------- 

119 `lsst.daf.butler.ButlerLocation` 

120 Minimal ButlerLocation containing just the locationList field 

121 (just enough information that bypass_defects can use it). 

122 """ 

123 defectFitsPath = self._defectLookup(dataId=dataId) 

124 if defectFitsPath is None: 

125 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

126 

127 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

128 dataId, self, 

129 storage=self.rootStorage) 

130 

131 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

132 """Return a defect based on the butler location returned by 

133 map_defects. 

134 

135 Parameters 

136 ---------- 

137 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

138 A ButlerLocation with locationList = path to defects FITS file. 

139 dataId : `dict` 

140 The usual data ID; "ccd" must be set. 

141 

142 Notes 

143 ----- 

144 The name "bypass_XXX" means the butler makes no attempt to convert 

145 the ButlerLocation into an object, which is what we want for now, 

146 since that conversion is a bit tricky. 

147 """ 

148 (ccdKey, ccdSerial) = self._getCcdKeyVal(dataId) 

149 defectsFitsPath = butlerLocation.locationList[0] 

150 with fits.open(defectsFitsPath) as hduList: 

151 for hdu in hduList[1:]: 

152 if str(hdu.header["SERIAL"]) != ccdSerial: 

153 continue 

154 

155 defectList = measAlg.Defects() 

156 for data in hdu.data: 

157 bbox = geom.Box2I( 

158 geom.Point2I(int(data['x0']), int(data['y0'])), 

159 geom.Extent2I(int(data['width']), int(data['height'])), 

160 ) 

161 defectList.append(bbox) 

162 return defectList 

163 

164 raise RuntimeError("No defects for ccdSerial %s in %s" % (ccdSerial, defectsFitsPath)) 

165 

166 def _defectLookup(self, dataId): 

167 """Find the defects for a given CCD. 

168 

169 Parameters 

170 ---------- 

171 dataId : `dict` 

172 Dataset identifier. 

173 

174 Returns 

175 ------- 

176 `str` or None 

177 Path to the defects file or None if not available. 

178 """ 

179 

180 if self.registry is None: 

181 raise RuntimeError("No registry for defect lookup") 

182 

183 rows = self.registry.executeQuery( 

184 ("defects",), 

185 ("raw",), 

186 [("visit", "?"), ("ccd", "?")], None, (dataId['visit'], dataId['ccd']), 

187 ) 

188 if len(rows) == 0: 

189 return None 

190 

191 if len(rows) == 1: 

192 return os.path.join(self.defectPath, rows[0][0]) 

193 else: 

194 raise RuntimeError("Querying for defects (%s) returns %d files: %s" % 

195 (dataId['id'], len(rows), ", ".join([_[0] for _ in rows]))) 

196 

197 def _getCcdKeyVal(self, dataId): 

198 ccdName = self._extractDetectorName(dataId) 

199 return ("ccdSerial", self.camera[ccdName].getSerial()) 

200 

201 def _extractDetectorName(self, dataId): 

202 return "ccd%02d" % dataId['ccd'] 

203 

204 def _computeCcdExposureId(self, dataId): 

205 """Compute the 64-bit (long) identifier for a CCD exposure. 

206 

207 Parameters 

208 ---------- 

209 dataId : `dict` 

210 Data identifier with visit, ccd. 

211 """ 

212 pathId = self._transformId(dataId) 

213 visit = int(pathId['visit']) 

214 ccd = int(pathId['ccd']) 

215 return visit * 36 + ccd 

216 

217 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

218 """Hook to retrieve identifier for CCD""" 

219 return self._computeCcdExposureId(dataId) 

220 

221 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

222 """Hook to retrieve number of bits in identifier for CCD""" 

223 return 32 

224 

225 def _computeCoaddExposureId(self, dataId, singleFilter): 

226 """Compute the 64-bit (long) identifier for a coadd. 

227 

228 Parameters 

229 ---------- 

230 dataId : `dict` 

231 Data identifier with tract and patch. 

232 singleFilter : `bool` 

233 True means the desired ID is for a single-filter coadd, 

234 in which case dataId must contain filter. 

235 """ 

236 tract = int(dataId['tract']) 

237 if tract < 0 or tract >= 2**MegacamMapper._nbit_tract: 

238 raise RuntimeError('tract not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

239 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

240 for p in (patchX, patchY): 

241 if p < 0 or p >= 2**MegacamMapper._nbit_patch: 

242 raise RuntimeError('patch not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

243 oid = (((tract << MegacamMapper._nbit_patch) + patchX) << MegacamMapper._nbit_patch) + patchY 

244 if singleFilter: 

245 return (oid << MegacamMapper._nbit_filter) + afwImageUtils.Filter(dataId['filter']).getId() 

246 return oid 

247 

248 def bypass_CoaddExposureId_bits(self, datasetType, pythonType, location, dataId): 

249 return 1 + 7 + 13*2 + 3 

250 

251 def bypass_CoaddExposureId(self, datasetType, pythonType, location, dataId): 

252 return self._computeCoaddExposureId(dataId, True) 

253 

254 bypass_deepCoaddId = bypass_CoaddExposureId 

255 

256 bypass_deepCoaddId_bits = bypass_CoaddExposureId_bits 

257 

258 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

259 return self._computeCoaddExposureId(dataId, False) 

260 

261 bypass_deepMergedCoaddId_bits = bypass_CoaddExposureId_bits 

262 

263 def _computeStackExposureId(self, dataId): 

264 """Compute the 64-bit (long) identifier for a Stack exposure. 

265 

266 Parameters 

267 ---------- 

268 dataId : `dict` 

269 Data identifier with stack, patch, filter 

270 """ 

271 nPatches = 1000000 

272 return (int(dataId["stack"]) * nPatches + int(dataId["patch"])) 

273 

274 def _standardizeDetrend(self, detrend, image, dataId, filter=False): 

275 """Hack up detrend images to remove troublesome keyword""" 

276 md = image.getMetadata() 

277 removeKeyword(md, 'RADECSYS') # Irrelevant, and use of "GAPPT" breaks wcslib 

278 md.set('TELAZ', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

279 md.set('TELALT', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

280 exp = exposureFromImage(image, logger=self.log) 

281 return self._standardizeExposure(self.calibrations[detrend], exp, dataId, filter=filter, 

282 trimmed=False) 

283 

284 def std_bias(self, image, dataId): 

285 return self._standardizeDetrend("bias", image, dataId, filter=False) 

286 

287 def std_dark(self, image, dataId): 

288 return self._standardizeDetrend("dark", image, dataId, filter=False) 

289 

290 def std_flat(self, image, dataId): 

291 return self._standardizeDetrend("flat", image, dataId, filter=True) 

292 

293 def std_fringe(self, image, dataId): 

294 return self._standardizeDetrend("fringe", image, dataId, filter=True) 

295 

296 

297def removeKeyword(md, key): 

298 """Remove a keyword from a header without raising an exception if it 

299 doesn't exist""" 

300 if md.exists(key): 

301 md.remove(key)