Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["MegacamMapper"] 

24 

25import os 

26 

27from astropy.io import fits 

28 

29import lsst.geom as geom 

30import lsst.afw.image.utils as afwImageUtils 

31import lsst.ip.isr as ipIsr 

32import lsst.daf.persistence as dafPersist 

33 

34from lsst.daf.persistence import Policy 

35from lsst.obs.base import CameraMapper, exposureFromImage 

36from .makeMegacamRawVisitInfo import MakeMegacamRawVisitInfo 

37from ._instrument import MegaPrime 

38 

39 

40class MegacamMapper(CameraMapper): 

41 """Camera Mapper for CFHT MegaCam.""" 

42 packageName = "obs_cfht" 

43 _gen3instrument = MegaPrime 

44 

45 MakeRawVisitInfoClass = MakeMegacamRawVisitInfo 

46 

47 def __init__(self, **kwargs): 

48 policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") 

49 policy = Policy(policyFile) 

50 repositoryDir = os.path.dirname(policyFile) 

51 super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs) 

52 

53 # Defect registry and root. Defects are stored with the camera and the 

54 # registry is loaded from the camera package, which is on the local 

55 # filesystem. 

56 self.defectRegistry = None 

57 if 'defects' in policy: 

58 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

59 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

60 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

61 

62 # The "ccd" provided by the user is translated through the registry 

63 # into an extension name for the "raw" template. The template 

64 # therefore doesn't include "ccd", so we need to ensure it's 

65 # explicitly included so the ArgumentParser can recognise and accept 

66 # it. 

67 

68 self.exposures['raw'].keyDict['ccd'] = int 

69 

70 afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") 

71 afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") 

72 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") 

73 afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") 

74 afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") 

75 afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") 

76 afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") 

77 afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") 

78 afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") 

79 afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") 

80 afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") 

81 

82 # define filters? 

83 self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) 

84 

85 # Ensure each dataset type of interest knows about the full range of 

86 # keys available from the registry 

87 keys = {'runId': str, 

88 'object': str, 

89 'visit': int, 

90 'ccd': int, 

91 'extension': int, 

92 'state': str, 

93 'filter': str, 

94 'date': str, 

95 'taiObs': str, 

96 'expTime': float, 

97 } 

98 for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): 

99 self.mappings[name].keyDict.update(keys) 

100 

101 # 

102 # The number of bits allocated for fields in object IDs, appropriate 

103 # for the default-configured Rings skymap. 

104 # 

105 

106 MegacamMapper._nbit_tract = 16 

107 MegacamMapper._nbit_patch = 5 

108 MegacamMapper._nbit_filter = 6 

109 

110 MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch 

111 + MegacamMapper._nbit_filter) 

112 

113 if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter: 

114 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

115 MegacamMapper._nbit_filter) 

116 

117 def map_defects(self, dataId, write=False): 

118 """Map defects dataset. 

119 

120 Returns 

121 ------- 

122 `lsst.daf.butler.ButlerLocation` 

123 Minimal ButlerLocation containing just the locationList field 

124 (just enough information that bypass_defects can use it). 

125 """ 

126 defectFitsPath = self._defectLookup(dataId=dataId) 

127 if defectFitsPath is None: 

128 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

129 

130 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

131 dataId, self, 

132 storage=self.rootStorage) 

133 

134 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

135 """Return a defect based on the butler location returned by 

136 map_defects. 

137 

138 Parameters 

139 ---------- 

140 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

141 A ButlerLocation with locationList = path to defects FITS file. 

142 dataId : `dict` 

143 The usual data ID; "ccd" must be set. 

144 

145 Notes 

146 ----- 

147 The name "bypass_XXX" means the butler makes no attempt to convert 

148 the ButlerLocation into an object, which is what we want for now, 

149 since that conversion is a bit tricky. 

150 """ 

151 (ccdKey, ccdSerial) = self._getCcdKeyVal(dataId) 

152 defectsFitsPath = butlerLocation.locationList[0] 

153 with fits.open(defectsFitsPath) as hduList: 

154 for hdu in hduList[1:]: 

155 if str(hdu.header["SERIAL"]) != ccdSerial: 

156 continue 

157 

158 defectList = ipIsr.Defects() 

159 for data in hdu.data: 

160 bbox = geom.Box2I( 

161 geom.Point2I(int(data['x0']), int(data['y0'])), 

162 geom.Extent2I(int(data['width']), int(data['height'])), 

163 ) 

164 defectList.append(bbox) 

165 return defectList 

166 

167 raise RuntimeError("No defects for ccdSerial %s in %s" % (ccdSerial, defectsFitsPath)) 

168 

169 def _defectLookup(self, dataId): 

170 """Find the defects for a given CCD. 

171 

172 Parameters 

173 ---------- 

174 dataId : `dict` 

175 Dataset identifier. 

176 

177 Returns 

178 ------- 

179 `str` or None 

180 Path to the defects file or None if not available. 

181 """ 

182 

183 if self.registry is None: 

184 raise RuntimeError("No registry for defect lookup") 

185 

186 rows = self.registry.executeQuery( 

187 ("defects",), 

188 ("raw",), 

189 [("visit", "?"), ("ccd", "?")], None, (dataId['visit'], dataId['ccd']), 

190 ) 

191 if len(rows) == 0: 

192 return None 

193 

194 if len(rows) == 1: 

195 return os.path.join(self.defectPath, rows[0][0]) 

196 else: 

197 raise RuntimeError("Querying for defects (%s) returns %d files: %s" % 

198 (dataId['id'], len(rows), ", ".join([_[0] for _ in rows]))) 

199 

200 def _getCcdKeyVal(self, dataId): 

201 ccdName = self._extractDetectorName(dataId) 

202 return ("ccdSerial", self.camera[ccdName].getSerial()) 

203 

204 def _extractDetectorName(self, dataId): 

205 return "ccd%02d" % dataId['ccd'] 

206 

207 def _computeCcdExposureId(self, dataId): 

208 """Compute the 64-bit (long) identifier for a CCD exposure. 

209 

210 Parameters 

211 ---------- 

212 dataId : `dict` 

213 Data identifier with visit, ccd. 

214 """ 

215 pathId = self._transformId(dataId) 

216 visit = int(pathId['visit']) 

217 ccd = int(pathId['ccd']) 

218 return visit * 36 + ccd 

219 

220 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

221 """Hook to retrieve identifier for CCD""" 

222 return self._computeCcdExposureId(dataId) 

223 

224 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

225 """Hook to retrieve number of bits in identifier for CCD""" 

226 return 32 

227 

228 def _computeCoaddExposureId(self, dataId, singleFilter): 

229 """Compute the 64-bit (long) identifier for a coadd. 

230 

231 Parameters 

232 ---------- 

233 dataId : `dict` 

234 Data identifier with tract and patch. 

235 singleFilter : `bool` 

236 True means the desired ID is for a single-filter coadd, 

237 in which case dataId must contain filter. 

238 """ 

239 tract = int(dataId['tract']) 

240 if tract < 0 or tract >= 2**MegacamMapper._nbit_tract: 

241 raise RuntimeError('tract not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

242 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

243 for p in (patchX, patchY): 

244 if p < 0 or p >= 2**MegacamMapper._nbit_patch: 

245 raise RuntimeError('patch not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

246 oid = (((tract << MegacamMapper._nbit_patch) + patchX) << MegacamMapper._nbit_patch) + patchY 

247 if singleFilter: 

248 return (oid << MegacamMapper._nbit_filter) + afwImageUtils.Filter(dataId['filter']).getId() 

249 return oid 

250 

251 def bypass_CoaddExposureId_bits(self, datasetType, pythonType, location, dataId): 

252 return 1 + 7 + 13*2 + 3 

253 

254 def bypass_CoaddExposureId(self, datasetType, pythonType, location, dataId): 

255 return self._computeCoaddExposureId(dataId, True) 

256 

257 bypass_deepCoaddId = bypass_CoaddExposureId 

258 

259 bypass_deepCoaddId_bits = bypass_CoaddExposureId_bits 

260 

261 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

262 return self._computeCoaddExposureId(dataId, False) 

263 

264 bypass_deepMergedCoaddId_bits = bypass_CoaddExposureId_bits 

265 

266 def _computeStackExposureId(self, dataId): 

267 """Compute the 64-bit (long) identifier for a Stack exposure. 

268 

269 Parameters 

270 ---------- 

271 dataId : `dict` 

272 Data identifier with stack, patch, filter 

273 """ 

274 nPatches = 1000000 

275 return (int(dataId["stack"]) * nPatches + int(dataId["patch"])) 

276 

277 def _standardizeDetrend(self, detrend, image, dataId, filter=False): 

278 """Hack up detrend images to remove troublesome keyword""" 

279 md = image.getMetadata() 

280 removeKeyword(md, 'RADECSYS') # Irrelevant, and use of "GAPPT" breaks wcslib 

281 md.set('TELAZ', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

282 md.set('TELALT', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

283 exp = exposureFromImage(image, logger=self.log) 

284 return self._standardizeExposure(self.calibrations[detrend], exp, dataId, filter=filter, 

285 trimmed=False) 

286 

287 def std_bias(self, image, dataId): 

288 return self._standardizeDetrend("bias", image, dataId, filter=False) 

289 

290 def std_dark(self, image, dataId): 

291 return self._standardizeDetrend("dark", image, dataId, filter=False) 

292 

293 def std_flat(self, image, dataId): 

294 return self._standardizeDetrend("flat", image, dataId, filter=True) 

295 

296 def std_fringe(self, image, dataId): 

297 return self._standardizeDetrend("fringe", image, dataId, filter=True) 

298 

299 

300def removeKeyword(md, key): 

301 """Remove a keyword from a header without raising an exception if it 

302 doesn't exist""" 

303 if md.exists(key): 

304 md.remove(key)