Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["MegacamMapper"] 

24 

25import os 

26import warnings 

27 

28from astropy.io import fits 

29 

30import lsst.geom as geom 

31import lsst.afw.image.utils as afwImageUtils 

32import lsst.ip.isr as ipIsr 

33import lsst.daf.persistence as dafPersist 

34 

35from lsst.daf.persistence import Policy 

36from lsst.obs.base import CameraMapper, exposureFromImage 

37from .makeMegacamRawVisitInfo import MakeMegacamRawVisitInfo 

38from ._instrument import MegaPrime 

39 

40 

41class MegacamMapper(CameraMapper): 

42 """Camera Mapper for CFHT MegaCam.""" 

43 packageName = "obs_cfht" 

44 _gen3instrument = MegaPrime 

45 

46 MakeRawVisitInfoClass = MakeMegacamRawVisitInfo 

47 

48 def __init__(self, **kwargs): 

49 policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") 

50 policy = Policy(policyFile) 

51 repositoryDir = os.path.dirname(policyFile) 

52 super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs) 

53 

54 # Defect registry and root. Defects are stored with the camera and the 

55 # registry is loaded from the camera package, which is on the local 

56 # filesystem. 

57 self.defectRegistry = None 

58 if 'defects' in policy: 

59 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

60 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

61 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

62 

63 # The "ccd" provided by the user is translated through the registry 

64 # into an extension name for the "raw" template. The template 

65 # therefore doesn't include "ccd", so we need to ensure it's 

66 # explicitly included so the ArgumentParser can recognise and accept 

67 # it. 

68 

69 self.exposures['raw'].keyDict['ccd'] = int 

70 

71 with warnings.catch_warnings(): 

72 # suppress Filter warnings; we already know this is deprecated 

73 warnings.simplefilter('ignore', category=FutureWarning) 

74 

75 afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") 

76 afwImageUtils.defineFilter('u2', lambdaEff=354, alias="u.MP9302") 

77 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") 

78 afwImageUtils.defineFilter('g2', lambdaEff=472, alias="g.MP9402") 

79 afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") 

80 afwImageUtils.defineFilter('r2', lambdaEff=640, alias="r.MP9602") 

81 afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") 

82 afwImageUtils.defineFilter('i2', lambdaEff=764, alias="i.MP9702") 

83 afwImageUtils.defineFilter('i3', lambdaEff=776, alias="i.MP9703") 

84 afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") 

85 afwImageUtils.defineFilter('z2', lambdaEff=926, alias="z.MP9901") 

86 

87 # define filters? 

88 self.filterIdMap = dict(u=0, g=1, r=2, i=3, z=4, i2=5, u2=6, g2=7, r2=8, i3=9, z2=10) 

89 

90 # Ensure each dataset type of interest knows about the full range of 

91 # keys available from the registry 

92 keys = {'runId': str, 

93 'object': str, 

94 'visit': int, 

95 'ccd': int, 

96 'extension': int, 

97 'state': str, 

98 'filter': str, 

99 'date': str, 

100 'taiObs': str, 

101 'expTime': float, 

102 } 

103 for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): 

104 self.mappings[name].keyDict.update(keys) 

105 

106 # 

107 # The number of bits allocated for fields in object IDs, appropriate 

108 # for the default-configured Rings skymap. 

109 # 

110 

111 MegacamMapper._nbit_tract = 16 

112 MegacamMapper._nbit_patch = 5 

113 MegacamMapper._nbit_filter = 6 

114 

115 MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch 

116 + MegacamMapper._nbit_filter) 

117 

118 with warnings.catch_warnings(): 

119 # suppress Filter warnings; we already know this is deprecated 

120 warnings.simplefilter('ignore', category=FutureWarning) 

121 if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter: 

122 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

123 MegacamMapper._nbit_filter) 

124 

125 def map_defects(self, dataId, write=False): 

126 """Map defects dataset. 

127 

128 Returns 

129 ------- 

130 `lsst.daf.butler.ButlerLocation` 

131 Minimal ButlerLocation containing just the locationList field 

132 (just enough information that bypass_defects can use it). 

133 """ 

134 defectFitsPath = self._defectLookup(dataId=dataId) 

135 if defectFitsPath is None: 

136 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

137 

138 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

139 dataId, self, 

140 storage=self.rootStorage) 

141 

142 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

143 """Return a defect based on the butler location returned by 

144 map_defects. 

145 

146 Parameters 

147 ---------- 

148 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

149 A ButlerLocation with locationList = path to defects FITS file. 

150 dataId : `dict` 

151 The usual data ID; "ccd" must be set. 

152 

153 Notes 

154 ----- 

155 The name "bypass_XXX" means the butler makes no attempt to convert 

156 the ButlerLocation into an object, which is what we want for now, 

157 since that conversion is a bit tricky. 

158 """ 

159 (ccdKey, ccdSerial) = self._getCcdKeyVal(dataId) 

160 defectsFitsPath = butlerLocation.locationList[0] 

161 with fits.open(defectsFitsPath) as hduList: 

162 for hdu in hduList[1:]: 

163 if str(hdu.header["SERIAL"]) != ccdSerial: 

164 continue 

165 

166 defectList = ipIsr.Defects() 

167 for data in hdu.data: 

168 bbox = geom.Box2I( 

169 geom.Point2I(int(data['x0']), int(data['y0'])), 

170 geom.Extent2I(int(data['width']), int(data['height'])), 

171 ) 

172 defectList.append(bbox) 

173 return defectList 

174 

175 raise RuntimeError("No defects for ccdSerial %s in %s" % (ccdSerial, defectsFitsPath)) 

176 

177 def _defectLookup(self, dataId): 

178 """Find the defects for a given CCD. 

179 

180 Parameters 

181 ---------- 

182 dataId : `dict` 

183 Dataset identifier. 

184 

185 Returns 

186 ------- 

187 `str` or None 

188 Path to the defects file or None if not available. 

189 """ 

190 

191 if self.registry is None: 

192 raise RuntimeError("No registry for defect lookup") 

193 

194 rows = self.registry.executeQuery( 

195 ("defects",), 

196 ("raw",), 

197 [("visit", "?"), ("ccd", "?")], None, (dataId['visit'], dataId['ccd']), 

198 ) 

199 if len(rows) == 0: 

200 return None 

201 

202 if len(rows) == 1: 

203 return os.path.join(self.defectPath, rows[0][0]) 

204 else: 

205 raise RuntimeError("Querying for defects (%s) returns %d files: %s" % 

206 (dataId['id'], len(rows), ", ".join([_[0] for _ in rows]))) 

207 

208 def _getCcdKeyVal(self, dataId): 

209 ccdName = self._extractDetectorName(dataId) 

210 return ("ccdSerial", self.camera[ccdName].getSerial()) 

211 

212 def _extractDetectorName(self, dataId): 

213 return "ccd%02d" % dataId['ccd'] 

214 

215 def _computeCcdExposureId(self, dataId): 

216 """Compute the 64-bit (long) identifier for a CCD exposure. 

217 

218 Parameters 

219 ---------- 

220 dataId : `dict` 

221 Data identifier with visit, ccd. 

222 """ 

223 pathId = self._transformId(dataId) 

224 visit = int(pathId['visit']) 

225 ccd = int(pathId['ccd']) 

226 return visit * 36 + ccd 

227 

228 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

229 """Hook to retrieve identifier for CCD""" 

230 return self._computeCcdExposureId(dataId) 

231 

232 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

233 """Hook to retrieve number of bits in identifier for CCD""" 

234 return 32 

235 

236 def _computeCoaddExposureId(self, dataId, singleFilter): 

237 """Compute the 64-bit (long) identifier for a coadd. 

238 

239 Parameters 

240 ---------- 

241 dataId : `dict` 

242 Data identifier with tract and patch. 

243 singleFilter : `bool` 

244 True means the desired ID is for a single-filter coadd, 

245 in which case dataId must contain filter. 

246 """ 

247 tract = int(dataId['tract']) 

248 if tract < 0 or tract >= 2**MegacamMapper._nbit_tract: 

249 raise RuntimeError('tract not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

250 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

251 for p in (patchX, patchY): 

252 if p < 0 or p >= 2**MegacamMapper._nbit_patch: 

253 raise RuntimeError('patch not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

254 oid = (((tract << MegacamMapper._nbit_patch) + patchX) << MegacamMapper._nbit_patch) + patchY 

255 if singleFilter: 

256 return (oid << MegacamMapper._nbit_filter) + afwImageUtils.Filter(dataId['filter']).getId() 

257 return oid 

258 

259 def bypass_CoaddExposureId_bits(self, datasetType, pythonType, location, dataId): 

260 return 1 + 7 + 13*2 + 3 

261 

262 def bypass_CoaddExposureId(self, datasetType, pythonType, location, dataId): 

263 return self._computeCoaddExposureId(dataId, True) 

264 

265 bypass_deepCoaddId = bypass_CoaddExposureId 

266 

267 bypass_deepCoaddId_bits = bypass_CoaddExposureId_bits 

268 

269 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

270 return self._computeCoaddExposureId(dataId, False) 

271 

272 bypass_deepMergedCoaddId_bits = bypass_CoaddExposureId_bits 

273 

274 def _computeStackExposureId(self, dataId): 

275 """Compute the 64-bit (long) identifier for a Stack exposure. 

276 

277 Parameters 

278 ---------- 

279 dataId : `dict` 

280 Data identifier with stack, patch, filter 

281 """ 

282 nPatches = 1000000 

283 return (int(dataId["stack"]) * nPatches + int(dataId["patch"])) 

284 

285 def _standardizeDetrend(self, detrend, image, dataId, filter=False): 

286 """Hack up detrend images to remove troublesome keyword""" 

287 md = image.getMetadata() 

288 removeKeyword(md, 'RADECSYS') # Irrelevant, and use of "GAPPT" breaks wcslib 

289 md.set('TELAZ', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

290 md.set('TELALT', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

291 exp = exposureFromImage(image, logger=self.log) 

292 return self._standardizeExposure(self.calibrations[detrend], exp, dataId, filter=filter, 

293 trimmed=False) 

294 

295 def std_bias(self, image, dataId): 

296 return self._standardizeDetrend("bias", image, dataId, filter=False) 

297 

298 def std_dark(self, image, dataId): 

299 return self._standardizeDetrend("dark", image, dataId, filter=False) 

300 

301 def std_flat(self, image, dataId): 

302 return self._standardizeDetrend("flat", image, dataId, filter=True) 

303 

304 def std_fringe(self, image, dataId): 

305 return self._standardizeDetrend("fringe", image, dataId, filter=True) 

306 

307 

308def removeKeyword(md, key): 

309 """Remove a keyword from a header without raising an exception if it 

310 doesn't exist""" 

311 if md.exists(key): 

312 md.remove(key)