Coverage for python/lsst/obs/cfht/megacamMapper.py: 26%

123 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-03 01:43 -0700

1# 

2# LSST Data Management System 

3# Copyright 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["MegacamMapper"] 

24 

25import os 

26import warnings 

27 

28from astropy.io import fits 

29 

30import lsst.geom as geom 

31import lsst.afw.image.utils as afwImageUtils 

32import lsst.ip.isr as ipIsr 

33import lsst.daf.persistence as dafPersist 

34 

35from lsst.daf.persistence import Policy 

36from lsst.obs.base import CameraMapper, exposureFromImage 

37from .makeMegacamRawVisitInfo import MakeMegacamRawVisitInfo 

38from ._instrument import MegaPrime 

39from .cfhtFilters import MEGAPRIME_FILTER_DEFINITIONS 

40 

41 

42class MegacamMapper(CameraMapper): 

43 """Camera Mapper for CFHT MegaCam.""" 

44 packageName = "obs_cfht" 

45 _gen3instrument = MegaPrime 

46 

47 MakeRawVisitInfoClass = MakeMegacamRawVisitInfo 

48 

49 def __init__(self, **kwargs): 

50 policyFile = Policy.defaultPolicyFile("obs_cfht", "MegacamMapper.yaml", "policy") 

51 policy = Policy(policyFile) 

52 repositoryDir = os.path.dirname(policyFile) 

53 super(MegacamMapper, self).__init__(policy, repositoryDir, **kwargs) 

54 

55 # Defect registry and root. Defects are stored with the camera and the 

56 # registry is loaded from the camera package, which is on the local 

57 # filesystem. 

58 self.defectRegistry = None 

59 if 'defects' in policy: 

60 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

61 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

62 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

63 

64 # The "ccd" provided by the user is translated through the registry 

65 # into an extension name for the "raw" template. The template 

66 # therefore doesn't include "ccd", so we need to ensure it's 

67 # explicitly included so the ArgumentParser can recognise and accept 

68 # it. 

69 

70 self.exposures['raw'].keyDict['ccd'] = int 

71 

72 MEGAPRIME_FILTER_DEFINITIONS.defineFilters() 

73 

74 # Ensure each dataset type of interest knows about the full range of 

75 # keys available from the registry 

76 keys = {'runId': str, 

77 'object': str, 

78 'visit': int, 

79 'ccd': int, 

80 'extension': int, 

81 'state': str, 

82 'filter': str, 

83 'date': str, 

84 'taiObs': str, 

85 'expTime': float, 

86 } 

87 for name in ("raw", "calexp", "postISRCCD", "src", "icSrc", "icMatch"): 

88 self.mappings[name].keyDict.update(keys) 

89 

90 # 

91 # The number of bits allocated for fields in object IDs, appropriate 

92 # for the default-configured Rings skymap. 

93 # 

94 

95 MegacamMapper._nbit_tract = 16 

96 MegacamMapper._nbit_patch = 5 

97 MegacamMapper._nbit_filter = 6 

98 

99 MegacamMapper._nbit_id = 64 - (MegacamMapper._nbit_tract + 2*MegacamMapper._nbit_patch 

100 + MegacamMapper._nbit_filter) 

101 

102 with warnings.catch_warnings(): 

103 # suppress Filter warnings; we already know this is deprecated 

104 warnings.simplefilter('ignore', category=FutureWarning) 

105 if len(afwImageUtils.Filter.getNames()) >= 2**MegacamMapper._nbit_filter: 

106 raise RuntimeError("You have more filters defined than fit into the %d bits allocated" % 

107 MegacamMapper._nbit_filter) 

108 

109 def map_defects(self, dataId, write=False): 

110 """Map defects dataset. 

111 

112 Returns 

113 ------- 

114 `lsst.daf.butler.ButlerLocation` 

115 Minimal ButlerLocation containing just the locationList field 

116 (just enough information that bypass_defects can use it). 

117 """ 

118 defectFitsPath = self._defectLookup(dataId=dataId) 

119 if defectFitsPath is None: 

120 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

121 

122 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

123 dataId, self, 

124 storage=self.rootStorage) 

125 

126 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

127 """Return a defect based on the butler location returned by 

128 map_defects. 

129 

130 Parameters 

131 ---------- 

132 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

133 A ButlerLocation with locationList = path to defects FITS file. 

134 dataId : `dict` 

135 The usual data ID; "ccd" must be set. 

136 

137 Notes 

138 ----- 

139 The name "bypass_XXX" means the butler makes no attempt to convert 

140 the ButlerLocation into an object, which is what we want for now, 

141 since that conversion is a bit tricky. 

142 """ 

143 (ccdKey, ccdSerial) = self._getCcdKeyVal(dataId) 

144 defectsFitsPath = butlerLocation.locationList[0] 

145 with fits.open(defectsFitsPath) as hduList: 

146 for hdu in hduList[1:]: 

147 if str(hdu.header["SERIAL"]) != ccdSerial: 

148 continue 

149 

150 defectList = ipIsr.Defects() 

151 for data in hdu.data: 

152 bbox = geom.Box2I( 

153 geom.Point2I(int(data['x0']), int(data['y0'])), 

154 geom.Extent2I(int(data['width']), int(data['height'])), 

155 ) 

156 defectList.append(bbox) 

157 return defectList 

158 

159 raise RuntimeError("No defects for ccdSerial %s in %s" % (ccdSerial, defectsFitsPath)) 

160 

161 def _defectLookup(self, dataId): 

162 """Find the defects for a given CCD. 

163 

164 Parameters 

165 ---------- 

166 dataId : `dict` 

167 Dataset identifier. 

168 

169 Returns 

170 ------- 

171 `str` or None 

172 Path to the defects file or None if not available. 

173 """ 

174 

175 if self.registry is None: 

176 raise RuntimeError("No registry for defect lookup") 

177 

178 rows = self.registry.executeQuery( 

179 ("defects",), 

180 ("raw",), 

181 [("visit", "?"), ("ccd", "?")], None, (dataId['visit'], dataId['ccd']), 

182 ) 

183 if len(rows) == 0: 

184 return None 

185 

186 if len(rows) == 1: 

187 return os.path.join(self.defectPath, rows[0][0]) 

188 else: 

189 raise RuntimeError("Querying for defects (%s) returns %d files: %s" % 

190 (dataId['id'], len(rows), ", ".join([_[0] for _ in rows]))) 

191 

192 def _getCcdKeyVal(self, dataId): 

193 ccdName = self._extractDetectorName(dataId) 

194 return ("ccdSerial", self.camera[ccdName].getSerial()) 

195 

196 def _extractDetectorName(self, dataId): 

197 return "ccd%02d" % dataId['ccd'] 

198 

199 def _computeCcdExposureId(self, dataId): 

200 """Compute the 64-bit (long) identifier for a CCD exposure. 

201 

202 Parameters 

203 ---------- 

204 dataId : `dict` 

205 Data identifier with visit, ccd. 

206 """ 

207 pathId = self._transformId(dataId) 

208 visit = int(pathId['visit']) 

209 ccd = int(pathId['ccd']) 

210 return visit * 36 + ccd 

211 

212 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

213 """Hook to retrieve identifier for CCD""" 

214 return self._computeCcdExposureId(dataId) 

215 

216 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

217 """Hook to retrieve number of bits in identifier for CCD""" 

218 return 32 

219 

220 def _computeCoaddExposureId(self, dataId, singleFilter): 

221 """Compute the 64-bit (long) identifier for a coadd. 

222 

223 Parameters 

224 ---------- 

225 dataId : `dict` 

226 Data identifier with tract and patch. 

227 singleFilter : `bool` 

228 True means the desired ID is for a single-filter coadd, 

229 in which case dataId must contain filter. 

230 """ 

231 tract = int(dataId['tract']) 

232 if tract < 0 or tract >= 2**MegacamMapper._nbit_tract: 

233 raise RuntimeError('tract not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

234 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

235 for p in (patchX, patchY): 

236 if p < 0 or p >= 2**MegacamMapper._nbit_patch: 

237 raise RuntimeError('patch not in range [0,%d)' % (2**MegacamMapper._nbit_tract)) 

238 oid = (((tract << MegacamMapper._nbit_patch) + patchX) << MegacamMapper._nbit_patch) + patchY 

239 if singleFilter: 

240 return (oid << MegacamMapper._nbit_filter) + afwImageUtils.Filter(dataId['filter']).getId() 

241 return oid 

242 

243 def bypass_CoaddExposureId_bits(self, datasetType, pythonType, location, dataId): 

244 return 1 + 7 + 13*2 + 3 

245 

246 def bypass_CoaddExposureId(self, datasetType, pythonType, location, dataId): 

247 return self._computeCoaddExposureId(dataId, True) 

248 

249 bypass_deepCoaddId = bypass_CoaddExposureId 

250 

251 bypass_deepCoaddId_bits = bypass_CoaddExposureId_bits 

252 

253 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

254 return self._computeCoaddExposureId(dataId, False) 

255 

256 bypass_deepMergedCoaddId_bits = bypass_CoaddExposureId_bits 

257 

258 def _computeStackExposureId(self, dataId): 

259 """Compute the 64-bit (long) identifier for a Stack exposure. 

260 

261 Parameters 

262 ---------- 

263 dataId : `dict` 

264 Data identifier with stack, patch, filter 

265 """ 

266 nPatches = 1000000 

267 return (int(dataId["stack"]) * nPatches + int(dataId["patch"])) 

268 

269 def _standardizeDetrend(self, detrend, image, dataId, filter=False): 

270 """Hack up detrend images to remove troublesome keyword""" 

271 md = image.getMetadata() 

272 removeKeyword(md, 'RADECSYS') # Irrelevant, and use of "GAPPT" breaks wcslib 

273 md.set('TELAZ', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

274 md.set('TELALT', 0) # Irrelevant, -9999 value breaks VisitInfo, and absence generates a warning 

275 exp = exposureFromImage(image, logger=self.log) 

276 return self._standardizeExposure(self.calibrations[detrend], exp, dataId, filter=filter, 

277 trimmed=False) 

278 

279 def std_bias(self, image, dataId): 

280 return self._standardizeDetrend("bias", image, dataId, filter=False) 

281 

282 def std_dark(self, image, dataId): 

283 return self._standardizeDetrend("dark", image, dataId, filter=False) 

284 

285 def std_flat(self, image, dataId): 

286 return self._standardizeDetrend("flat", image, dataId, filter=True) 

287 

288 def std_fringe(self, image, dataId): 

289 return self._standardizeDetrend("fringe", image, dataId, filter=True) 

290 

291 

292def removeKeyword(md, key): 

293 """Remove a keyword from a header without raising an exception if it 

294 doesn't exist""" 

295 if md.exists(key): 

296 md.remove(key)