Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012, 2013 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["LsstSimMapper"] 

24 

25import os 

26import re 

27from astropy.io import fits 

28 

29import lsst.daf.base as dafBase 

30import lsst.afw.image.utils as afwImageUtils 

31import lsst.geom as geom 

32import lsst.daf.persistence as dafPersist 

33from lsst.meas.algorithms import Defects 

34from .makeLsstSimRawVisitInfo import MakeLsstSimRawVisitInfo 

35from lsst.utils import getPackageDir 

36 

37from lsst.obs.base import CameraMapper 

38 

39# Solely to get boost serialization registrations for Measurement subclasses 

40 

41 

42class LsstSimMapper(CameraMapper): 

43 packageName = 'obs_lsstSim' 

44 

45 MakeRawVisitInfoClass = MakeLsstSimRawVisitInfo 

46 

47 _CcdNameRe = re.compile(r"R:(\d,\d) S:(\d,\d(?:,[AB])?)$") 

48 

49 def __init__(self, inputPolicy=None, **kwargs): 

50 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, "LsstSimMapper.yaml", "policy") 

51 policy = dafPersist.Policy(policyFile) 

52 repositoryDir = os.path.join(getPackageDir(self.packageName), 'policy') 

53 self.defectRegistry = None 

54 if 'defects' in policy: 

55 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

56 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

57 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

58 

59 self.doFootprints = False 

60 if inputPolicy is not None: 

61 for kw in inputPolicy.paramNames(True): 

62 if kw == "doFootprints": 

63 self.doFootprints = True 

64 else: 

65 kwargs[kw] = inputPolicy.get(kw) 

66 

67 super(LsstSimMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

68 self.filterIdMap = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5, 'i2': 5} 

69 

70 # The LSST Filters from L. Jones 04/07/10 

71 afwImageUtils.resetFilters() 

72 afwImageUtils.defineFilter('u', lambdaEff=364.59, lambdaMin=324.0, lambdaMax=395.0) 

73 afwImageUtils.defineFilter('g', lambdaEff=476.31, lambdaMin=405.0, lambdaMax=552.0) 

74 afwImageUtils.defineFilter('r', lambdaEff=619.42, lambdaMin=552.0, lambdaMax=691.0) 

75 afwImageUtils.defineFilter('i', lambdaEff=752.06, lambdaMin=818.0, lambdaMax=921.0) 

76 afwImageUtils.defineFilter('z', lambdaEff=866.85, lambdaMin=922.0, lambdaMax=997.0) 

77 # official y filter 

78 afwImageUtils.defineFilter('y', lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4']) 

79 # If/when y3 sim data becomes available, uncomment this and 

80 # modify the schema appropriately 

81 # afwImageUtils.defineFilter('y3', 1002.44) # candidate y-band 

82 

83 def _transformId(self, dataId): 

84 """Transform an ID dict into standard form for LSST 

85 

86 Standard keys are as follows: 

87 - raft: in the form <x>,<y> 

88 - sensor: in the form <x>,<y>,<c> where <c> = A or B 

89 - channel: in the form <x>,<y> 

90 - snap: exposure number 

91 

92 Other supported keys, which are used to set the above, if not already set: 

93 - ccd: an alias for sensor (hence NOT the full ccd name) 

94 - ccdName or sensorName: full ccd name in the form R:<x>,<y> S:<x>,<y>[,<c>] 

95 if found, used to set raft and sensor, if not already set 

96 - channelName, ampName: an alternate way to specify channel, in the form: IDxx 

97 - amp: an alias for channel 

98 - exposure: an alias for snap 

99 

100 @param dataId[in] (dict) Dataset identifier; this must not be modified 

101 @return (dict) Transformed dataset identifier 

102 @raise RuntimeError if a value is not valid 

103 """ 

104 actualId = dataId.copy() 

105 for ccdAlias in ("ccdName", "sensorName"): 

106 if ccdAlias in actualId: 

107 ccdName = actualId[ccdAlias].upper() 

108 m = self._CcdNameRe.match(ccdName) 

109 if m is None: 

110 raise RuntimeError("Invalid value for %s: %r" % (ccdAlias, ccdName)) 

111 actualId.setdefault("raft", m.group(1)) 

112 actualId.setdefault("sensor", m.group(2)) 

113 break 

114 if "ccd" in actualId: 

115 actualId.setdefault("sensor", actualId["ccd"]) 

116 if "amp" in actualId: 

117 actualId.setdefault("channel", actualId["amp"]) 

118 elif "channel" not in actualId: 

119 for ampName in ("ampName", "channelName"): 

120 if ampName in actualId: 

121 m = re.match(r'ID(\d+)$', actualId[ampName]) 

122 channelNumber = int(m.group(1)) 

123 channelX = channelNumber % 8 

124 channelY = channelNumber // 8 

125 actualId['channel'] = str(channelX) + "," + str(channelY) 

126 break 

127 if "exposure" in actualId: 

128 actualId.setdefault("snap", actualId["exposure"]) 

129 

130 # why strip out the commas after carefully adding them? 

131 if "raft" in actualId: 

132 actualId['raft'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['raft']) 

133 if "sensor" in actualId: 

134 actualId['sensor'] = actualId['sensor'].replace(",", "") 

135 if "channel" in actualId: 

136 actualId['channel'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['channel']) 

137 return actualId 

138 

139 def validate(self, dataId): 

140 for component in ("raft", "sensor", "channel"): 

141 if component not in dataId: 

142 continue 

143 val = dataId[component] 

144 if not isinstance(val, str): 

145 raise RuntimeError( 

146 "%s identifier should be type str, not %s: %r" % (component.title(), type(val), val)) 

147 if component == "sensor": 

148 if not re.search(r'^\d,\d(,[AB])?$', val): 

149 raise RuntimeError("Invalid %s identifier: %r" % (component, val)) 

150 else: 

151 if not re.search(r'^(\d),(\d)$', val): 

152 raise RuntimeError("Invalid %s identifier: %r" % (component, val)) 

153 return dataId 

154 

155 def _extractDetectorName(self, dataId): 

156 return "R:%(raft)s S:%(sensor)s" % dataId 

157 

158 def getDataId(self, visit, ccdId): 

159 """get dataId dict from visit and ccd identifier 

160 

161 @param visit 32 or 64-bit depending on camera 

162 @param ccdId detector name: same as detector.getName() 

163 """ 

164 dataId = {'visit': int(visit)} 

165 m = self._CcdNameRe.match(ccdId) 

166 if m is None: 

167 raise RuntimeError("Cannot parse ccdId=%r" % (ccdId,)) 

168 dataId['raft'] = m.group(0) 

169 dataId['sensor'] = m.group(1) 

170 return dataId 

171 

172 def _computeAmpExposureId(self, dataId): 

173 # visit, snap, raft, sensor, channel): 

174 """Compute the 64-bit (long) identifier for an amp exposure. 

175 

176 @param dataId (dict) Data identifier with visit, snap, raft, sensor, channel 

177 """ 

178 

179 pathId = self._transformId(dataId) 

180 visit = pathId['visit'] 

181 snap = pathId['snap'] 

182 raft = pathId['raft'] # "xy" e.g. "20" 

183 sensor = pathId['sensor'] # "xy" e.g. "11" 

184 channel = pathId['channel'] # "yx" e.g. "05" (NB: yx, not xy, in original comment) 

185 

186 r1, r2 = raft 

187 s1, s2 = sensor 

188 c1, c2 = channel 

189 return (visit << 13) + (snap << 12) + \ 

190 (int(r1) * 5 + int(r2)) * 160 + \ 

191 (int(s1) * 3 + int(s2)) * 16 + \ 

192 (int(c1) * 8 + int(c2)) 

193 

194 def _computeCcdExposureId(self, dataId): 

195 """Compute the 64-bit (long) identifier for a CCD exposure. 

196 

197 @param dataId (dict) Data identifier with visit, raft, sensor 

198 """ 

199 

200 pathId = self._transformId(dataId) 

201 visit = pathId['visit'] 

202 raft = pathId['raft'] # "xy" e.g. "20" 

203 sensor = pathId['sensor'] # "xy" e.g. "11" 

204 

205 r1, r2 = raft 

206 s1, s2 = sensor 

207 return (visit << 9) + \ 

208 (int(r1) * 5 + int(r2)) * 10 + \ 

209 (int(s1) * 3 + int(s2)) 

210 

211 def _computeCoaddExposureId(self, dataId, singleFilter): 

212 """Compute the 64-bit (long) identifier for a coadd. 

213 

214 @param dataId (dict) Data identifier with tract and patch. 

215 @param singleFilter (bool) True means the desired ID is for a single- 

216 filter coadd, in which case dataId 

217 must contain filter. 

218 """ 

219 tract = int(dataId['tract']) 

220 if tract < 0 or tract >= 128: 

221 raise RuntimeError('tract not in range [0,128)') 

222 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

223 for p in (patchX, patchY): 

224 if p < 0 or p >= 2**13: 

225 raise RuntimeError('patch component not in range [0, 8192)') 

226 id = (tract * 2**13 + patchX) * 2**13 + patchY 

227 if singleFilter: 

228 return id * 8 + self.filterIdMap[dataId['filter']] 

229 return id 

230 

231 def _defectLookup(self, dataId, dateKey='taiObs'): 

232 """Find the defects for a given CCD. 

233 

234 Parameters 

235 ---------- 

236 dataId : `dict` 

237 Dataset identifier 

238 

239 Returns 

240 ------- 

241 `str` 

242 Path to the defects file or None if not available. 

243 """ 

244 if self.defectRegistry is None: 

245 return None 

246 if self.registry is None: 

247 raise RuntimeError("No registry for defect lookup") 

248 

249 ccdKey, ccdVal = self._getCcdKeyVal(dataId) 

250 

251 dataIdForLookup = {'visit': dataId['visit']} 

252 # .lookup will fail in a posix registry because there is no template to provide. 

253 rows = self.registry.lookup((dateKey), ('raw_visit'), dataIdForLookup) 

254 if len(rows) == 0: 

255 return None 

256 assert len(rows) == 1 

257 dayObs = rows[0][0] 

258 

259 # Lookup the defects for this CCD serial number that are valid at the exposure midpoint. 

260 rows = self.defectRegistry.executeQuery(("path",), ("defect",), 

261 [(ccdKey, "?")], 

262 ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"), 

263 (ccdVal, dayObs)) 

264 if not rows or len(rows) == 0: 

265 return None 

266 if len(rows) == 1: 

267 return os.path.join(self.defectPath, rows[0][0]) 

268 else: 

269 raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" % 

270 (ccdVal, dayObs, len(rows), ", ".join([_[0] for _ in rows]))) 

271 

272 def map_defects(self, dataId, write=False): 

273 """Map defects dataset. 

274 

275 Returns 

276 ------- 

277 `lsst.daf.butler.ButlerLocation` 

278 Minimal ButlerLocation containing just the locationList field 

279 (just enough information that bypass_defects can use it). 

280 """ 

281 defectFitsPath = self._defectLookup(dataId=dataId) 

282 if defectFitsPath is None: 

283 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

284 

285 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

286 dataId, self, 

287 storage=self.rootStorage) 

288 

289 def map_linearizer(self, dataId, write=False): 

290 return None 

291 

292 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

293 """Return a defect based on the butler location returned by map_defects 

294 

295 Parameters 

296 ---------- 

297 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

298 locationList = path to defects FITS file 

299 dataId : `dict` 

300 Butler data ID; "ccd" must be set. 

301 

302 Note: the name "bypass_XXX" means the butler makes no attempt to 

303 convert the ButlerLocation into an object, which is what we want for 

304 now, since that conversion is a bit tricky. 

305 """ 

306 detectorName = self._extractDetectorName(dataId) 

307 defectsFitsPath = butlerLocation.locationList[0] 

308 

309 with fits.open(defectsFitsPath) as hduList: 

310 for hdu in hduList[1:]: 

311 if hdu.header["name"] != detectorName: 

312 continue 

313 

314 defectList = Defects() 

315 for data in hdu.data: 

316 bbox = geom.Box2I( 

317 geom.Point2I(int(data['x0']), int(data['y0'])), 

318 geom.Extent2I(int(data['width']), int(data['height'])), 

319 ) 

320 defectList.append(bbox) 

321 return defectList 

322 

323 raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath)) 

324 

325 _nbit_id = 30 

326 

327 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

328 """The number of bits used up for patch ID bits""" 

329 return 64 - self._nbit_id 

330 

331 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

332 return self._computeCoaddExposureId(dataId, False) 

333 

334 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

335 """The number of bits used up for patch ID bits""" 

336 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

337 

338 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

339 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

340 

341 @staticmethod 

342 def getShortCcdName(ccdId): 

343 """Convert a CCD name to a form useful as a filename 

344 

345 This LSST version converts spaces to underscores and elides colons and commas. 

346 """ 

347 return re.sub("[:,]", "", ccdId.replace(" ", "_")) 

348 

349 def _setAmpExposureId(self, propertyList, dataId): 

350 propertyList.set("Computed_ampExposureId", self._computeAmpExposureId(dataId)) 

351 return propertyList 

352 

353 def _setCcdExposureId(self, propertyList, dataId): 

354 propertyList.set("Computed_ccdExposureId", self._computeCcdExposureId(dataId)) 

355 return propertyList 

356 

357############################################################################### 

358 

359 def std_raw(self, item, dataId): 

360 md = item.getMetadata() 

361 if md.exists("VERSION") and md.getInt("VERSION") < 16952: 

362 # CRVAL is FK5 at date of observation 

363 dateObsTaiMjd = md.getScalar("TAI") 

364 dateObs = dafBase.DateTime(dateObsTaiMjd, 

365 system=dafBase.DateTime.MJD, 

366 scale=dafBase.DateTime.TAI) 

367 correctedEquinox = dateObs.get(system=dafBase.DateTime.EPOCH, 

368 scale=dafBase.DateTime.TAI) 

369 md.set("EQUINOX", correctedEquinox) 

370 md.set("RADESYS", "FK5") 

371 print("****** changing equinox to", correctedEquinox) 

372 return super(LsstSimMapper, self).std_raw(item, dataId) 

373 

374 def std_eimage(self, item, dataId): 

375 """Standardize a eimage dataset by converting it to an Exposure instead of an Image""" 

376 return self._standardizeExposure(self.exposures['eimage'], item, dataId, trimmed=True) 

377 

378 def _createInitialSkyWcs(self, exposure): 

379 """Create a SkyWcs from the header metadata. 

380 

381 PhoSim data may not have self-consistent boresight and crval/crpix 

382 values, and/or may have been written in FK5, so we just use the 

383 metadata here, and ignore VisitInfo/CameraGeom. 

384 

385 Parameters 

386 ---------- 

387 exposure : `lsst.afw.image.Exposure` 

388 The exposure to get data from, and attach the SkyWcs to. 

389 """ 

390 self._createSkyWcsFromMetadata(exposure) 

391 

392############################################################################### 

393 

394 def _getCcdKeyVal(self, dataId): 

395 """Return CCD key and value used to look a defect in the defect 

396 registry 

397 

398 The default implementation simply returns ("ccd", full detector name) 

399 """ 

400 return ("ccd", self._extractDetectorName(dataId)) 

401 

402 def bypass_ampExposureId(self, datasetType, pythonType, location, dataId): 

403 return self._computeAmpExposureId(dataId) 

404 

405 def bypass_ampExposureId_bits(self, datasetType, pythonType, location, dataId): 

406 return 45 

407 

408 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

409 return self._computeCcdExposureId(dataId) 

410 

411 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

412 return 41 

413 

414 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

415 return self._computeCoaddExposureId(dataId, True) 

416 

417 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

418 return 1 + 7 + 13*2 + 3 

419 

420 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

421 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

422 

423 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

424 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

425 

426############################################################################### 

427 

428 def add_sdqaAmp(self, dataId): 

429 ampExposureId = self._computeAmpExposureId(dataId) 

430 return {"ampExposureId": ampExposureId, "sdqaRatingScope": "AMP"} 

431 

432 def add_sdqaCcd(self, dataId): 

433 ccdExposureId = self._computeCcdExposureId(dataId) 

434 return {"ccdExposureId": ccdExposureId, "sdqaRatingScope": "CCD"} 

435 

436############################################################################### 

437 

438 

439for dsType in ("raw", "postISR"): 

440 setattr(LsstSimMapper, "std_" + dsType + "_md", 440 ↛ exitline 440 didn't jump to the function exit

441 lambda self, item, dataId: self._setAmpExposureId(item, dataId)) 

442for dsType in ("eimage", "postISRCCD", "visitim", "calexp", "calsnap"): 

443 setattr(LsstSimMapper, "std_" + dsType + "_md", 443 ↛ exitline 443 didn't jump to the function exit

444 lambda self, item, dataId: self._setCcdExposureId(item, dataId))