Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012, 2013 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["LsstSimMapper"] 

24 

25import os 

26import re 

27from astropy.io import fits 

28 

29import lsst.daf.base as dafBase 

30import lsst.afw.image.utils as afwImageUtils 

31import lsst.geom as geom 

32import lsst.daf.persistence as dafPersist 

33from lsst.meas.algorithms import Defects 

34from .makeLsstSimRawVisitInfo import MakeLsstSimRawVisitInfo 

35from lsst.utils import getPackageDir 

36 

37from lsst.obs.base import CameraMapper 

38 

39# Solely to get boost serialization registrations for Measurement subclasses 

40 

41 

42class LsstSimMapper(CameraMapper): 

43 packageName = 'obs_lsstSim' 

44 

45 MakeRawVisitInfoClass = MakeLsstSimRawVisitInfo 

46 

47 _CcdNameRe = re.compile(r"R:(\d,\d) S:(\d,\d(?:,[AB])?)$") 

48 

49 def __init__(self, inputPolicy=None, **kwargs): 

50 policyFile = dafPersist.Policy.defaultPolicyFile(self.packageName, "LsstSimMapper.yaml", "policy") 

51 policy = dafPersist.Policy(policyFile) 

52 repositoryDir = os.path.join(getPackageDir(self.packageName), 'policy') 

53 self.defectRegistry = None 

54 if 'defects' in policy: 

55 self.defectPath = os.path.join(repositoryDir, policy['defects']) 

56 defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") 

57 self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) 

58 

59 self.doFootprints = False 

60 if inputPolicy is not None: 

61 for kw in inputPolicy.paramNames(True): 

62 if kw == "doFootprints": 

63 self.doFootprints = True 

64 else: 

65 kwargs[kw] = inputPolicy.get(kw) 

66 

67 super(LsstSimMapper, self).__init__(policy, os.path.dirname(policyFile), **kwargs) 

68 self.filterIdMap = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5, 'i2': 5} 

69 

70 # The LSST Filters from L. Jones 04/07/10 

71 afwImageUtils.resetFilters() 

72 afwImageUtils.defineFilter('u', lambdaEff=364.59, lambdaMin=324.0, lambdaMax=395.0) 

73 afwImageUtils.defineFilter('g', lambdaEff=476.31, lambdaMin=405.0, lambdaMax=552.0) 

74 afwImageUtils.defineFilter('r', lambdaEff=619.42, lambdaMin=552.0, lambdaMax=691.0) 

75 afwImageUtils.defineFilter('i', lambdaEff=752.06, lambdaMin=818.0, lambdaMax=921.0) 

76 afwImageUtils.defineFilter('z', lambdaEff=866.85, lambdaMin=922.0, lambdaMax=997.0) 

77 # official y filter 

78 afwImageUtils.defineFilter('y', lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4']) 

79 # If/when y3 sim data becomes available, uncomment this and 

80 # modify the schema appropriately 

81 # afwImageUtils.defineFilter('y3', 1002.44) # candidate y-band 

82 

83 def _transformId(self, dataId): 

84 """Transform an ID dict into standard form for LSST 

85 

86 Standard keys are as follows: 

87 - raft: in the form <x>,<y> 

88 - sensor: in the form <x>,<y>,<c> where <c> = A or B 

89 - channel: in the form <x>,<y> 

90 - snap: exposure number 

91 

92 Other supported keys, which are used to set the above, if not already set: 

93 - ccd: an alias for sensor (hence NOT the full ccd name) 

94 - ccdName or sensorName: full ccd name in the form R:<x>,<y> S:<x>,<y>[,<c>] 

95 if found, used to set raft and sensor, if not already set 

96 - channelName, ampName: an alternate way to specify channel, in the form: IDxx 

97 - amp: an alias for channel 

98 - exposure: an alias for snap 

99 

100 @param dataId[in] (dict) Dataset identifier; this must not be modified 

101 @return (dict) Transformed dataset identifier 

102 @raise RuntimeError if a value is not valid 

103 """ 

104 actualId = dataId.copy() 

105 for ccdAlias in ("ccdName", "sensorName"): 

106 if ccdAlias in actualId: 

107 ccdName = actualId[ccdAlias].upper() 

108 m = self._CcdNameRe.match(ccdName) 

109 if m is None: 

110 raise RuntimeError("Invalid value for %s: %r" % (ccdAlias, ccdName)) 

111 actualId.setdefault("raft", m.group(1)) 

112 actualId.setdefault("sensor", m.group(2)) 

113 break 

114 if "ccd" in actualId: 

115 actualId.setdefault("sensor", actualId["ccd"]) 

116 if "amp" in actualId: 

117 actualId.setdefault("channel", actualId["amp"]) 

118 elif "channel" not in actualId: 

119 for ampName in ("ampName", "channelName"): 

120 if ampName in actualId: 

121 m = re.match(r'ID(\d+)$', actualId[ampName]) 

122 channelNumber = int(m.group(1)) 

123 channelX = channelNumber % 8 

124 channelY = channelNumber // 8 

125 actualId['channel'] = str(channelX) + "," + str(channelY) 

126 break 

127 if "exposure" in actualId: 

128 actualId.setdefault("snap", actualId["exposure"]) 

129 

130 # why strip out the commas after carefully adding them? 

131 if "raft" in actualId: 

132 actualId['raft'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['raft']) 

133 if "sensor" in actualId: 

134 actualId['sensor'] = actualId['sensor'].replace(",", "") 

135 if "channel" in actualId: 

136 actualId['channel'] = re.sub(r'(\d),(\d)', r'\1\2', actualId['channel']) 

137 return actualId 

138 

139 def validate(self, dataId): 

140 for component in ("raft", "sensor", "channel"): 

141 if component not in dataId: 

142 continue 

143 val = dataId[component] 

144 if not isinstance(val, str): 

145 raise RuntimeError( 

146 "%s identifier should be type str, not %s: %r" % (component.title(), type(val), val)) 

147 if component == "sensor": 

148 if not re.search(r'^\d,\d(,[AB])?$', val): 

149 raise RuntimeError("Invalid %s identifier: %r" % (component, val)) 

150 else: 

151 if not re.search(r'^(\d),(\d)$', val): 

152 raise RuntimeError("Invalid %s identifier: %r" % (component, val)) 

153 return dataId 

154 

155 def _extractDetectorName(self, dataId): 

156 return "R:%(raft)s S:%(sensor)s" % dataId 

157 

158 def getDataId(self, visit, ccdId): 

159 """get dataId dict from visit and ccd identifier 

160 

161 @param visit 32 or 64-bit depending on camera 

162 @param ccdId detector name: same as detector.getName() 

163 """ 

164 dataId = {'visit': int(visit)} 

165 m = self._CcdNameRe.match(ccdId) 

166 if m is None: 

167 raise RuntimeError("Cannot parse ccdId=%r" % (ccdId,)) 

168 dataId['raft'] = m.group(0) 

169 dataId['sensor'] = m.group(1) 

170 return dataId 

171 

172 def _extractAmpId(self, dataId): 

173 m = re.match(r'(\d),(\d)', dataId['channel']) 

174 # Note that indices are swapped in the camera geometry vs. official 

175 # channel specification. 

176 return (self._extractDetectorName(dataId), 

177 int(m.group(1)), int(m.group(2))) 

178 

179 def _computeAmpExposureId(self, dataId): 

180 # visit, snap, raft, sensor, channel): 

181 """Compute the 64-bit (long) identifier for an amp exposure. 

182 

183 @param dataId (dict) Data identifier with visit, snap, raft, sensor, channel 

184 """ 

185 

186 pathId = self._transformId(dataId) 

187 visit = pathId['visit'] 

188 snap = pathId['snap'] 

189 raft = pathId['raft'] # "xy" e.g. "20" 

190 sensor = pathId['sensor'] # "xy" e.g. "11" 

191 channel = pathId['channel'] # "yx" e.g. "05" (NB: yx, not xy, in original comment) 

192 

193 r1, r2 = raft 

194 s1, s2 = sensor 

195 c1, c2 = channel 

196 return (visit << 13) + (snap << 12) + \ 

197 (int(r1) * 5 + int(r2)) * 160 + \ 

198 (int(s1) * 3 + int(s2)) * 16 + \ 

199 (int(c1) * 8 + int(c2)) 

200 

201 def _computeCcdExposureId(self, dataId): 

202 """Compute the 64-bit (long) identifier for a CCD exposure. 

203 

204 @param dataId (dict) Data identifier with visit, raft, sensor 

205 """ 

206 

207 pathId = self._transformId(dataId) 

208 visit = pathId['visit'] 

209 raft = pathId['raft'] # "xy" e.g. "20" 

210 sensor = pathId['sensor'] # "xy" e.g. "11" 

211 

212 r1, r2 = raft 

213 s1, s2 = sensor 

214 return (visit << 9) + \ 

215 (int(r1) * 5 + int(r2)) * 10 + \ 

216 (int(s1) * 3 + int(s2)) 

217 

218 def _computeCoaddExposureId(self, dataId, singleFilter): 

219 """Compute the 64-bit (long) identifier for a coadd. 

220 

221 @param dataId (dict) Data identifier with tract and patch. 

222 @param singleFilter (bool) True means the desired ID is for a single- 

223 filter coadd, in which case dataId 

224 must contain filter. 

225 """ 

226 tract = int(dataId['tract']) 

227 if tract < 0 or tract >= 128: 

228 raise RuntimeError('tract not in range [0,128)') 

229 patchX, patchY = list(map(int, dataId['patch'].split(','))) 

230 for p in (patchX, patchY): 

231 if p < 0 or p >= 2**13: 

232 raise RuntimeError('patch component not in range [0, 8192)') 

233 id = (tract * 2**13 + patchX) * 2**13 + patchY 

234 if singleFilter: 

235 return id * 8 + self.filterIdMap[dataId['filter']] 

236 return id 

237 

238 def _defectLookup(self, dataId, dateKey='taiObs'): 

239 """Find the defects for a given CCD. 

240 

241 Parameters 

242 ---------- 

243 dataId : `dict` 

244 Dataset identifier 

245 

246 Returns 

247 ------- 

248 `str` 

249 Path to the defects file or None if not available. 

250 """ 

251 if self.defectRegistry is None: 

252 return None 

253 if self.registry is None: 

254 raise RuntimeError("No registry for defect lookup") 

255 

256 ccdKey, ccdVal = self._getCcdKeyVal(dataId) 

257 

258 dataIdForLookup = {'visit': dataId['visit']} 

259 # .lookup will fail in a posix registry because there is no template to provide. 

260 rows = self.registry.lookup((dateKey), ('raw_visit'), dataIdForLookup) 

261 if len(rows) == 0: 

262 return None 

263 assert len(rows) == 1 

264 dayObs = rows[0][0] 

265 

266 # Lookup the defects for this CCD serial number that are valid at the exposure midpoint. 

267 rows = self.defectRegistry.executeQuery(("path",), ("defect",), 

268 [(ccdKey, "?")], 

269 ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"), 

270 (ccdVal, dayObs)) 

271 if not rows or len(rows) == 0: 

272 return None 

273 if len(rows) == 1: 

274 return os.path.join(self.defectPath, rows[0][0]) 

275 else: 

276 raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" % 

277 (ccdVal, dayObs, len(rows), ", ".join([_[0] for _ in rows]))) 

278 

279 def map_defects(self, dataId, write=False): 

280 """Map defects dataset. 

281 

282 Returns 

283 ------- 

284 `lsst.daf.butler.ButlerLocation` 

285 Minimal ButlerLocation containing just the locationList field 

286 (just enough information that bypass_defects can use it). 

287 """ 

288 defectFitsPath = self._defectLookup(dataId=dataId) 

289 if defectFitsPath is None: 

290 raise RuntimeError("No defects available for dataId=%s" % (dataId,)) 

291 

292 return dafPersist.ButlerLocation(None, None, None, defectFitsPath, 

293 dataId, self, 

294 storage=self.rootStorage) 

295 

296 def map_linearizer(self, dataId, write=False): 

297 return None 

298 

299 def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId): 

300 """Return a defect based on the butler location returned by map_defects 

301 

302 Parameters 

303 ---------- 

304 butlerLocation : `lsst.daf.persistence.ButlerLocation` 

305 locationList = path to defects FITS file 

306 dataId : `dict` 

307 Butler data ID; "ccd" must be set. 

308 

309 Note: the name "bypass_XXX" means the butler makes no attempt to 

310 convert the ButlerLocation into an object, which is what we want for 

311 now, since that conversion is a bit tricky. 

312 """ 

313 detectorName = self._extractDetectorName(dataId) 

314 defectsFitsPath = butlerLocation.locationList[0] 

315 

316 with fits.open(defectsFitsPath) as hduList: 

317 for hdu in hduList[1:]: 

318 if hdu.header["name"] != detectorName: 

319 continue 

320 

321 defectList = Defects() 

322 for data in hdu.data: 

323 bbox = geom.Box2I( 

324 geom.Point2I(int(data['x0']), int(data['y0'])), 

325 geom.Extent2I(int(data['width']), int(data['height'])), 

326 ) 

327 defectList.append(bbox) 

328 return defectList 

329 

330 raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath)) 

331 

332 _nbit_id = 30 

333 

334 def bypass_deepMergedCoaddId_bits(self, *args, **kwargs): 

335 """The number of bits used up for patch ID bits""" 

336 return 64 - self._nbit_id 

337 

338 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

339 return self._computeCoaddExposureId(dataId, False) 

340 

341 def bypass_dcrMergedCoaddId_bits(self, *args, **kwargs): 

342 """The number of bits used up for patch ID bits""" 

343 return self.bypass_deepMergedCoaddId_bits(*args, **kwargs) 

344 

345 def bypass_dcrMergedCoaddId(self, datasetType, pythonType, location, dataId): 

346 return self.bypass_deepMergedCoaddId(datasetType, pythonType, location, dataId) 

347 

348 @staticmethod 

349 def getShortCcdName(ccdId): 

350 """Convert a CCD name to a form useful as a filename 

351 

352 This LSST version converts spaces to underscores and elides colons and commas. 

353 """ 

354 return re.sub("[:,]", "", ccdId.replace(" ", "_")) 

355 

356 def _setAmpExposureId(self, propertyList, dataId): 

357 propertyList.set("Computed_ampExposureId", self._computeAmpExposureId(dataId)) 

358 return propertyList 

359 

360 def _setCcdExposureId(self, propertyList, dataId): 

361 propertyList.set("Computed_ccdExposureId", self._computeCcdExposureId(dataId)) 

362 return propertyList 

363 

364############################################################################### 

365 

366 def std_raw(self, item, dataId): 

367 md = item.getMetadata() 

368 if md.exists("VERSION") and md.getInt("VERSION") < 16952: 

369 # CRVAL is FK5 at date of observation 

370 dateObsTaiMjd = md.getScalar("TAI") 

371 dateObs = dafBase.DateTime(dateObsTaiMjd, 

372 system=dafBase.DateTime.MJD, 

373 scale=dafBase.DateTime.TAI) 

374 correctedEquinox = dateObs.get(system=dafBase.DateTime.EPOCH, 

375 scale=dafBase.DateTime.TAI) 

376 md.set("EQUINOX", correctedEquinox) 

377 md.set("RADESYS", "FK5") 

378 print("****** changing equinox to", correctedEquinox) 

379 return super(LsstSimMapper, self).std_raw(item, dataId) 

380 

381 def std_eimage(self, item, dataId): 

382 """Standardize a eimage dataset by converting it to an Exposure instead of an Image""" 

383 return self._standardizeExposure(self.exposures['eimage'], item, dataId, trimmed=True) 

384 

385 def _createInitialSkyWcs(self, exposure): 

386 """Create a SkyWcs from the header metadata. 

387 

388 PhoSim data may not have self-consistent boresight and crval/crpix 

389 values, and/or may have been written in FK5, so we just use the 

390 metadata here, and ignore VisitInfo/CameraGeom. 

391 

392 Parameters 

393 ---------- 

394 exposure : `lsst.afw.image.Exposure` 

395 The exposure to get data from, and attach the SkyWcs to. 

396 """ 

397 self._createSkyWcsFromMetadata(exposure) 

398 

399############################################################################### 

400 

401 def _getCcdKeyVal(self, dataId): 

402 """Return CCD key and value used to look a defect in the defect 

403 registry 

404 

405 The default implementation simply returns ("ccd", full detector name) 

406 """ 

407 return ("ccd", self._extractDetectorName(dataId)) 

408 

409 def bypass_ampExposureId(self, datasetType, pythonType, location, dataId): 

410 return self._computeAmpExposureId(dataId) 

411 

412 def bypass_ampExposureId_bits(self, datasetType, pythonType, location, dataId): 

413 return 45 

414 

415 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

416 return self._computeCcdExposureId(dataId) 

417 

418 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

419 return 41 

420 

421 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

422 return self._computeCoaddExposureId(dataId, True) 

423 

424 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

425 return 1 + 7 + 13*2 + 3 

426 

427 def bypass_dcrCoaddId(self, datasetType, pythonType, location, dataId): 

428 return self.bypass_deepCoaddId(datasetType, pythonType, location, dataId) 

429 

430 def bypass_dcrCoaddId_bits(self, datasetType, pythonType, location, dataId): 

431 return self.bypass_deepCoaddId_bits(datasetType, pythonType, location, dataId) 

432 

433############################################################################### 

434 

435 def add_sdqaAmp(self, dataId): 

436 ampExposureId = self._computeAmpExposureId(dataId) 

437 return {"ampExposureId": ampExposureId, "sdqaRatingScope": "AMP"} 

438 

439 def add_sdqaCcd(self, dataId): 

440 ccdExposureId = self._computeCcdExposureId(dataId) 

441 return {"ccdExposureId": ccdExposureId, "sdqaRatingScope": "CCD"} 

442 

443############################################################################### 

444 

445 

446for dsType in ("raw", "postISR"): 

447 setattr(LsstSimMapper, "std_" + dsType + "_md", 447 ↛ exitline 447 didn't jump to the function exit

448 lambda self, item, dataId: self._setAmpExposureId(item, dataId)) 

449for dsType in ("eimage", "postISRCCD", "visitim", "calexp", "calsnap"): 

450 setattr(LsstSimMapper, "std_" + dsType + "_md", 450 ↛ exitline 450 didn't jump to the function exit

451 lambda self, item, dataId: self._setCcdExposureId(item, dataId))