Coverage for python/lsst/validate/drp/util.py: 21%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

112 statements  

1# LSST Data Management System 

2# Copyright 2008-2016 AURA/LSST. 

3# 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# 

7# This program is free software: you can redistribute it and/or modify 

8# it under the terms of the GNU General Public License as published by 

9# the Free Software Foundation, either version 3 of the License, or 

10# (at your option) any later version. 

11# 

12# This program is distributed in the hope that it will be useful, 

13# but WITHOUT ANY WARRANTY; without even the implied warranty of 

14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

15# GNU General Public License for more details. 

16# 

17# You should have received a copy of the LSST License Statement and 

18# the GNU General Public License along with this program. If not, 

19# see <https://www.lsstcorp.org/LegalNotices/>. 

20"""Miscellaneous functions to support lsst.validate.drp.""" 

21 

22import os 

23 

24import numpy as np 

25 

26import yaml 

27 

28import lsst.daf.persistence as dafPersist 

29import lsst.pipe.base as pipeBase 

30import lsst.geom as geom 

31 

32 

33def ellipticity_from_cat(cat, slot_shape='slot_Shape'): 

34 """Calculate the ellipticity of the Shapes in a catalog from the 2nd moments. 

35 

36 Parameters 

37 ---------- 

38 cat : `lsst.afw.table.BaseCatalog` 

39 A catalog with 'slot_Shape' defined and '_xx', '_xy', '_yy' 

40 entries for the target of 'slot_Shape'. 

41 E.g., 'slot_shape' defined as 'base_SdssShape' 

42 And 'base_SdssShape_xx', 'base_SdssShape_xy', 'base_SdssShape_yy' defined. 

43 slot_shape : str, optional 

44 Specify what slot shape requested. Intended use is to get the PSF shape 

45 estimates by specifying 'slot_shape=slot_PsfShape' 

46 instead of the default 'slot_shape=slot_Shape'. 

47 

48 Returns 

49 ------- 

50 e, e1, e2 : complex, float, float 

51 Complex ellipticity, real part, imaginary part 

52 """ 

53 i_xx, i_xy, i_yy = cat.get(slot_shape+'_xx'), cat.get(slot_shape+'_xy'), cat.get(slot_shape+'_yy') 

54 return ellipticity(i_xx, i_xy, i_yy) 

55 

56 

57def ellipticity_from_shape(shape): 

58 """Calculate the ellipticty of shape from its moments. 

59 

60 Parameters 

61 ---------- 

62 shape : `lsst.afw.geom.ellipses.Quadrupole` 

63 The LSST generic shape object returned by psf.computeShape() 

64 or source.getShape() for a specific source. 

65 Imeplementation: just needs to have .getIxx, .getIxy, .getIyy methods 

66 that each return a float describing the respective second moments. 

67 

68 Returns 

69 ------- 

70 e, e1, e2 : complex, float, float 

71 Complex ellipticity, real part, imaginary part 

72 """ 

73 i_xx, i_xy, i_yy = shape.getIxx(), shape.getIxy(), shape.getIyy() 

74 return ellipticity(i_xx, i_xy, i_yy) 

75 

76 

77def ellipticity(i_xx, i_xy, i_yy): 

78 """Calculate ellipticity from second moments. 

79 

80 Parameters 

81 ---------- 

82 i_xx : float or `numpy.array` 

83 i_xy : float or `numpy.array` 

84 i_yy : float or `numpy.array` 

85 

86 Returns 

87 ------- 

88 e, e1, e2 : (float, float, float) or (numpy.array, numpy.array, numpy.array) 

89 Complex ellipticity, real component, imaginary component 

90 """ 

91 e = (i_xx - i_yy + 2j*i_xy) / (i_xx + i_yy) 

92 e1 = np.real(e) 

93 e2 = np.imag(e) 

94 return e, e1, e2 

95 

96 

97def averageRaDec(ra, dec): 

98 """Calculate average RA, Dec from input lists using spherical geometry. 

99 

100 Parameters 

101 ---------- 

102 ra : `list` [`float`] 

103 RA in [radians] 

104 dec : `list` [`float`] 

105 Dec in [radians] 

106 

107 Returns 

108 ------- 

109 float, float 

110 meanRa, meanDec -- Tuple of average RA, Dec [radians] 

111 """ 

112 assert(len(ra) == len(dec)) 

113 

114 angleRa = [geom.Angle(r, geom.radians) for r in ra] 

115 angleDec = [geom.Angle(d, geom.radians) for d in dec] 

116 coords = [geom.SpherePoint(ar, ad, geom.radians) for (ar, ad) in zip(angleRa, angleDec)] 

117 

118 meanRa, meanDec = geom.averageSpherePoint(coords) 

119 

120 return meanRa.asRadians(), meanDec.asRadians() 

121 

122 

123def averageRaDecFromCat(cat): 

124 """Calculate the average right ascension and declination from a catalog. 

125 

126 Convenience wrapper around averageRaDec 

127 

128 Parameters 

129 ---------- 

130 cat : collection 

131 Object with .get method for 'coord_ra', 'coord_dec' that returns radians. 

132 

133 Returns 

134 ------- 

135 ra_mean : `float` 

136 Mean RA in radians. 

137 dec_mean : `float` 

138 Mean Dec in radians. 

139 """ 

140 return averageRaDec(cat.get('coord_ra'), cat.get('coord_dec')) 

141 

142 

143def positionRms(ra_mean, dec_mean, ra, dec): 

144 """Calculate the RMS between an array of coordinates and a reference (mean) position. 

145 

146 Parameters 

147 ---------- 

148 ra_mean : `float` 

149 Mean RA in radians. 

150 dec_mean : `float` 

151 Mean Dec in radians. 

152 ra : `numpy.array` [`float`] 

153 Array of RA in radians. 

154 dec : `numpy.array` [`float`] 

155 Array of Dec in radians. 

156 

157 Returns 

158 ------- 

159 pos_rms : `float` 

160 RMS scatter of positions in milliarcseconds. 

161 

162 Notes 

163 ----- 

164 The RMS of a single-element array will be returned as 0. 

165 The RMS of an empty array will be returned as NaN. 

166 """ 

167 separations = sphDist(ra_mean, dec_mean, ra, dec) 

168 # Note we don't want `np.std` of separations, which would give us the 

169 # std around the average of separations. 

170 # We've already taken out the average, 

171 # so we want the sqrt of the mean of the squares. 

172 pos_rms_rad = np.sqrt(np.mean(separations**2)) # radians 

173 pos_rms_mas = geom.radToMas(pos_rms_rad) # milliarcsec 

174 

175 return pos_rms_mas 

176 

177 

178def positionRmsFromCat(cat): 

179 """Calculate the RMS for RA, Dec for a set of observations an object. 

180 

181 Parameters 

182 ---------- 

183 cat : collection 

184 Object with .get method for 'coord_ra', 'coord_dec' that returns radians. 

185 

186 Returns 

187 ------- 

188 pos_rms : `float` 

189 RMS scatter of positions in milliarcseconds. 

190 """ 

191 ra_avg, dec_avg = averageRaDecFromCat(cat) 

192 ra, dec = cat.get('coord_ra'), cat.get('coord_dec') 

193 return positionRms(ra_avg, dec_avg, ra, dec) 

194 

195 

196def sphDist(ra_mean, dec_mean, ra, dec): 

197 """Calculate distance on the surface of a unit sphere. 

198 

199 Parameters 

200 ---------- 

201 ra_mean : `float` 

202 Mean RA in radians. 

203 dec_mean : `float` 

204 Mean Dec in radians. 

205 ra : `numpy.array` [`float`] 

206 Array of RA in radians. 

207 dec : `numpy.array` [`float`] 

208 Array of Dec in radians. 

209 

210 Notes 

211 ----- 

212 Uses the Haversine formula to preserve accuracy at small angles. 

213 

214 Law of cosines approach doesn't work well for the typically very small 

215 differences that we're looking at here. 

216 """ 

217 # Haversine 

218 dra = ra - ra_mean 

219 ddec = dec - dec_mean 

220 a = np.square(np.sin(ddec/2)) + \ 

221 np.cos(dec_mean)*np.cos(dec)*np.square(np.sin(dra/2)) 

222 dist = 2 * np.arcsin(np.sqrt(a)) 

223 

224 # This is what the law of cosines would look like 

225 # dist = np.arccos(np.sin(dec1)*np.sin(dec2) + np.cos(dec1)*np.cos(dec2)*np.cos(ra1 - ra2)) 

226 

227 # This will also work, but must run separately for each element 

228 # whereas the numpy version will run on either scalars or arrays: 

229 # sp1 = geom.SpherePoint(ra1, dec1, geom.radians) 

230 # sp2 = geom.SpherePoint(ra2, dec2, geom.radians) 

231 # return sp1.separation(sp2).asRadians() 

232 

233 return dist 

234 

235 

236def averageRaFromCat(cat): 

237 """Compute the average right ascension from a catalog of measurements. 

238 

239 This function is used as an aggregate function to extract just RA 

240 from lsst.validate.drp.matchreduce.build_matched_dataset 

241 

242 The actual computation involves both RA and Dec. 

243 

244 The intent is to use this for a set of measurements of the same source 

245 but that's neither enforced nor required. 

246 

247 Parameters 

248 ---------- 

249 cat : collection 

250 Object with .get method for 'coord_ra', 'coord_dec' that returns radians. 

251 

252 Returns 

253 ------- 

254 ra_mean : `float` 

255 Mean RA in radians. 

256 """ 

257 meanRa, meanDec = averageRaDecFromCat(cat) 

258 return meanRa 

259 

260 

261def averageDecFromCat(cat): 

262 """Compute the average declination from a catalog of measurements. 

263 

264 This function is used as an aggregate function to extract just declination 

265 from lsst.validate.drp.matchreduce.build_matched_dataset 

266 

267 The actual computation involves both RA and Dec. 

268 

269 The intent is to use this for a set of measurements of the same source 

270 but that's neither enforced nor required. 

271 

272 Parameters 

273 ---------- 

274 cat : collection 

275 Object with .get method for 'coord_ra', 'coord_dec' that returns radians. 

276 

277 Returns 

278 ------- 

279 dec_mean : `float` 

280 Mean Dec in radians. 

281 """ 

282 meanRa, meanDec = averageRaDecFromCat(cat) 

283 return meanDec 

284 

285 

286def medianEllipticityResidualsFromCat(cat): 

287 """Compute the median ellipticty residuals from a catalog of measurements. 

288 

289 This function is used as an aggregate function to extract just declination 

290 from lsst.validate.drp.matchreduce.build_matched_dataset 

291 

292 The intent is to use this for a set of measurements of the same source 

293 but that's neither enforced nor required. 

294 

295 Parameters 

296 ---------- 

297 cat : collection 

298 Object with .get method for 'e1', 'e2' that returns radians. 

299 

300 Returns 

301 ------- 

302 e1_median : `float` 

303 Median real ellipticity residual. 

304 e2_median : `float` 

305 Median imaginary ellipticity residual. 

306 """ 

307 e1_median = np.median(cat.get('e1') - cat.get('psf_e1')) 

308 e2_median = np.median(cat.get('e2') - cat.get('psf_e2')) 

309 return e1_median, e2_median 

310 

311 

312def medianEllipticity1ResidualsFromCat(cat): 

313 """Compute the median real ellipticty residuals from a catalog of measurements. 

314 

315 Parameters 

316 ---------- 

317 cat : collection 

318 Object with .get method for 'e1', 'psf_e1' that returns radians. 

319 

320 Returns 

321 ------- 

322 e1_median : `float` 

323 Median imaginary ellipticity residual. 

324 """ 

325 e1_median = np.median(cat.get('e1') - cat.get('psf_e1')) 

326 return e1_median 

327 

328 

329def medianEllipticity2ResidualsFromCat(cat): 

330 """Compute the median imaginary ellipticty residuals from a catalog of measurements. 

331 

332 Parameters 

333 ---------- 

334 cat : collection 

335 Object with .get method for 'e2', 'psf_e2' that returns radians. 

336 

337 Returns 

338 ------- 

339 e2_median : `float` 

340 Median imaginary ellipticity residual. 

341 """ 

342 e2_median = np.median(cat.get('e2') - cat.get('psf_e2')) 

343 return e2_median 

344 

345 

346def getCcdKeyName(dataId): 

347 """Return the key in a dataId that's referring to the CCD or moral equivalent. 

348 

349 Parameters 

350 ---------- 

351 dataId : `dict` 

352 A dictionary that will be searched for a key that matches 

353 an entry in the hardcoded list of possible names for the CCD field. 

354 

355 Returns 

356 ------- 

357 name : `str` 

358 The name of the key. 

359 

360 Notes 

361 ----- 

362 Motivation: Different camera mappings use different keys to indicate 

363 the different amps/ccds in the same exposure. This function looks 

364 through the reference dataId to locate a field that could be the one. 

365 """ 

366 possibleCcdFieldNames = ['detector', 'ccd', 'ccdnum', 'camcol', 'sensor'] 

367 

368 for name in possibleCcdFieldNames: 

369 if name in dataId: 

370 return name 

371 else: 

372 return 'ccd' 

373 

374 

375def raftSensorToInt(visitId): 

376 """Construct an int that encodes raft, sensor coordinates. 

377 

378 Parameters 

379 ---------- 

380 visitId : `dict` 

381 A dictionary containing raft and sensor keys. 

382 

383 Returns 

384 ------- 

385 id : `int` 

386 The integer id of the raft/sensor. 

387 

388 Examples 

389 -------- 

390 >>> vId = {'filter': 'y', 'raft': '2,2', 'sensor': '1,2', 'visit': 307} 

391 >>> raftSensorToInt(vId) 

392 2212 

393 """ 

394 def pair_to_int(tuple_string): 

395 x, y = tuple_string.split(',') 

396 return 10 * int(x) + 1 * int(y) 

397 

398 raft_int = pair_to_int(visitId['raft']) 

399 sensor_int = pair_to_int(visitId['sensor']) 

400 return 100*raft_int + sensor_int 

401 

402 

403def repoNameToPrefix(repo): 

404 """Generate a base prefix for plots based on the repo name. 

405 

406 Parameters 

407 ---------- 

408 repo : `str` 

409 The repo path. 

410 

411 Returns 

412 ------- 

413 repo_base : `str` 

414 The base prefix for the repo. 

415 

416 Examples 

417 -------- 

418 >>> repoNameToPrefix('a/b/c') 

419 'a_b_c' 

420 >>> repoNameToPrefix('/bar/foo/') 

421 'bar_foo' 

422 >>> repoNameToPrefix('CFHT/output') 

423 'CFHT_output' 

424 >>> repoNameToPrefix('./CFHT/output') 

425 'CFHT_output' 

426 >>> repoNameToPrefix('.a/CFHT/output') 

427 'a_CFHT_output' 

428 >>> repoNameToPrefix('bar/foo.json') 

429 'bar_foo' 

430 """ 

431 

432 repo_base, ext = os.path.splitext(repo) 

433 return repo_base.lstrip('.').strip(os.sep).replace(os.sep, "_") 

434 

435 

436def discoverDataIds(repo, **kwargs): 

437 """Retrieve a list of all dataIds in a repo. 

438 

439 Parameters 

440 ---------- 

441 repo : `str` 

442 Path of a repository with 'src' entries. 

443 

444 Returns 

445 ------- 

446 dataIds : `list` 

447 dataIds in the butler that exist. 

448 

449 Notes 

450 ----- 

451 May consider making this an iterator if large N becomes important. 

452 However, will likely need to know things like, "all unique filters" 

453 of a data set anyway, so would need to go through chain at least once. 

454 """ 

455 butler = dafPersist.Butler(repo) 

456 thisSubset = butler.subset(datasetType='src', **kwargs) 

457 # This totally works, but would be better to do this as a TaskRunner? 

458 dataIds = [dr.dataId for dr in thisSubset 

459 if dr.datasetExists(datasetType='src') and dr.datasetExists(datasetType='calexp')] 

460 # Make sure we have the filter information 

461 for dId in dataIds: 

462 response = butler.queryMetadata(datasetType='src', format=['filter'], dataId=dId) 

463 filterForThisDataId = response[0] 

464 dId['filter'] = filterForThisDataId 

465 

466 return dataIds 

467 

468 

469def loadParameters(configFile): 

470 """Load configuration parameters from a yaml file. 

471 

472 Parameters 

473 ---------- 

474 configFile : `str` 

475 YAML file that stores visit, filter, ccd, 

476 good_mag_limit, medianAstromscatterRef, medianPhotoscatterRef, matchRef 

477 and other parameters 

478 

479 Returns 

480 ------- 

481 pipeStruct: `lsst.pipe.base.Struct` 

482 Struct with configuration parameters. 

483 """ 

484 with open(configFile, mode='r') as stream: 

485 data = yaml.safe_load(stream) 

486 

487 return pipeBase.Struct(**data) 

488 

489 

490def loadDataIdsAndParameters(configFile): 

491 """Load data IDs, magnitude range, and expected metrics from a yaml file. 

492 

493 Parameters 

494 ---------- 

495 configFile : `str` 

496 YAML file that stores visit, filter, ccd, 

497 and additional configuration parameters such as 

498 brightSnrMin, medianAstromscatterRef, medianPhotoscatterRef, matchRef 

499 

500 Returns 

501 ------- 

502 pipeStruct: `lsst.pipe.base.Struct` 

503 Struct with attributes of dataIds - dict and configuration parameters. 

504 """ 

505 parameters = loadParameters(configFile).getDict() 

506 

507 ccdKeyName = getCcdKeyName(parameters) 

508 try: 

509 dataIds = constructDataIds(parameters['filter'], parameters['visits'], 

510 parameters[ccdKeyName], ccdKeyName) 

511 for key in ['filter', 'visits', ccdKeyName]: 

512 del parameters[key] 

513 

514 except KeyError: 

515 # If the above parameters are not in the `parameters` dict, 

516 # presumably because they were not in the configFile 

517 # then we return no dataIds. 

518 dataIds = [] 

519 

520 return pipeBase.Struct(dataIds=dataIds, **parameters) 

521 

522 

523def constructDataIds(filters, visits, ccds, ccdKeyName='ccd'): 

524 """Returns a list of dataIds consisting of every combination of visit & ccd for each filter. 

525 

526 Parameters 

527 ---------- 

528 filters : `str` or `list` [`str`] 

529 If str, will be interpreted as one filter to be applied to all visits. 

530 visits : `list` [`int`] 

531 ccds : `list` [`int`] 

532 ccdKeyName : `str`, optional 

533 Name to distinguish different parts of a focal plane. 

534 Generally 'ccd', but might be 'ccdnum', or 'amp', or 'ccdamp'. 

535 Refer to your `obs_*/policy/*Mapper.paf`. 

536 

537 Returns 

538 ------- 

539 dataIds : `list` 

540 dataIDs suitable to be used with the LSST Butler. 

541 

542 Examples 

543 -------- 

544 >>> dataIds = constructDataIds('r', [100, 200], [10, 11, 12]) 

545 >>> for dataId in dataIds: print(dataId) 

546 {'filter': 'r', 'visit': 100, 'ccd': 10} 

547 {'filter': 'r', 'visit': 100, 'ccd': 11} 

548 {'filter': 'r', 'visit': 100, 'ccd': 12} 

549 {'filter': 'r', 'visit': 200, 'ccd': 10} 

550 {'filter': 'r', 'visit': 200, 'ccd': 11} 

551 {'filter': 'r', 'visit': 200, 'ccd': 12} 

552 """ 

553 if isinstance(filters, str): 

554 filters = [filters for _ in visits] 

555 

556 assert len(filters) == len(visits) 

557 dataIds = [{'filter': f, 'visit': v, ccdKeyName: c} 

558 for (f, v) in zip(filters, visits) 

559 for c in ccds] 

560 

561 return dataIds 

562 

563 

564def loadRunList(configFile): 

565 """Load run list from a YAML file. 

566 

567 Parameters 

568 ---------- 

569 configFile : `str` 

570 YAML file that stores visit, filter, ccd, 

571 

572 Returns 

573 ------- 

574 runList : `list` 

575 run list lines. 

576 

577 Examples 

578 -------- 

579 An example YAML file would include entries of (for some CFHT data) 

580 visits: [849375, 850587] 

581 filter: 'r' 

582 ccd: [12, 13, 14, 21, 22, 23] 

583 or (for some DECam data) 

584 visits: [176837, 176846] 

585 filter: 'z' 

586 ccdnum: [10, 11, 12, 13, 14, 15, 16, 17, 18] 

587 

588 Note 'ccd' for CFHT and 'ccdnum' for DECam. These entries will be used to build 

589 dataIds, so these fields should be as the camera mapping defines them. 

590 

591 `visits` and `ccd` (or `ccdnum`) must be lists, even if there's only one element. 

592 """ 

593 stream = open(configFile, mode='r') 

594 data = yaml.safe_load(stream) 

595 

596 ccdKeyName = getCcdKeyName(data) 

597 runList = constructRunList(data['visits'], data[ccdKeyName], ccdKeyName=ccdKeyName) 

598 

599 return runList 

600 

601 

602def constructRunList(visits, ccds, ccdKeyName='ccd'): 

603 """Construct a comprehensive runList for processCcd.py. 

604 

605 Parameters 

606 ---------- 

607 visits : `list` of `int` 

608 The desired visits. 

609 ccds : `list` of `int` 

610 The desired ccds. 

611 

612 Returns 

613 ------- 

614 `list` 

615 list of strings suitable to be used with the LSST Butler. 

616 

617 Examples 

618 -------- 

619 >>> runList = constructRunList([100, 200], [10, 11, 12]) 

620 >>> print(runList) 

621 ['--id visit=100 ccd=10^11^12', '--id visit=200 ccd=10^11^12'] 

622 >>> runList = constructRunList([100, 200], [10, 11, 12], ccdKeyName='ccdnum') 

623 >>> print(runList) 

624 ['--id visit=100 ccdnum=10^11^12', '--id visit=200 ccdnum=10^11^12'] 

625 

626 Notes 

627 ----- 

628 The LSST parsing convention is to use '^' as list separators 

629 for arguments to `--id`. While surprising, this convention 

630 allows for CCD names to include ','. E.g., 'R1,2'. 

631 Currently ignores `filter` because `visit` should be unique w.r.t filter. 

632 """ 

633 runList = ["--id visit=%d %s=%s" % (v, ccdKeyName, "^".join([str(c) for c in ccds])) 

634 for v in visits] 

635 

636 return runList