Coverage for python/lsst/summit/utils/utils.py: 21%

216 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-12 11:07 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import numpy as np 

24from scipy.ndimage.filters import gaussian_filter 

25import lsst.afw.detection as afwDetect 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.pipe.base as pipeBase 

29import lsst.utils.packages as packageUtils 

30from lsst.daf.butler.cli.cliLog import CliLog 

31import datetime 

32from dateutil.tz import gettz 

33 

34from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER 

35from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION 

36 

37from astro_metadata_translator import ObservationInfo 

38from astropy.coordinates import SkyCoord, AltAz 

39from astropy.coordinates.earth import EarthLocation 

40import astropy.units as u 

41 

42__all__ = ["SIGMATOFWHM", 

43 "FWHMTOSIGMA", 

44 "EFD_CLIENT_MISSING_MSG", 

45 "GOOGLE_CLOUD_MISSING_MSG", 

46 "AUXTEL_LOCATION", 

47 "countPixels", 

48 "quickSmooth", 

49 "argMax2d", 

50 "getImageStats", 

51 "detectObjectsInExp", 

52 "humanNameForCelestialObject", 

53 "getFocusFromHeader", 

54 "dayObsIntToString", 

55 "dayObsSeqNumToVisitId", 

56 "setupLogging", 

57 "getCurrentDayObs_datetime", 

58 "getCurrentDayObs_int", 

59 "getCurrentDayObs_humanStr", 

60 "getSite", 

61 "getExpPositionOffset", 

62 ] 

63 

64 

65SIGMATOFWHM = 2.0*np.sqrt(2.0*np.log(2.0)) 

66FWHMTOSIGMA = 1/SIGMATOFWHM 

67 

68EFD_CLIENT_MISSING_MSG = ('ImportError: lsst_efd_client not found. Please install with:\n' 

69 ' pip install lsst-efd-client') 

70 

71GOOGLE_CLOUD_MISSING_MSG = ('ImportError: Google cloud storage not found. Please install with:\n' 

72 ' pip install google-cloud-storage') 

73 

74 

75def countPixels(maskedImage, maskPlane): 

76 """Count the number of pixels in an image with a given mask bit set. 

77 

78 Parameters 

79 ---------- 

80 maskedImage : `lsst.afw.image.MaskedImage` 

81 The masked image, 

82 maskPlane : `str` 

83 The name of the bitmask. 

84 

85 Returns 

86 ------- 

87 count : `int`` 

88 The number of pixels in with the selected mask bit 

89 """ 

90 bit = maskedImage.mask.getPlaneBitMask(maskPlane) 

91 return len(np.where(np.bitwise_and(maskedImage.mask.array, bit))[0]) 

92 

93 

94def quickSmooth(data, sigma=2): 

95 """Perform a quick smoothing of the image. 

96 

97 Not to be used for scientific purposes, but improves the stretch and 

98 visual rendering of low SNR against the sky background in cutouts. 

99 

100 Parameters 

101 ---------- 

102 data : `np.array` 

103 The image data to smooth 

104 sigma : `float`, optional 

105 The size of the smoothing kernel. 

106 

107 Returns 

108 ------- 

109 smoothData : `np.array` 

110 The smoothed data 

111 """ 

112 kernel = [sigma, sigma] 

113 smoothData = gaussian_filter(data, kernel, mode='constant') 

114 return smoothData 

115 

116 

117def argMax2d(array): 

118 """Get the index of the max value of an array and whether it's unique. 

119 

120 If its not unique, returns a list of the other locations containing the 

121 maximum value, e.g. returns 

122 

123 (12, 34), False, [(56,78), (910, 1112)] 

124 

125 Parameters 

126 ---------- 

127 array : `np.array` 

128 The data 

129 

130 Returns 

131 ------- 

132 maxLocation : `tuple` 

133 The coords of the first instance of the max value 

134 unique : `bool` 

135 Whether it's the only location 

136 otherLocations : `list` of `tuple` 

137 List of the other max values' locations, empty if False 

138 """ 

139 uniqueMaximum = False 

140 maxCoords = np.where(array == np.max(array)) 

141 maxCoords = [coord for coord in zip(*maxCoords)] # list of coords as tuples 

142 if len(maxCoords) == 1: # single unambiguous value 

143 uniqueMaximum = True 

144 

145 return maxCoords[0], uniqueMaximum, maxCoords[1:] 

146 

147 

148def dayObsIntToString(dayObs): 

149 """Convert an integer dayObs to a dash-delimited string. 

150 

151 e.g. convert the hard to read 20210101 to 2021-01-01 

152 

153 Parameters 

154 ---------- 

155 dayObs : `int` 

156 The dayObs. 

157 

158 Returns 

159 ------- 

160 dayObs : `str` 

161 The dayObs as a string. 

162 """ 

163 assert isinstance(dayObs, int) 

164 dStr = str(dayObs) 

165 assert len(dStr) == 8 

166 return '-'.join([dStr[0:4], dStr[4:6], dStr[6:8]]) 

167 

168 

169def dayObsSeqNumToVisitId(dayObs, seqNum): 

170 """Get the visit id for a given dayObs/seqNum. 

171 

172 Parameters 

173 ---------- 

174 dayObs : `int` 

175 The dayObs. 

176 seqNum : `int` 

177 The seqNum. 

178 

179 Returns 

180 ------- 

181 visitId : `int` 

182 The visitId. 

183 

184 Notes 

185 ----- 

186 TODO: Remove this horrible hack once DM-30948 makes this possible 

187 programatically/via the butler. 

188 """ 

189 if dayObs < 19700101 or dayObs > 35000101: 

190 raise ValueError(f'dayObs value {dayObs} outside plausible range') 

191 return int(f"{dayObs}{seqNum:05}") 

192 

193 

194def getImageStats(exp): 

195 """Calculate a grab-bag of stats for an image. Must remain fast. 

196 

197 Parameters 

198 ---------- 

199 exp : `lsst.afw.image.Exposure` 

200 The input exposure. 

201 

202 Returns 

203 ------- 

204 stats : `lsst.pipe.base.Struct` 

205 A container with attributes containing measurements and statistics 

206 for the image. 

207 """ 

208 result = pipeBase.Struct() 

209 

210 info = exp.getInfo() 

211 vi = info.getVisitInfo() 

212 expTime = vi.getExposureTime() 

213 md = exp.getMetadata() 

214 

215 obj = vi.object 

216 mjd = vi.getDate().get() 

217 result.object = obj 

218 result.mjd = mjd 

219 

220 fullFilterString = info.getFilterLabel().physicalLabel 

221 filt = fullFilterString.split(FILTER_DELIMITER)[0] 

222 grating = fullFilterString.split(FILTER_DELIMITER)[1] 

223 

224 airmass = vi.getBoresightAirmass() 

225 rotangle = vi.getBoresightRotAngle().asDegrees() 

226 

227 azAlt = vi.getBoresightAzAlt() 

228 az = azAlt[0].asDegrees() 

229 el = azAlt[1].asDegrees() 

230 

231 result.expTime = expTime 

232 result.filter = filt 

233 result.grating = grating 

234 result.airmass = airmass 

235 result.rotangle = rotangle 

236 result.az = az 

237 result.el = el 

238 result.focus = md.get('FOCUSZ') 

239 

240 data = exp.image.array 

241 result.maxValue = np.max(data) 

242 

243 peak, uniquePeak, otherPeaks = argMax2d(data) 

244 result.maxPixelLocation = peak 

245 result.multipleMaxPixels = uniquePeak 

246 

247 result.nBadPixels = countPixels(exp.maskedImage, 'BAD') 

248 result.nSatPixels = countPixels(exp.maskedImage, 'SAT') 

249 result.percentile99 = np.percentile(data, 99) 

250 result.percentile9999 = np.percentile(data, 99.99) 

251 

252 sctrl = afwMath.StatisticsControl() 

253 sctrl.setNumSigmaClip(5) 

254 sctrl.setNumIter(2) 

255 statTypes = afwMath.MEANCLIP | afwMath.STDEVCLIP 

256 stats = afwMath.makeStatistics(exp.maskedImage, statTypes, sctrl) 

257 std, stderr = stats.getResult(afwMath.STDEVCLIP) 

258 mean, meanerr = stats.getResult(afwMath.MEANCLIP) 

259 

260 result.clippedMean = mean 

261 result.clippedStddev = std 

262 

263 return result 

264 

265 

266def detectObjectsInExp(exp, nSigma=10, nPixMin=10, grow=0): 

267 """Quick and dirty object detection for an expsure. 

268 

269 Return the footPrintSet for the objects in a preferably-postISR exposure. 

270 

271 Parameters 

272 ---------- 

273 exp : `lsst.afw.image.Exposure` 

274 The exposure to detect objects in. 

275 nSigma : `float` 

276 The number of sigma for detection. 

277 nPixMin : `int` 

278 The minimum number of pixels in an object for detection. 

279 grow : `int` 

280 The number of pixels to grow the footprint by after detection. 

281 

282 Returns 

283 ------- 

284 footPrintSet : `lsst.afw.detection.FootprintSet` 

285 The set of footprints in the image. 

286 """ 

287 median = np.nanmedian(exp.image.array) 

288 exp.image -= median 

289 

290 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV) 

291 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin) 

292 if grow > 0: 

293 isotropic = True 

294 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic) 

295 

296 exp.image += median # add back in to leave background unchanged 

297 return footPrintSet 

298 

299 

300def humanNameForCelestialObject(objName): 

301 """Returns a list of all human names for obj, or [] if none are found. 

302 

303 Parameters 

304 ---------- 

305 objName : `str` 

306 The/a name of the object. 

307 

308 Returns 

309 ------- 

310 names : `list` of `str` 

311 The names found for the object 

312 """ 

313 from astroquery.simbad import Simbad 

314 results = [] 

315 try: 

316 simbadResult = Simbad.query_objectids(objName) 

317 for row in simbadResult: 

318 if row['ID'].startswith('NAME'): 

319 results.append(row['ID'].replace('NAME ', '')) 

320 return results 

321 except Exception: 

322 return [] # same behavior as for found but un-named objects 

323 

324 

325def _getAltAzZenithsFromSeqNum(butler, dayObs, seqNumList): 

326 """Get the alt, az and zenith angle for the seqNums of a given dayObs. 

327 

328 Parameters 

329 ---------- 

330 butler : `lsst.daf.butler.Butler` 

331 The butler to query. 

332 dayObs : `int` 

333 The dayObs. 

334 seqNumList : `list` of `int` 

335 The seqNums for which to return the alt, az and zenith 

336 

337 Returns 

338 ------- 

339 azimuths : `list` of `float` 

340 List of the azimuths for each seqNum 

341 elevations : `list` of `float` 

342 List of the elevations for each seqNum 

343 zeniths : `list` of `float` 

344 List of the zenith angles for each seqNum 

345 """ 

346 azimuths, elevations, zeniths = [], [], [] 

347 for seqNum in seqNumList: 

348 md = butler.get('raw.metadata', day_obs=dayObs, seq_num=seqNum, detector=0) 

349 obsInfo = ObservationInfo(md) 

350 alt = obsInfo.altaz_begin.alt.value 

351 az = obsInfo.altaz_begin.az.value 

352 elevations.append(alt) 

353 zeniths.append(90-alt) 

354 azimuths.append(az) 

355 return azimuths, elevations, zeniths 

356 

357 

358def getFocusFromHeader(exp): 

359 """Get the raw focus value from the header. 

360 

361 Parameters 

362 ---------- 

363 exp : `lsst.afw.image.exposure` 

364 The exposure. 

365 

366 Returns 

367 ------- 

368 focus : `float` or `None` 

369 The focus value if found, else ``None``. 

370 """ 

371 md = exp.getMetadata() 

372 if 'FOCUSZ' in md: 

373 return md['FOCUSZ'] 

374 return None 

375 

376 

377def checkStackSetup(): 

378 """Check which weekly tag is being used and which local packages are setup. 

379 

380 Designed primarily for use in notbooks/observing, this prints the weekly 

381 tag(s) are setup for lsst_distrib, and lists any locally setup packages and 

382 the path to each. 

383 

384 Notes 

385 ----- 

386 Uses print() instead of logger messages as this should simply print them 

387 without being vulnerable to any log messages potentially being diverted. 

388 """ 

389 packages = packageUtils.getEnvironmentPackages(include_all=True) 

390 

391 lsstDistribHashAndTags = packages['lsst_distrib'] # looks something like 'g4eae7cb9+1418867f (w_2022_13)' 

392 lsstDistribTags = lsstDistribHashAndTags.split()[1] 

393 if len(lsstDistribTags.split()) == 1: 

394 tag = lsstDistribTags.replace('(', '') 

395 tag = tag.replace(')', '') 

396 print(f"You are running {tag} of lsst_distrib") 

397 else: # multiple weekly tags found for lsst_distrib! 

398 print(f'The version of lsst_distrib you have is compatible with: {lsstDistribTags}') 

399 

400 localPackages = [] 

401 localPaths = [] 

402 for package, tags in packages.items(): 

403 if tags.startswith('LOCAL:'): 

404 path = tags.split('LOCAL:')[1] 

405 path = path.split('@')[0] # don't need the git SHA etc 

406 localPaths.append(path) 

407 localPackages.append(package) 

408 

409 if localPackages: 

410 print("\nLocally setup packages:") 

411 print("-----------------------") 

412 maxLen = max(len(package) for package in localPackages) 

413 for package, path in zip(localPackages, localPaths): 

414 print(f"{package:<{maxLen}s} at {path}") 

415 else: 

416 print("\nNo locally setup packages (using a vanilla stack)") 

417 

418 

419def setupLogging(longlog=False): 

420 """Setup logging in the same way as one would get from pipetask run. 

421 

422 Code that isn't run through the butler CLI defaults to WARNING level 

423 messages and no logger names. This sets the behaviour to follow whatever 

424 the pipeline default is, currently 

425 <logger_name> <level>: <message> e.g. 

426 lsst.isr INFO: Masking defects. 

427 """ 

428 CliLog.initLog(longlog=longlog) 

429 

430 

431def getCurrentDayObs_datetime(): 

432 """Get the current day_obs - the observatory rolls the date over at UTC-12 

433 

434 Returned as datetime.date(2022, 4, 28) 

435 """ 

436 utc = gettz("UTC") 

437 nowUtc = datetime.datetime.now().astimezone(utc) 

438 offset = datetime.timedelta(hours=-12) 

439 dayObs = (nowUtc + offset).date() 

440 return dayObs 

441 

442 

443def getCurrentDayObs_int(): 

444 """Return the current dayObs as an int in the form 20220428 

445 """ 

446 return int(getCurrentDayObs_datetime().strftime("%Y%m%d")) 

447 

448 

449def getCurrentDayObs_humanStr(): 

450 """Return the current dayObs as a string in the form '2022-04-28' 

451 """ 

452 return dayObsIntToString(getCurrentDayObs_int()) 

453 

454 

455def getSite(): 

456 """Returns where the code is running. 

457 

458 Returns 

459 ------- 

460 location : `str` 

461 One of ['tucson', 'summit', 'base', 'staff-rsp', 'rubin-devl'] 

462 

463 Raises 

464 ------ 

465 ValueError 

466 Raised if location cannot be determined. 

467 """ 

468 # All nublado instances guarantee that EXTERNAL_URL is set and uniquely 

469 # identifies it. 

470 location = os.getenv('EXTERNAL_URL', "") 

471 if location == "https://tucson-teststand.lsst.codes": 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 return 'tucson' 

473 elif location == "https://summit-lsp.lsst.codes": 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true

474 return 'summit' 

475 elif location == "https://base-lsp.lsst.codes": 475 ↛ 476line 475 didn't jump to line 476, because the condition on line 475 was never true

476 return 'base' 

477 elif location == "https://usdf-rsp.slac.stanford.edu": 477 ↛ 478line 477 didn't jump to line 478, because the condition on line 477 was never true

478 return 'staff-rsp' 

479 

480 # if no EXTERNAL_URL, try HOSTNAME to see if we're on the dev nodes 

481 # it is expected that this will be extensible to SLAC 

482 hostname = os.getenv('HOSTNAME', "") 

483 if hostname.startswith('sdfrome'): 483 ↛ 484line 483 didn't jump to line 484, because the condition on line 483 was never true

484 return 'rubin-devl' 

485 

486 # we have failed 

487 raise ValueError('Location could not be determined') 

488 

489 

490def getAltAzFromSkyPosition(skyPos, visitInfo): 

491 """Get the alt/az from the position on the sky and the time and location 

492 of the observation. 

493 

494 Parameters 

495 ---------- 

496 skyPos : `lsst.geom.SpherePoint` 

497 The position on the sky. 

498 visitInfo : `lsst.afw.image.VisitInfo` 

499 The visit info containing the time of the observation. 

500 

501 Returns 

502 ------- 

503 alt : `lsst.geom.Angle` 

504 The altitude. 

505 az : `lsst.geom.Angle` 

506 The azimuth. 

507 """ 

508 skyLocation = SkyCoord(skyPos.getRa().asRadians(), skyPos.getDec().asRadians(), unit=u.rad) 

509 long = visitInfo.observatory.getLongitude() 

510 lat = visitInfo.observatory.getLatitude() 

511 ele = visitInfo.observatory.getElevation() 

512 earthLocation = EarthLocation.from_geodetic(long.asDegrees(), lat.asDegrees(), ele) 

513 altAz = AltAz(obstime=visitInfo.date.toPython(), location=earthLocation) 

514 obsAltAz = skyLocation.transform_to(altAz) 

515 alt = geom.Angle(obsAltAz.alt.degree, geom.degrees) 

516 az = geom.Angle(obsAltAz.az.degree, geom.degrees) 

517 

518 return alt, az 

519 

520 

521def getExpPositionOffset(exp1, exp2, useWcs=True): 

522 """Get the change in sky position between two exposures. 

523 

524 Given two exposures, calculate the offset on the sky between the images. 

525 If useWcs then use the (fitted or unfitted) skyOrigin from their WCSs, and 

526 calculate the alt/az from the observation times, otherwise use the nominal 

527 values in the exposures' visitInfos. Note that if using the visitInfo 

528 values that for a given pointing the ra/dec will be ~identical, regardless 

529 of whether astrometric fitting has been performed. 

530 

531 Values are given as exp1-exp2. 

532 

533 Parameters 

534 ---------- 

535 exp1 : `lsst.afw.image.Exposure` 

536 The first exposure. 

537 exp2 : `lsst.afw.image.Exposure` 

538 The second exposure. 

539 useWcs : `bool` 

540 Use the WCS for the ra/dec and alt/az if True, else use the nominal/ 

541 boresight values from the exposures' visitInfos. 

542 

543 Returns 

544 ------- 

545 offsets : `lsst.pipe.base.Struct` 

546 A struct containing the offsets: 

547 ``deltaRa`` 

548 The diference in ra (`lsst.geom.Angle`) 

549 ``deltaDec`` 

550 The diference in dec (`lsst.geom.Angle`) 

551 ``deltaAlt`` 

552 The diference in alt (`lsst.geom.Angle`) 

553 ``deltaAz`` 

554 The diference in az (`lsst.geom.Angle`) 

555 ``deltaPixels`` 

556 The diference in pixels (`float`) 

557 """ 

558 

559 wcs1 = exp1.getWcs() 

560 wcs2 = exp2.getWcs() 

561 pixScaleArcSec = wcs1.getPixelScale().asArcseconds() 

562 assert np.isclose(pixScaleArcSec, wcs2.getPixelScale().asArcseconds()), \ 

563 "Pixel scales in the exposures differ." 

564 

565 if useWcs: 

566 p1 = wcs1.getSkyOrigin() 

567 p2 = wcs2.getSkyOrigin() 

568 alt1, az1 = getAltAzFromSkyPosition(p1, exp1.getInfo().getVisitInfo()) 

569 alt2, az2 = getAltAzFromSkyPosition(p2, exp2.getInfo().getVisitInfo()) 

570 ra1 = p1[0] 

571 ra2 = p2[0] 

572 dec1 = p1[1] 

573 dec2 = p2[1] 

574 else: 

575 az1 = exp1.visitInfo.boresightAzAlt[0] 

576 az2 = exp2.visitInfo.boresightAzAlt[0] 

577 alt1 = exp1.visitInfo.boresightAzAlt[1] 

578 alt2 = exp2.visitInfo.boresightAzAlt[1] 

579 

580 ra1 = exp1.visitInfo.boresightRaDec[0] 

581 ra2 = exp2.visitInfo.boresightRaDec[0] 

582 dec1 = exp1.visitInfo.boresightRaDec[1] 

583 dec2 = exp2.visitInfo.boresightRaDec[1] 

584 

585 p1 = exp1.visitInfo.boresightRaDec 

586 p2 = exp2.visitInfo.boresightRaDec 

587 

588 angular_offset = p1.separation(p2).asArcseconds() 

589 deltaPixels = angular_offset / pixScaleArcSec 

590 

591 ret = pipeBase.Struct(deltaRa=(ra1-ra2).wrapNear(geom.Angle(0.0)), 

592 deltaDec=dec1-dec2, 

593 deltaAlt=alt1-alt2, 

594 deltaAz=(az1-az2).wrapNear(geom.Angle(0.0)), 

595 deltaPixels=deltaPixels 

596 ) 

597 

598 return ret