Coverage for python/lsst/summit/utils/utils.py: 21%

216 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-06 03:25 -0800

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import numpy as np 

24from scipy.ndimage.filters import gaussian_filter 

25import lsst.afw.detection as afwDetect 

26import lsst.afw.math as afwMath 

27import lsst.geom as geom 

28import lsst.pipe.base as pipeBase 

29import lsst.utils.packages as packageUtils 

30from lsst.daf.butler.cli.cliLog import CliLog 

31import datetime 

32from dateutil.tz import gettz 

33 

34from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER 

35from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION 

36 

37from astro_metadata_translator import ObservationInfo 

38from astropy.coordinates import SkyCoord, AltAz 

39from astropy.coordinates.earth import EarthLocation 

40import astropy.units as u 

41 

42__all__ = ["SIGMATOFWHM", 

43 "FWHMTOSIGMA", 

44 "EFD_CLIENT_MISSING_MSG", 

45 "GOOGLE_CLOUD_MISSING_MSG", 

46 "AUXTEL_LOCATION", 

47 "countPixels", 

48 "quickSmooth", 

49 "argMax2d", 

50 "getImageStats", 

51 "detectObjectsInExp", 

52 "humanNameForCelestialObject", 

53 "getFocusFromHeader", 

54 "dayObsIntToString", 

55 "dayObsSeqNumToVisitId", 

56 "setupLogging", 

57 "getCurrentDayObs_datetime", 

58 "getCurrentDayObs_int", 

59 "getCurrentDayObs_humanStr", 

60 "getSite", 

61 "getExpPositionOffset", 

62 ] 

63 

64 

65SIGMATOFWHM = 2.0*np.sqrt(2.0*np.log(2.0)) 

66FWHMTOSIGMA = 1/SIGMATOFWHM 

67 

68EFD_CLIENT_MISSING_MSG = ('ImportError: lsst_efd_client not found. Please install with:\n' 

69 ' pip install lsst-efd-client') 

70 

71GOOGLE_CLOUD_MISSING_MSG = ('ImportError: Google cloud storage not found. Please install with:\n' 

72 ' pip install google-cloud-storage') 

73 

74 

75def countPixels(maskedImage, maskPlane): 

76 """Count the number of pixels in an image with a given mask bit set. 

77 

78 Parameters 

79 ---------- 

80 maskedImage : `lsst.afw.image.MaskedImage` 

81 The masked image, 

82 maskPlane : `str` 

83 The name of the bitmask. 

84 

85 Returns 

86 ------- 

87 count : `int`` 

88 The number of pixels in with the selected mask bit 

89 """ 

90 bit = maskedImage.mask.getPlaneBitMask(maskPlane) 

91 return len(np.where(np.bitwise_and(maskedImage.mask.array, bit))[0]) 

92 

93 

94def quickSmooth(data, sigma=2): 

95 """Perform a quick smoothing of the image. 

96 

97 Not to be used for scientific purposes, but improves the stretch and 

98 visual rendering of low SNR against the sky background in cutouts. 

99 

100 Parameters 

101 ---------- 

102 data : `np.array` 

103 The image data to smooth 

104 sigma : `float`, optional 

105 The size of the smoothing kernel. 

106 

107 Returns 

108 ------- 

109 smoothData : `np.array` 

110 The smoothed data 

111 """ 

112 kernel = [sigma, sigma] 

113 smoothData = gaussian_filter(data, kernel, mode='constant') 

114 return smoothData 

115 

116 

117def argMax2d(array): 

118 """Get the index of the max value of an array and whether it's unique. 

119 

120 If its not unique, returns a list of the other locations containing the 

121 maximum value, e.g. returns 

122 

123 (12, 34), False, [(56,78), (910, 1112)] 

124 

125 Parameters 

126 ---------- 

127 array : `np.array` 

128 The data 

129 

130 Returns 

131 ------- 

132 maxLocation : `tuple` 

133 The coords of the first instance of the max value 

134 unique : `bool` 

135 Whether it's the only location 

136 otherLocations : `list` of `tuple` 

137 List of the other max values' locations, empty if False 

138 """ 

139 uniqueMaximum = False 

140 maxCoords = np.where(array == np.max(array)) 

141 maxCoords = [coord for coord in zip(*maxCoords)] # list of coords as tuples 

142 if len(maxCoords) == 1: # single unambiguous value 

143 uniqueMaximum = True 

144 

145 return maxCoords[0], uniqueMaximum, maxCoords[1:] 

146 

147 

148def dayObsIntToString(dayObs): 

149 """Convert an integer dayObs to a dash-delimited string. 

150 

151 e.g. convert the hard to read 20210101 to 2021-01-01 

152 

153 Parameters 

154 ---------- 

155 dayObs : `int` 

156 The dayObs. 

157 

158 Returns 

159 ------- 

160 dayObs : `str` 

161 The dayObs as a string. 

162 """ 

163 assert isinstance(dayObs, int) 

164 dStr = str(dayObs) 

165 assert len(dStr) == 8 

166 return '-'.join([dStr[0:4], dStr[4:6], dStr[6:8]]) 

167 

168 

169def dayObsSeqNumToVisitId(dayObs, seqNum): 

170 """Get the visit id for a given dayObs/seqNum. 

171 

172 Parameters 

173 ---------- 

174 dayObs : `int` 

175 The dayObs. 

176 seqNum : `int` 

177 The seqNum. 

178 

179 Returns 

180 ------- 

181 visitId : `int` 

182 The visitId. 

183 

184 Notes 

185 ----- 

186 TODO: Remove this horrible hack once DM-30948 makes this possible 

187 programatically/via the butler. 

188 """ 

189 if dayObs < 19700101 or dayObs > 35000101: 

190 raise ValueError(f'dayObs value {dayObs} outside plausible range') 

191 return int(f"{dayObs}{seqNum:05}") 

192 

193 

194def getImageStats(exp): 

195 """Calculate a grab-bag of stats for an image. Must remain fast. 

196 

197 Parameters 

198 ---------- 

199 exp : `lsst.afw.image.Exposure` 

200 The input exposure. 

201 

202 Returns 

203 ------- 

204 stats : `lsst.pipe.base.Struct` 

205 A container with attributes containing measurements and statistics 

206 for the image. 

207 """ 

208 result = pipeBase.Struct() 

209 

210 vi = exp.visitInfo 

211 expTime = vi.exposureTime 

212 md = exp.getMetadata() 

213 

214 obj = vi.object 

215 mjd = vi.getDate().get() 

216 result.object = obj 

217 result.mjd = mjd 

218 

219 fullFilterString = exp.filter.physicalLabel 

220 filt = fullFilterString.split(FILTER_DELIMITER)[0] 

221 grating = fullFilterString.split(FILTER_DELIMITER)[1] 

222 

223 airmass = vi.getBoresightAirmass() 

224 rotangle = vi.getBoresightRotAngle().asDegrees() 

225 

226 azAlt = vi.getBoresightAzAlt() 

227 az = azAlt[0].asDegrees() 

228 el = azAlt[1].asDegrees() 

229 

230 result.expTime = expTime 

231 result.filter = filt 

232 result.grating = grating 

233 result.airmass = airmass 

234 result.rotangle = rotangle 

235 result.az = az 

236 result.el = el 

237 result.focus = md.get('FOCUSZ') 

238 

239 data = exp.image.array 

240 result.maxValue = np.max(data) 

241 

242 peak, uniquePeak, otherPeaks = argMax2d(data) 

243 result.maxPixelLocation = peak 

244 result.multipleMaxPixels = uniquePeak 

245 

246 result.nBadPixels = countPixels(exp.maskedImage, 'BAD') 

247 result.nSatPixels = countPixels(exp.maskedImage, 'SAT') 

248 result.percentile99 = np.percentile(data, 99) 

249 result.percentile9999 = np.percentile(data, 99.99) 

250 

251 sctrl = afwMath.StatisticsControl() 

252 sctrl.setNumSigmaClip(5) 

253 sctrl.setNumIter(2) 

254 statTypes = afwMath.MEANCLIP | afwMath.STDEVCLIP 

255 stats = afwMath.makeStatistics(exp.maskedImage, statTypes, sctrl) 

256 std, stderr = stats.getResult(afwMath.STDEVCLIP) 

257 mean, meanerr = stats.getResult(afwMath.MEANCLIP) 

258 

259 result.clippedMean = mean 

260 result.clippedStddev = std 

261 

262 return result 

263 

264 

265def detectObjectsInExp(exp, nSigma=10, nPixMin=10, grow=0): 

266 """Quick and dirty object detection for an expsure. 

267 

268 Return the footPrintSet for the objects in a preferably-postISR exposure. 

269 

270 Parameters 

271 ---------- 

272 exp : `lsst.afw.image.Exposure` 

273 The exposure to detect objects in. 

274 nSigma : `float` 

275 The number of sigma for detection. 

276 nPixMin : `int` 

277 The minimum number of pixels in an object for detection. 

278 grow : `int` 

279 The number of pixels to grow the footprint by after detection. 

280 

281 Returns 

282 ------- 

283 footPrintSet : `lsst.afw.detection.FootprintSet` 

284 The set of footprints in the image. 

285 """ 

286 median = np.nanmedian(exp.image.array) 

287 exp.image -= median 

288 

289 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV) 

290 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin) 

291 if grow > 0: 

292 isotropic = True 

293 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic) 

294 

295 exp.image += median # add back in to leave background unchanged 

296 return footPrintSet 

297 

298 

299def humanNameForCelestialObject(objName): 

300 """Returns a list of all human names for obj, or [] if none are found. 

301 

302 Parameters 

303 ---------- 

304 objName : `str` 

305 The/a name of the object. 

306 

307 Returns 

308 ------- 

309 names : `list` of `str` 

310 The names found for the object 

311 """ 

312 from astroquery.simbad import Simbad 

313 results = [] 

314 try: 

315 simbadResult = Simbad.query_objectids(objName) 

316 for row in simbadResult: 

317 if row['ID'].startswith('NAME'): 

318 results.append(row['ID'].replace('NAME ', '')) 

319 return results 

320 except Exception: 

321 return [] # same behavior as for found but un-named objects 

322 

323 

324def _getAltAzZenithsFromSeqNum(butler, dayObs, seqNumList): 

325 """Get the alt, az and zenith angle for the seqNums of a given dayObs. 

326 

327 Parameters 

328 ---------- 

329 butler : `lsst.daf.butler.Butler` 

330 The butler to query. 

331 dayObs : `int` 

332 The dayObs. 

333 seqNumList : `list` of `int` 

334 The seqNums for which to return the alt, az and zenith 

335 

336 Returns 

337 ------- 

338 azimuths : `list` of `float` 

339 List of the azimuths for each seqNum 

340 elevations : `list` of `float` 

341 List of the elevations for each seqNum 

342 zeniths : `list` of `float` 

343 List of the zenith angles for each seqNum 

344 """ 

345 azimuths, elevations, zeniths = [], [], [] 

346 for seqNum in seqNumList: 

347 md = butler.get('raw.metadata', day_obs=dayObs, seq_num=seqNum, detector=0) 

348 obsInfo = ObservationInfo(md) 

349 alt = obsInfo.altaz_begin.alt.value 

350 az = obsInfo.altaz_begin.az.value 

351 elevations.append(alt) 

352 zeniths.append(90-alt) 

353 azimuths.append(az) 

354 return azimuths, elevations, zeniths 

355 

356 

357def getFocusFromHeader(exp): 

358 """Get the raw focus value from the header. 

359 

360 Parameters 

361 ---------- 

362 exp : `lsst.afw.image.exposure` 

363 The exposure. 

364 

365 Returns 

366 ------- 

367 focus : `float` or `None` 

368 The focus value if found, else ``None``. 

369 """ 

370 md = exp.getMetadata() 

371 if 'FOCUSZ' in md: 

372 return md['FOCUSZ'] 

373 return None 

374 

375 

376def checkStackSetup(): 

377 """Check which weekly tag is being used and which local packages are setup. 

378 

379 Designed primarily for use in notbooks/observing, this prints the weekly 

380 tag(s) are setup for lsst_distrib, and lists any locally setup packages and 

381 the path to each. 

382 

383 Notes 

384 ----- 

385 Uses print() instead of logger messages as this should simply print them 

386 without being vulnerable to any log messages potentially being diverted. 

387 """ 

388 packages = packageUtils.getEnvironmentPackages(include_all=True) 

389 

390 lsstDistribHashAndTags = packages['lsst_distrib'] # looks something like 'g4eae7cb9+1418867f (w_2022_13)' 

391 lsstDistribTags = lsstDistribHashAndTags.split()[1] 

392 if len(lsstDistribTags.split()) == 1: 

393 tag = lsstDistribTags.replace('(', '') 

394 tag = tag.replace(')', '') 

395 print(f"You are running {tag} of lsst_distrib") 

396 else: # multiple weekly tags found for lsst_distrib! 

397 print(f'The version of lsst_distrib you have is compatible with: {lsstDistribTags}') 

398 

399 localPackages = [] 

400 localPaths = [] 

401 for package, tags in packages.items(): 

402 if tags.startswith('LOCAL:'): 

403 path = tags.split('LOCAL:')[1] 

404 path = path.split('@')[0] # don't need the git SHA etc 

405 localPaths.append(path) 

406 localPackages.append(package) 

407 

408 if localPackages: 

409 print("\nLocally setup packages:") 

410 print("-----------------------") 

411 maxLen = max(len(package) for package in localPackages) 

412 for package, path in zip(localPackages, localPaths): 

413 print(f"{package:<{maxLen}s} at {path}") 

414 else: 

415 print("\nNo locally setup packages (using a vanilla stack)") 

416 

417 

418def setupLogging(longlog=False): 

419 """Setup logging in the same way as one would get from pipetask run. 

420 

421 Code that isn't run through the butler CLI defaults to WARNING level 

422 messages and no logger names. This sets the behaviour to follow whatever 

423 the pipeline default is, currently 

424 <logger_name> <level>: <message> e.g. 

425 lsst.isr INFO: Masking defects. 

426 """ 

427 CliLog.initLog(longlog=longlog) 

428 

429 

430def getCurrentDayObs_datetime(): 

431 """Get the current day_obs - the observatory rolls the date over at UTC-12 

432 

433 Returned as datetime.date(2022, 4, 28) 

434 """ 

435 utc = gettz("UTC") 

436 nowUtc = datetime.datetime.now().astimezone(utc) 

437 offset = datetime.timedelta(hours=-12) 

438 dayObs = (nowUtc + offset).date() 

439 return dayObs 

440 

441 

442def getCurrentDayObs_int(): 

443 """Return the current dayObs as an int in the form 20220428 

444 """ 

445 return int(getCurrentDayObs_datetime().strftime("%Y%m%d")) 

446 

447 

448def getCurrentDayObs_humanStr(): 

449 """Return the current dayObs as a string in the form '2022-04-28' 

450 """ 

451 return dayObsIntToString(getCurrentDayObs_int()) 

452 

453 

454def getSite(): 

455 """Returns where the code is running. 

456 

457 Returns 

458 ------- 

459 location : `str` 

460 One of ['tucson', 'summit', 'base', 'staff-rsp', 'rubin-devl'] 

461 

462 Raises 

463 ------ 

464 ValueError 

465 Raised if location cannot be determined. 

466 """ 

467 # All nublado instances guarantee that EXTERNAL_URL is set and uniquely 

468 # identifies it. 

469 location = os.getenv('EXTERNAL_URL', "") 

470 if location == "https://tucson-teststand.lsst.codes": 470 ↛ 471line 470 didn't jump to line 471, because the condition on line 470 was never true

471 return 'tucson' 

472 elif location == "https://summit-lsp.lsst.codes": 472 ↛ 473line 472 didn't jump to line 473, because the condition on line 472 was never true

473 return 'summit' 

474 elif location == "https://base-lsp.lsst.codes": 474 ↛ 475line 474 didn't jump to line 475, because the condition on line 474 was never true

475 return 'base' 

476 elif location == "https://usdf-rsp.slac.stanford.edu": 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 return 'staff-rsp' 

478 

479 # if no EXTERNAL_URL, try HOSTNAME to see if we're on the dev nodes 

480 # it is expected that this will be extensible to SLAC 

481 hostname = os.getenv('HOSTNAME', "") 

482 if hostname.startswith('sdfrome'): 482 ↛ 483line 482 didn't jump to line 483, because the condition on line 482 was never true

483 return 'rubin-devl' 

484 

485 # we have failed 

486 raise ValueError('Location could not be determined') 

487 

488 

489def getAltAzFromSkyPosition(skyPos, visitInfo): 

490 """Get the alt/az from the position on the sky and the time and location 

491 of the observation. 

492 

493 Parameters 

494 ---------- 

495 skyPos : `lsst.geom.SpherePoint` 

496 The position on the sky. 

497 visitInfo : `lsst.afw.image.VisitInfo` 

498 The visit info containing the time of the observation. 

499 

500 Returns 

501 ------- 

502 alt : `lsst.geom.Angle` 

503 The altitude. 

504 az : `lsst.geom.Angle` 

505 The azimuth. 

506 """ 

507 skyLocation = SkyCoord(skyPos.getRa().asRadians(), skyPos.getDec().asRadians(), unit=u.rad) 

508 long = visitInfo.observatory.getLongitude() 

509 lat = visitInfo.observatory.getLatitude() 

510 ele = visitInfo.observatory.getElevation() 

511 earthLocation = EarthLocation.from_geodetic(long.asDegrees(), lat.asDegrees(), ele) 

512 altAz = AltAz(obstime=visitInfo.date.toPython(), location=earthLocation) 

513 obsAltAz = skyLocation.transform_to(altAz) 

514 alt = geom.Angle(obsAltAz.alt.degree, geom.degrees) 

515 az = geom.Angle(obsAltAz.az.degree, geom.degrees) 

516 

517 return alt, az 

518 

519 

520def getExpPositionOffset(exp1, exp2, useWcs=True, allowDifferentPlateScales=False): 

521 """Get the change in sky position between two exposures. 

522 

523 Given two exposures, calculate the offset on the sky between the images. 

524 If useWcs then use the (fitted or unfitted) skyOrigin from their WCSs, and 

525 calculate the alt/az from the observation times, otherwise use the nominal 

526 values in the exposures' visitInfos. Note that if using the visitInfo 

527 values that for a given pointing the ra/dec will be ~identical, regardless 

528 of whether astrometric fitting has been performed. 

529 

530 Values are given as exp1-exp2. 

531 

532 Parameters 

533 ---------- 

534 exp1 : `lsst.afw.image.Exposure` 

535 The first exposure. 

536 exp2 : `lsst.afw.image.Exposure` 

537 The second exposure. 

538 useWcs : `bool` 

539 Use the WCS for the ra/dec and alt/az if True, else use the nominal/ 

540 boresight values from the exposures' visitInfos. 

541 allowDifferentPlateScales : `bool`, optional 

542 Use to disable checking that plate scales are the same. Generally, 

543 differing plate scales would indicate an error, but where blind-solving 

544 has been undertaken during commissioning plate scales can be different 

545 enough to warrant setting this to ``True``. 

546 

547 Returns 

548 ------- 

549 offsets : `lsst.pipe.base.Struct` 

550 A struct containing the offsets: 

551 ``deltaRa`` 

552 The diference in ra (`lsst.geom.Angle`) 

553 ``deltaDec`` 

554 The diference in dec (`lsst.geom.Angle`) 

555 ``deltaAlt`` 

556 The diference in alt (`lsst.geom.Angle`) 

557 ``deltaAz`` 

558 The diference in az (`lsst.geom.Angle`) 

559 ``deltaPixels`` 

560 The diference in pixels (`float`) 

561 """ 

562 

563 wcs1 = exp1.getWcs() 

564 wcs2 = exp2.getWcs() 

565 pixScaleArcSec = wcs1.getPixelScale().asArcseconds() 

566 if not allowDifferentPlateScales: 

567 assert np.isclose(pixScaleArcSec, wcs2.getPixelScale().asArcseconds()), \ 

568 "Pixel scales in the exposures differ." 

569 

570 if useWcs: 

571 p1 = wcs1.getSkyOrigin() 

572 p2 = wcs2.getSkyOrigin() 

573 alt1, az1 = getAltAzFromSkyPosition(p1, exp1.getInfo().getVisitInfo()) 

574 alt2, az2 = getAltAzFromSkyPosition(p2, exp2.getInfo().getVisitInfo()) 

575 ra1 = p1[0] 

576 ra2 = p2[0] 

577 dec1 = p1[1] 

578 dec2 = p2[1] 

579 else: 

580 az1 = exp1.visitInfo.boresightAzAlt[0] 

581 az2 = exp2.visitInfo.boresightAzAlt[0] 

582 alt1 = exp1.visitInfo.boresightAzAlt[1] 

583 alt2 = exp2.visitInfo.boresightAzAlt[1] 

584 

585 ra1 = exp1.visitInfo.boresightRaDec[0] 

586 ra2 = exp2.visitInfo.boresightRaDec[0] 

587 dec1 = exp1.visitInfo.boresightRaDec[1] 

588 dec2 = exp2.visitInfo.boresightRaDec[1] 

589 

590 p1 = exp1.visitInfo.boresightRaDec 

591 p2 = exp2.visitInfo.boresightRaDec 

592 

593 angular_offset = p1.separation(p2).asArcseconds() 

594 deltaPixels = angular_offset / pixScaleArcSec 

595 

596 ret = pipeBase.Struct(deltaRa=(ra1-ra2).wrapNear(geom.Angle(0.0)), 

597 deltaDec=dec1-dec2, 

598 deltaAlt=alt1-alt2, 

599 deltaAz=(az1-az2).wrapNear(geom.Angle(0.0)), 

600 deltaPixels=deltaPixels 

601 ) 

602 

603 return ret