Coverage for python/lsst/summit/utils/utils.py: 21%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22import os

23import numpy as np

24from scipy.ndimage.filters import gaussian_filter

25import lsst.afw.detection as afwDetect

26import lsst.afw.math as afwMath

27import lsst.geom as geom

28import lsst.pipe.base as pipeBase

29import lsst.utils.packages as packageUtils

30from lsst.daf.butler.cli.cliLog import CliLog

31import datetime

32from dateutil.tz import gettz

34from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER

35from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION

37from astro_metadata_translator import ObservationInfo

38from astropy.coordinates import SkyCoord, AltAz

39from astropy.coordinates.earth import EarthLocation

40import astropy.units as u

42__all__ = ["SIGMATOFWHM",

43 "FWHMTOSIGMA",

44 "EFD_CLIENT_MISSING_MSG",

45 "GOOGLE_CLOUD_MISSING_MSG",

46 "AUXTEL_LOCATION",

47 "countPixels",

48 "quickSmooth",

49 "argMax2d",

50 "getImageStats",

51 "detectObjectsInExp",

52 "humanNameForCelestialObject",

53 "getFocusFromHeader",

54 "dayObsIntToString",

55 "dayObsSeqNumToVisitId",

56 "setupLogging",

57 "getCurrentDayObs_datetime",

58 "getCurrentDayObs_int",

59 "getCurrentDayObs_humanStr",

60 "getSite",

61 "getExpPositionOffset",

62 ]

65SIGMATOFWHM = 2.0*np.sqrt(2.0*np.log(2.0))

66FWHMTOSIGMA = 1/SIGMATOFWHM

68EFD_CLIENT_MISSING_MSG = ('ImportError: lsst_efd_client not found. Please install with:\n'

69 ' pip install lsst-efd-client')

71GOOGLE_CLOUD_MISSING_MSG = ('ImportError: Google cloud storage not found. Please install with:\n'

72 ' pip install google-cloud-storage')

75def countPixels(maskedImage, maskPlane):

76 """Count the number of pixels in an image with a given mask bit set.

78 Parameters

79 ----------

80 maskedImage : `lsst.afw.image.MaskedImage`

81 The masked image,

82 maskPlane : `str`

83 The name of the bitmask.

85 Returns

86 -------

87 count : `int``

88 The number of pixels in with the selected mask bit

89 """

90 bit = maskedImage.mask.getPlaneBitMask(maskPlane)

91 return len(np.where(np.bitwise_and(maskedImage.mask.array, bit))[0])

94def quickSmooth(data, sigma=2):

95 """Perform a quick smoothing of the image.

97 Not to be used for scientific purposes, but improves the stretch and

98 visual rendering of low SNR against the sky background in cutouts.

100 Parameters

101 ----------

102 data : `np.array`

103 The image data to smooth

104 sigma : `float`, optional

105 The size of the smoothing kernel.

106

107 Returns

108 -------

109 smoothData : `np.array`

110 The smoothed data

111 """

112 kernel = [sigma, sigma]

113 smoothData = gaussian_filter(data, kernel, mode='constant')

114 return smoothData

115

116

117def argMax2d(array):

118 """Get the index of the max value of an array and whether it's unique.

119

120 If its not unique, returns a list of the other locations containing the

121 maximum value, e.g. returns

122

123 (12, 34), False, [(56,78), (910, 1112)]

124

125 Parameters

126 ----------

127 array : `np.array`

128 The data

129

130 Returns

131 -------

132 maxLocation : `tuple`

133 The coords of the first instance of the max value

134 unique : `bool`

135 Whether it's the only location

136 otherLocations : `list` of `tuple`

137 List of the other max values' locations, empty if False

138 """

139 uniqueMaximum = False

140 maxCoords = np.where(array == np.max(array))

141 maxCoords = [coord for coord in zip(*maxCoords)] # list of coords as tuples

142 if len(maxCoords) == 1: # single unambiguous value

143 uniqueMaximum = True

144

145 return maxCoords[0], uniqueMaximum, maxCoords[1:]

146

147

148def dayObsIntToString(dayObs):

149 """Convert an integer dayObs to a dash-delimited string.

150

151 e.g. convert the hard to read 20210101 to 2021-01-01

152

153 Parameters

154 ----------

155 dayObs : `int`

156 The dayObs.

157

158 Returns

159 -------

160 dayObs : `str`

161 The dayObs as a string.

162 """

163 assert isinstance(dayObs, int)

164 dStr = str(dayObs)

165 assert len(dStr) == 8

166 return '-'.join([dStr[0:4], dStr[4:6], dStr[6:8]])

167

168

169def dayObsSeqNumToVisitId(dayObs, seqNum):

170 """Get the visit id for a given dayObs/seqNum.

171

172 Parameters

173 ----------

174 dayObs : `int`

175 The dayObs.

176 seqNum : `int`

177 The seqNum.

178

179 Returns

180 -------

181 visitId : `int`

182 The visitId.

183

184 Notes

185 -----

186 TODO: Remove this horrible hack once DM-30948 makes this possible

187 programatically/via the butler.

188 """

189 if dayObs < 19700101 or dayObs > 35000101:

190 raise ValueError(f'dayObs value {dayObs} outside plausible range')

191 return int(f"{dayObs}{seqNum:05}")

192

193

194def getImageStats(exp):

195 """Calculate a grab-bag of stats for an image. Must remain fast.

196

197 Parameters

198 ----------

199 exp : `lsst.afw.image.Exposure`

200 The input exposure.

201

202 Returns

203 -------

204 stats : `lsst.pipe.base.Struct`

205 A container with attributes containing measurements and statistics

206 for the image.

207 """

208 result = pipeBase.Struct()

209

210 info = exp.getInfo()

211 vi = info.getVisitInfo()

212 expTime = vi.getExposureTime()

213 md = exp.getMetadata()

214

215 obj = vi.object

216 mjd = vi.getDate().get()

217 result.object = obj

218 result.mjd = mjd

219

220 fullFilterString = info.getFilterLabel().physicalLabel

221 filt = fullFilterString.split(FILTER_DELIMITER)[0]

222 grating = fullFilterString.split(FILTER_DELIMITER)[1]

223

224 airmass = vi.getBoresightAirmass()

225 rotangle = vi.getBoresightRotAngle().asDegrees()

226

227 azAlt = vi.getBoresightAzAlt()

228 az = azAlt[0].asDegrees()

229 el = azAlt[1].asDegrees()

230

231 result.expTime = expTime

232 result.filter = filt

233 result.grating = grating

234 result.airmass = airmass

235 result.rotangle = rotangle

236 result.az = az

237 result.el = el

238 result.focus = md.get('FOCUSZ')

239

240 data = exp.image.array

241 result.maxValue = np.max(data)

242

243 peak, uniquePeak, otherPeaks = argMax2d(data)

244 result.maxPixelLocation = peak

245 result.multipleMaxPixels = uniquePeak

246

247 result.nBadPixels = countPixels(exp.maskedImage, 'BAD')

248 result.nSatPixels = countPixels(exp.maskedImage, 'SAT')

249 result.percentile99 = np.percentile(data, 99)

250 result.percentile9999 = np.percentile(data, 99.99)

251

252 sctrl = afwMath.StatisticsControl()

253 sctrl.setNumSigmaClip(5)

254 sctrl.setNumIter(2)

255 statTypes = afwMath.MEANCLIP | afwMath.STDEVCLIP

256 stats = afwMath.makeStatistics(exp.maskedImage, statTypes, sctrl)

257 std, stderr = stats.getResult(afwMath.STDEVCLIP)

258 mean, meanerr = stats.getResult(afwMath.MEANCLIP)

259

260 result.clippedMean = mean

261 result.clippedStddev = std

262

263 return result

264

265

266def detectObjectsInExp(exp, nSigma=10, nPixMin=10, grow=0):

267 """Quick and dirty object detection for an expsure.

268

269 Return the footPrintSet for the objects in a preferably-postISR exposure.

270

271 Parameters

272 ----------

273 exp : `lsst.afw.image.Exposure`

274 The exposure to detect objects in.

275 nSigma : `float`

276 The number of sigma for detection.

277 nPixMin : `int`

278 The minimum number of pixels in an object for detection.

279 grow : `int`

280 The number of pixels to grow the footprint by after detection.

281

282 Returns

283 -------

284 footPrintSet : `lsst.afw.detection.FootprintSet`

285 The set of footprints in the image.

286 """

287 median = np.nanmedian(exp.image.array)

288 exp.image -= median

289

290 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV)

291 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin)

292 if grow > 0:

293 isotropic = True

294 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic)

295

296 exp.image += median # add back in to leave background unchanged

297 return footPrintSet

298

299

300def humanNameForCelestialObject(objName):

301 """Returns a list of all human names for obj, or [] if none are found.

302

303 Parameters

304 ----------

305 objName : `str`

306 The/a name of the object.

307

308 Returns

309 -------

310 names : `list` of `str`

311 The names found for the object

312 """

313 from astroquery.simbad import Simbad

314 results = []

315 try:

316 simbadResult = Simbad.query_objectids(objName)

317 for row in simbadResult:

318 if row['ID'].startswith('NAME'):

319 results.append(row['ID'].replace('NAME ', ''))

320 return results

321 except Exception:

322 return [] # same behavior as for found but un-named objects

323

324

325def _getAltAzZenithsFromSeqNum(butler, dayObs, seqNumList):

326 """Get the alt, az and zenith angle for the seqNums of a given dayObs.

327

328 Parameters

329 ----------

330 butler : `lsst.daf.butler.Butler`

331 The butler to query.

332 dayObs : `int`

333 The dayObs.

334 seqNumList : `list` of `int`

335 The seqNums for which to return the alt, az and zenith

336

337 Returns

338 -------

339 azimuths : `list` of `float`

340 List of the azimuths for each seqNum

341 elevations : `list` of `float`

342 List of the elevations for each seqNum

343 zeniths : `list` of `float`

344 List of the zenith angles for each seqNum

345 """

346 azimuths, elevations, zeniths = [], [], []

347 for seqNum in seqNumList:

348 md = butler.get('raw.metadata', day_obs=dayObs, seq_num=seqNum, detector=0)

349 obsInfo = ObservationInfo(md)

350 alt = obsInfo.altaz_begin.alt.value

351 az = obsInfo.altaz_begin.az.value

352 elevations.append(alt)

353 zeniths.append(90-alt)

354 azimuths.append(az)

355 return azimuths, elevations, zeniths

356

357

358def getFocusFromHeader(exp):

359 """Get the raw focus value from the header.

360

361 Parameters

362 ----------

363 exp : `lsst.afw.image.exposure`

364 The exposure.

365

366 Returns

367 -------

368 focus : `float` or `None`

369 The focus value if found, else ``None``.

370 """

371 md = exp.getMetadata()

372 if 'FOCUSZ' in md:

373 return md['FOCUSZ']

374 return None

375

376

377def checkStackSetup():

378 """Check which weekly tag is being used and which local packages are setup.

379

380 Designed primarily for use in notbooks/observing, this prints the weekly

381 tag(s) are setup for lsst_distrib, and lists any locally setup packages and

382 the path to each.

383

384 Notes

385 -----

386 Uses print() instead of logger messages as this should simply print them

387 without being vulnerable to any log messages potentially being diverted.

388 """

389 packages = packageUtils.getEnvironmentPackages(include_all=True)

390

391 lsstDistribHashAndTags = packages['lsst_distrib'] # looks something like 'g4eae7cb9+1418867f (w_2022_13)'

392 lsstDistribTags = lsstDistribHashAndTags.split()[1]

393 if len(lsstDistribTags.split()) == 1:

394 tag = lsstDistribTags.replace('(', '')

395 tag = tag.replace(')', '')

396 print(f"You are running {tag} of lsst_distrib")

397 else: # multiple weekly tags found for lsst_distrib!

398 print(f'The version of lsst_distrib you have is compatible with: {lsstDistribTags}')

399

400 localPackages = []

401 localPaths = []

402 for package, tags in packages.items():

403 if tags.startswith('LOCAL:'):

404 path = tags.split('LOCAL:')[1]

405 path = path.split('@')[0] # don't need the git SHA etc

406 localPaths.append(path)

407 localPackages.append(package)

408

409 if localPackages:

410 print("\nLocally setup packages:")

411 print("-----------------------")

412 maxLen = max(len(package) for package in localPackages)

413 for package, path in zip(localPackages, localPaths):

414 print(f"{package:<{maxLen}s} at {path}")

415 else:

416 print("\nNo locally setup packages (using a vanilla stack)")

417

418

419def setupLogging(longlog=False):

420 """Setup logging in the same way as one would get from pipetask run.

421

422 Code that isn't run through the butler CLI defaults to WARNING level

423 messages and no logger names. This sets the behaviour to follow whatever

424 the pipeline default is, currently

425 <logger_name> <level>: <message> e.g.

426 lsst.isr INFO: Masking defects.

427 """

428 CliLog.initLog(longlog=longlog)

429

430

431def getCurrentDayObs_datetime():

432 """Get the current day_obs - the observatory rolls the date over at UTC-12

433

434 Returned as datetime.date(2022, 4, 28)

435 """

436 utc = gettz("UTC")

437 nowUtc = datetime.datetime.now().astimezone(utc)

438 offset = datetime.timedelta(hours=-12)

439 dayObs = (nowUtc + offset).date()

440 return dayObs

441

442

443def getCurrentDayObs_int():

444 """Return the current dayObs as an int in the form 20220428

445 """

446 return int(getCurrentDayObs_datetime().strftime("%Y%m%d"))

447

448

449def getCurrentDayObs_humanStr():

450 """Return the current dayObs as a string in the form '2022-04-28'

451 """

452 return dayObsIntToString(getCurrentDayObs_int())

453

454

455def getSite():

456 """Returns where the code is running.

457

458 Returns

459 -------

460 location : `str`

461 One of ['tucson', 'summit', 'base', 'staff-rsp', 'rubin-devl']

462

463 Raises

464 ------

465 ValueError

466 Raised if location cannot be determined.

467 """

468 # All nublado instances guarantee that EXTERNAL_URL is set and uniquely

469 # identifies it.

470 location = os.getenv('EXTERNAL_URL', "")

471 if location == "https://tucson-teststand.lsst.codes": 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 return 'tucson'

473 elif location == "https://summit-lsp.lsst.codes": 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true

474 return 'summit'

475 elif location == "https://base-lsp.lsst.codes": 475 ↛ 476line 475 didn't jump to line 476, because the condition on line 475 was never true

476 return 'base'

477 elif location == "https://usdf-rsp.slac.stanford.edu": 477 ↛ 478line 477 didn't jump to line 478, because the condition on line 477 was never true

478 return 'staff-rsp'

479

480 # if no EXTERNAL_URL, try HOSTNAME to see if we're on the dev nodes

481 # it is expected that this will be extensible to SLAC

482 hostname = os.getenv('HOSTNAME', "")

483 if hostname.startswith('sdfrome'): 483 ↛ 484line 483 didn't jump to line 484, because the condition on line 483 was never true

484 return 'rubin-devl'

485

486 # we have failed

487 raise ValueError('Location could not be determined')

488

489

490def getAltAzFromSkyPosition(skyPos, visitInfo):

491 """Get the alt/az from the position on the sky and the time and location

492 of the observation.

493

494 Parameters

495 ----------

496 skyPos : `lsst.geom.SpherePoint`

497 The position on the sky.

498 visitInfo : `lsst.afw.image.VisitInfo`

499 The visit info containing the time of the observation.

500

501 Returns

502 -------

503 alt : `lsst.geom.Angle`

504 The altitude.

505 az : `lsst.geom.Angle`

506 The azimuth.

507 """

508 skyLocation = SkyCoord(skyPos.getRa().asRadians(), skyPos.getDec().asRadians(), unit=u.rad)

509 long = visitInfo.observatory.getLongitude()

510 lat = visitInfo.observatory.getLatitude()

511 ele = visitInfo.observatory.getElevation()

512 earthLocation = EarthLocation.from_geodetic(long.asDegrees(), lat.asDegrees(), ele)

513 altAz = AltAz(obstime=visitInfo.date.toPython(), location=earthLocation)

514 obsAltAz = skyLocation.transform_to(altAz)

515 alt = geom.Angle(obsAltAz.alt.degree, geom.degrees)

516 az = geom.Angle(obsAltAz.az.degree, geom.degrees)

517

518 return alt, az

519

520

521def getExpPositionOffset(exp1, exp2, useWcs=True):

522 """Get the change in sky position between two exposures.

523

524 Given two exposures, calculate the offset on the sky between the images.

525 If useWcs then use the (fitted or unfitted) skyOrigin from their WCSs, and

526 calculate the alt/az from the observation times, otherwise use the nominal

527 values in the exposures' visitInfos. Note that if using the visitInfo

528 values that for a given pointing the ra/dec will be ~identical, regardless

529 of whether astrometric fitting has been performed.

530

531 Values are given as exp1-exp2.

532

533 Parameters

534 ----------

535 exp1 : `lsst.afw.image.Exposure`

536 The first exposure.

537 exp2 : `lsst.afw.image.Exposure`

538 The second exposure.

539 useWcs : `bool`

540 Use the WCS for the ra/dec and alt/az if True, else use the nominal/

541 boresight values from the exposures' visitInfos.

542

543 Returns

544 -------

545 offsets : `lsst.pipe.base.Struct`

546 A struct containing the offsets:

547 ``deltaRa``

548 The diference in ra (`lsst.geom.Angle`)

549 ``deltaDec``

550 The diference in dec (`lsst.geom.Angle`)

551 ``deltaAlt``

552 The diference in alt (`lsst.geom.Angle`)

553 ``deltaAz``

554 The diference in az (`lsst.geom.Angle`)

555 ``deltaPixels``

556 The diference in pixels (`float`)

557 """

558

559 wcs1 = exp1.getWcs()

560 wcs2 = exp2.getWcs()

561 pixScaleArcSec = wcs1.getPixelScale().asArcseconds()

562 assert np.isclose(pixScaleArcSec, wcs2.getPixelScale().asArcseconds()), \

563 "Pixel scales in the exposures differ."

564

565 if useWcs:

566 p1 = wcs1.getSkyOrigin()

567 p2 = wcs2.getSkyOrigin()

568 alt1, az1 = getAltAzFromSkyPosition(p1, exp1.getInfo().getVisitInfo())

569 alt2, az2 = getAltAzFromSkyPosition(p2, exp2.getInfo().getVisitInfo())

570 ra1 = p1[0]

571 ra2 = p2[0]

572 dec1 = p1[1]

573 dec2 = p2[1]

574 else:

575 az1 = exp1.visitInfo.boresightAzAlt[0]

576 az2 = exp2.visitInfo.boresightAzAlt[0]

577 alt1 = exp1.visitInfo.boresightAzAlt[1]

578 alt2 = exp2.visitInfo.boresightAzAlt[1]

579

580 ra1 = exp1.visitInfo.boresightRaDec[0]

581 ra2 = exp2.visitInfo.boresightRaDec[0]

582 dec1 = exp1.visitInfo.boresightRaDec[1]

583 dec2 = exp2.visitInfo.boresightRaDec[1]

584

585 p1 = exp1.visitInfo.boresightRaDec

586 p2 = exp2.visitInfo.boresightRaDec

587

588 angular_offset = p1.separation(p2).asArcseconds()

589 deltaPixels = angular_offset / pixScaleArcSec

590

591 ret = pipeBase.Struct(deltaRa=(ra1-ra2).wrapNear(geom.Angle(0.0)),

592 deltaDec=dec1-dec2,

593 deltaAlt=alt1-alt2,

594 deltaAz=(az1-az2).wrapNear(geom.Angle(0.0)),

595 deltaPixels=deltaPixels

596 )

597

598 return ret