Coverage for python/lsst/summit/utils/utils.py: 21%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22import os

23import numpy as np

24from scipy.ndimage.filters import gaussian_filter

25import lsst.afw.detection as afwDetect

26import lsst.afw.math as afwMath

27import lsst.geom as geom

28import lsst.pipe.base as pipeBase

29import lsst.utils.packages as packageUtils

30from lsst.daf.butler.cli.cliLog import CliLog

31import datetime

32from dateutil.tz import gettz

34from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER

35from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION

37from astro_metadata_translator import ObservationInfo

38from astropy.coordinates import SkyCoord, AltAz

39from astropy.coordinates.earth import EarthLocation

40import astropy.units as u

42__all__ = ["SIGMATOFWHM",

43 "FWHMTOSIGMA",

44 "EFD_CLIENT_MISSING_MSG",

45 "GOOGLE_CLOUD_MISSING_MSG",

46 "AUXTEL_LOCATION",

47 "countPixels",

48 "quickSmooth",

49 "argMax2d",

50 "getImageStats",

51 "detectObjectsInExp",

52 "humanNameForCelestialObject",

53 "getFocusFromHeader",

54 "dayObsIntToString",

55 "dayObsSeqNumToVisitId",

56 "setupLogging",

57 "getCurrentDayObs_datetime",

58 "getCurrentDayObs_int",

59 "getCurrentDayObs_humanStr",

60 "getSite",

61 "getExpPositionOffset",

62 ]

65SIGMATOFWHM = 2.0*np.sqrt(2.0*np.log(2.0))

66FWHMTOSIGMA = 1/SIGMATOFWHM

68EFD_CLIENT_MISSING_MSG = ('ImportError: lsst_efd_client not found. Please install with:\n'

69 ' pip install lsst-efd-client')

71GOOGLE_CLOUD_MISSING_MSG = ('ImportError: Google cloud storage not found. Please install with:\n'

72 ' pip install google-cloud-storage')

75def countPixels(maskedImage, maskPlane):

76 """Count the number of pixels in an image with a given mask bit set.

78 Parameters

79 ----------

80 maskedImage : `lsst.afw.image.MaskedImage`

81 The masked image,

82 maskPlane : `str`

83 The name of the bitmask.

85 Returns

86 -------

87 count : `int``

88 The number of pixels in with the selected mask bit

89 """

90 bit = maskedImage.mask.getPlaneBitMask(maskPlane)

91 return len(np.where(np.bitwise_and(maskedImage.mask.array, bit))[0])

94def quickSmooth(data, sigma=2):

95 """Perform a quick smoothing of the image.

97 Not to be used for scientific purposes, but improves the stretch and

98 visual rendering of low SNR against the sky background in cutouts.

100 Parameters

101 ----------

102 data : `np.array`

103 The image data to smooth

104 sigma : `float`, optional

105 The size of the smoothing kernel.

106

107 Returns

108 -------

109 smoothData : `np.array`

110 The smoothed data

111 """

112 kernel = [sigma, sigma]

113 smoothData = gaussian_filter(data, kernel, mode='constant')

114 return smoothData

115

116

117def argMax2d(array):

118 """Get the index of the max value of an array and whether it's unique.

119

120 If its not unique, returns a list of the other locations containing the

121 maximum value, e.g. returns

122

123 (12, 34), False, [(56,78), (910, 1112)]

124

125 Parameters

126 ----------

127 array : `np.array`

128 The data

129

130 Returns

131 -------

132 maxLocation : `tuple`

133 The coords of the first instance of the max value

134 unique : `bool`

135 Whether it's the only location

136 otherLocations : `list` of `tuple`

137 List of the other max values' locations, empty if False

138 """

139 uniqueMaximum = False

140 maxCoords = np.where(array == np.max(array))

141 maxCoords = [coord for coord in zip(*maxCoords)] # list of coords as tuples

142 if len(maxCoords) == 1: # single unambiguous value

143 uniqueMaximum = True

144

145 return maxCoords[0], uniqueMaximum, maxCoords[1:]

146

147

148def dayObsIntToString(dayObs):

149 """Convert an integer dayObs to a dash-delimited string.

150

151 e.g. convert the hard to read 20210101 to 2021-01-01

152

153 Parameters

154 ----------

155 dayObs : `int`

156 The dayObs.

157

158 Returns

159 -------

160 dayObs : `str`

161 The dayObs as a string.

162 """

163 assert isinstance(dayObs, int)

164 dStr = str(dayObs)

165 assert len(dStr) == 8

166 return '-'.join([dStr[0:4], dStr[4:6], dStr[6:8]])

167

168

169def dayObsSeqNumToVisitId(dayObs, seqNum):

170 """Get the visit id for a given dayObs/seqNum.

171

172 Parameters

173 ----------

174 dayObs : `int`

175 The dayObs.

176 seqNum : `int`

177 The seqNum.

178

179 Returns

180 -------

181 visitId : `int`

182 The visitId.

183

184 Notes

185 -----

186 TODO: Remove this horrible hack once DM-30948 makes this possible

187 programatically/via the butler.

188 """

189 if dayObs < 19700101 or dayObs > 35000101:

190 raise ValueError(f'dayObs value {dayObs} outside plausible range')

191 return int(f"{dayObs}{seqNum:05}")

192

193

194def getImageStats(exp):

195 """Calculate a grab-bag of stats for an image. Must remain fast.

196

197 Parameters

198 ----------

199 exp : `lsst.afw.image.Exposure`

200 The input exposure.

201

202 Returns

203 -------

204 stats : `lsst.pipe.base.Struct`

205 A container with attributes containing measurements and statistics

206 for the image.

207 """

208 result = pipeBase.Struct()

209

210 vi = exp.visitInfo

211 expTime = vi.exposureTime

212 md = exp.getMetadata()

213

214 obj = vi.object

215 mjd = vi.getDate().get()

216 result.object = obj

217 result.mjd = mjd

218

219 fullFilterString = exp.filter.physicalLabel

220 filt = fullFilterString.split(FILTER_DELIMITER)[0]

221 grating = fullFilterString.split(FILTER_DELIMITER)[1]

222

223 airmass = vi.getBoresightAirmass()

224 rotangle = vi.getBoresightRotAngle().asDegrees()

225

226 azAlt = vi.getBoresightAzAlt()

227 az = azAlt[0].asDegrees()

228 el = azAlt[1].asDegrees()

229

230 result.expTime = expTime

231 result.filter = filt

232 result.grating = grating

233 result.airmass = airmass

234 result.rotangle = rotangle

235 result.az = az

236 result.el = el

237 result.focus = md.get('FOCUSZ')

238

239 data = exp.image.array

240 result.maxValue = np.max(data)

241

242 peak, uniquePeak, otherPeaks = argMax2d(data)

243 result.maxPixelLocation = peak

244 result.multipleMaxPixels = uniquePeak

245

246 result.nBadPixels = countPixels(exp.maskedImage, 'BAD')

247 result.nSatPixels = countPixels(exp.maskedImage, 'SAT')

248 result.percentile99 = np.percentile(data, 99)

249 result.percentile9999 = np.percentile(data, 99.99)

250

251 sctrl = afwMath.StatisticsControl()

252 sctrl.setNumSigmaClip(5)

253 sctrl.setNumIter(2)

254 statTypes = afwMath.MEANCLIP | afwMath.STDEVCLIP

255 stats = afwMath.makeStatistics(exp.maskedImage, statTypes, sctrl)

256 std, stderr = stats.getResult(afwMath.STDEVCLIP)

257 mean, meanerr = stats.getResult(afwMath.MEANCLIP)

258

259 result.clippedMean = mean

260 result.clippedStddev = std

261

262 return result

263

264

265def detectObjectsInExp(exp, nSigma=10, nPixMin=10, grow=0):

266 """Quick and dirty object detection for an expsure.

267

268 Return the footPrintSet for the objects in a preferably-postISR exposure.

269

270 Parameters

271 ----------

272 exp : `lsst.afw.image.Exposure`

273 The exposure to detect objects in.

274 nSigma : `float`

275 The number of sigma for detection.

276 nPixMin : `int`

277 The minimum number of pixels in an object for detection.

278 grow : `int`

279 The number of pixels to grow the footprint by after detection.

280

281 Returns

282 -------

283 footPrintSet : `lsst.afw.detection.FootprintSet`

284 The set of footprints in the image.

285 """

286 median = np.nanmedian(exp.image.array)

287 exp.image -= median

288

289 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV)

290 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin)

291 if grow > 0:

292 isotropic = True

293 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic)

294

295 exp.image += median # add back in to leave background unchanged

296 return footPrintSet

297

298

299def humanNameForCelestialObject(objName):

300 """Returns a list of all human names for obj, or [] if none are found.

301

302 Parameters

303 ----------

304 objName : `str`

305 The/a name of the object.

306

307 Returns

308 -------

309 names : `list` of `str`

310 The names found for the object

311 """

312 from astroquery.simbad import Simbad

313 results = []

314 try:

315 simbadResult = Simbad.query_objectids(objName)

316 for row in simbadResult:

317 if row['ID'].startswith('NAME'):

318 results.append(row['ID'].replace('NAME ', ''))

319 return results

320 except Exception:

321 return [] # same behavior as for found but un-named objects

322

323

324def _getAltAzZenithsFromSeqNum(butler, dayObs, seqNumList):

325 """Get the alt, az and zenith angle for the seqNums of a given dayObs.

326

327 Parameters

328 ----------

329 butler : `lsst.daf.butler.Butler`

330 The butler to query.

331 dayObs : `int`

332 The dayObs.

333 seqNumList : `list` of `int`

334 The seqNums for which to return the alt, az and zenith

335

336 Returns

337 -------

338 azimuths : `list` of `float`

339 List of the azimuths for each seqNum

340 elevations : `list` of `float`

341 List of the elevations for each seqNum

342 zeniths : `list` of `float`

343 List of the zenith angles for each seqNum

344 """

345 azimuths, elevations, zeniths = [], [], []

346 for seqNum in seqNumList:

347 md = butler.get('raw.metadata', day_obs=dayObs, seq_num=seqNum, detector=0)

348 obsInfo = ObservationInfo(md)

349 alt = obsInfo.altaz_begin.alt.value

350 az = obsInfo.altaz_begin.az.value

351 elevations.append(alt)

352 zeniths.append(90-alt)

353 azimuths.append(az)

354 return azimuths, elevations, zeniths

355

356

357def getFocusFromHeader(exp):

358 """Get the raw focus value from the header.

359

360 Parameters

361 ----------

362 exp : `lsst.afw.image.exposure`

363 The exposure.

364

365 Returns

366 -------

367 focus : `float` or `None`

368 The focus value if found, else ``None``.

369 """

370 md = exp.getMetadata()

371 if 'FOCUSZ' in md:

372 return md['FOCUSZ']

373 return None

374

375

376def checkStackSetup():

377 """Check which weekly tag is being used and which local packages are setup.

378

379 Designed primarily for use in notbooks/observing, this prints the weekly

380 tag(s) are setup for lsst_distrib, and lists any locally setup packages and

381 the path to each.

382

383 Notes

384 -----

385 Uses print() instead of logger messages as this should simply print them

386 without being vulnerable to any log messages potentially being diverted.

387 """

388 packages = packageUtils.getEnvironmentPackages(include_all=True)

389

390 lsstDistribHashAndTags = packages['lsst_distrib'] # looks something like 'g4eae7cb9+1418867f (w_2022_13)'

391 lsstDistribTags = lsstDistribHashAndTags.split()[1]

392 if len(lsstDistribTags.split()) == 1:

393 tag = lsstDistribTags.replace('(', '')

394 tag = tag.replace(')', '')

395 print(f"You are running {tag} of lsst_distrib")

396 else: # multiple weekly tags found for lsst_distrib!

397 print(f'The version of lsst_distrib you have is compatible with: {lsstDistribTags}')

398

399 localPackages = []

400 localPaths = []

401 for package, tags in packages.items():

402 if tags.startswith('LOCAL:'):

403 path = tags.split('LOCAL:')[1]

404 path = path.split('@')[0] # don't need the git SHA etc

405 localPaths.append(path)

406 localPackages.append(package)

407

408 if localPackages:

409 print("\nLocally setup packages:")

410 print("-----------------------")

411 maxLen = max(len(package) for package in localPackages)

412 for package, path in zip(localPackages, localPaths):

413 print(f"{package:<{maxLen}s} at {path}")

414 else:

415 print("\nNo locally setup packages (using a vanilla stack)")

416

417

418def setupLogging(longlog=False):

419 """Setup logging in the same way as one would get from pipetask run.

420

421 Code that isn't run through the butler CLI defaults to WARNING level

422 messages and no logger names. This sets the behaviour to follow whatever

423 the pipeline default is, currently

424 <logger_name> <level>: <message> e.g.

425 lsst.isr INFO: Masking defects.

426 """

427 CliLog.initLog(longlog=longlog)

428

429

430def getCurrentDayObs_datetime():

431 """Get the current day_obs - the observatory rolls the date over at UTC-12

432

433 Returned as datetime.date(2022, 4, 28)

434 """

435 utc = gettz("UTC")

436 nowUtc = datetime.datetime.now().astimezone(utc)

437 offset = datetime.timedelta(hours=-12)

438 dayObs = (nowUtc + offset).date()

439 return dayObs

440

441

442def getCurrentDayObs_int():

443 """Return the current dayObs as an int in the form 20220428

444 """

445 return int(getCurrentDayObs_datetime().strftime("%Y%m%d"))

446

447

448def getCurrentDayObs_humanStr():

449 """Return the current dayObs as a string in the form '2022-04-28'

450 """

451 return dayObsIntToString(getCurrentDayObs_int())

452

453

454def getSite():

455 """Returns where the code is running.

456

457 Returns

458 -------

459 location : `str`

460 One of ['tucson', 'summit', 'base', 'staff-rsp', 'rubin-devl']

461

462 Raises

463 ------

464 ValueError

465 Raised if location cannot be determined.

466 """

467 # All nublado instances guarantee that EXTERNAL_URL is set and uniquely

468 # identifies it.

469 location = os.getenv('EXTERNAL_URL', "")

470 if location == "https://tucson-teststand.lsst.codes": 470 ↛ 471line 470 didn't jump to line 471, because the condition on line 470 was never true

471 return 'tucson'

472 elif location == "https://summit-lsp.lsst.codes": 472 ↛ 473line 472 didn't jump to line 473, because the condition on line 472 was never true

473 return 'summit'

474 elif location == "https://base-lsp.lsst.codes": 474 ↛ 475line 474 didn't jump to line 475, because the condition on line 474 was never true

475 return 'base'

476 elif location == "https://usdf-rsp.slac.stanford.edu": 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 return 'staff-rsp'

478

479 # if no EXTERNAL_URL, try HOSTNAME to see if we're on the dev nodes

480 # it is expected that this will be extensible to SLAC

481 hostname = os.getenv('HOSTNAME', "")

482 if hostname.startswith('sdfrome'): 482 ↛ 483line 482 didn't jump to line 483, because the condition on line 482 was never true

483 return 'rubin-devl'

484

485 # we have failed

486 raise ValueError('Location could not be determined')

487

488

489def getAltAzFromSkyPosition(skyPos, visitInfo):

490 """Get the alt/az from the position on the sky and the time and location

491 of the observation.

492

493 Parameters

494 ----------

495 skyPos : `lsst.geom.SpherePoint`

496 The position on the sky.

497 visitInfo : `lsst.afw.image.VisitInfo`

498 The visit info containing the time of the observation.

499

500 Returns

501 -------

502 alt : `lsst.geom.Angle`

503 The altitude.

504 az : `lsst.geom.Angle`

505 The azimuth.

506 """

507 skyLocation = SkyCoord(skyPos.getRa().asRadians(), skyPos.getDec().asRadians(), unit=u.rad)

508 long = visitInfo.observatory.getLongitude()

509 lat = visitInfo.observatory.getLatitude()

510 ele = visitInfo.observatory.getElevation()

511 earthLocation = EarthLocation.from_geodetic(long.asDegrees(), lat.asDegrees(), ele)

512 altAz = AltAz(obstime=visitInfo.date.toPython(), location=earthLocation)

513 obsAltAz = skyLocation.transform_to(altAz)

514 alt = geom.Angle(obsAltAz.alt.degree, geom.degrees)

515 az = geom.Angle(obsAltAz.az.degree, geom.degrees)

516

517 return alt, az

518

519

520def getExpPositionOffset(exp1, exp2, useWcs=True):

521 """Get the change in sky position between two exposures.

522

523 Given two exposures, calculate the offset on the sky between the images.

524 If useWcs then use the (fitted or unfitted) skyOrigin from their WCSs, and

525 calculate the alt/az from the observation times, otherwise use the nominal

526 values in the exposures' visitInfos. Note that if using the visitInfo

527 values that for a given pointing the ra/dec will be ~identical, regardless

528 of whether astrometric fitting has been performed.

529

530 Values are given as exp1-exp2.

531

532 Parameters

533 ----------

534 exp1 : `lsst.afw.image.Exposure`

535 The first exposure.

536 exp2 : `lsst.afw.image.Exposure`

537 The second exposure.

538 useWcs : `bool`

539 Use the WCS for the ra/dec and alt/az if True, else use the nominal/

540 boresight values from the exposures' visitInfos.

541

542 Returns

543 -------

544 offsets : `lsst.pipe.base.Struct`

545 A struct containing the offsets:

546 ``deltaRa``

547 The diference in ra (`lsst.geom.Angle`)

548 ``deltaDec``

549 The diference in dec (`lsst.geom.Angle`)

550 ``deltaAlt``

551 The diference in alt (`lsst.geom.Angle`)

552 ``deltaAz``

553 The diference in az (`lsst.geom.Angle`)

554 ``deltaPixels``

555 The diference in pixels (`float`)

556 """

557

558 wcs1 = exp1.getWcs()

559 wcs2 = exp2.getWcs()

560 pixScaleArcSec = wcs1.getPixelScale().asArcseconds()

561 assert np.isclose(pixScaleArcSec, wcs2.getPixelScale().asArcseconds()), \

562 "Pixel scales in the exposures differ."

563

564 if useWcs:

565 p1 = wcs1.getSkyOrigin()

566 p2 = wcs2.getSkyOrigin()

567 alt1, az1 = getAltAzFromSkyPosition(p1, exp1.getInfo().getVisitInfo())

568 alt2, az2 = getAltAzFromSkyPosition(p2, exp2.getInfo().getVisitInfo())

569 ra1 = p1[0]

570 ra2 = p2[0]

571 dec1 = p1[1]

572 dec2 = p2[1]

573 else:

574 az1 = exp1.visitInfo.boresightAzAlt[0]

575 az2 = exp2.visitInfo.boresightAzAlt[0]

576 alt1 = exp1.visitInfo.boresightAzAlt[1]

577 alt2 = exp2.visitInfo.boresightAzAlt[1]

578

579 ra1 = exp1.visitInfo.boresightRaDec[0]

580 ra2 = exp2.visitInfo.boresightRaDec[0]

581 dec1 = exp1.visitInfo.boresightRaDec[1]

582 dec2 = exp2.visitInfo.boresightRaDec[1]

583

584 p1 = exp1.visitInfo.boresightRaDec

585 p2 = exp2.visitInfo.boresightRaDec

586

587 angular_offset = p1.separation(p2).asArcseconds()

588 deltaPixels = angular_offset / pixScaleArcSec

589

590 ret = pipeBase.Struct(deltaRa=(ra1-ra2).wrapNear(geom.Angle(0.0)),

591 deltaDec=dec1-dec2,

592 deltaAlt=alt1-alt2,

593 deltaAz=(az1-az2).wrapNear(geom.Angle(0.0)),

594 deltaPixels=deltaPixels

595 )

596

597 return ret