Coverage for python/lsst/summit/utils/utils.py: 19%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22import os

23import numpy as np

24import logging

25from scipy.ndimage.filters import gaussian_filter

26import lsst.afw.image as afwImage

27import lsst.afw.detection as afwDetect

28import lsst.afw.math as afwMath

29import lsst.daf.base as dafBase

30import lsst.geom as geom

31import lsst.pipe.base as pipeBase

32import lsst.utils.packages as packageUtils

33from lsst.daf.butler.cli.cliLog import CliLog

34import datetime

35from dateutil.tz import gettz

37from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER

38from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION

40from astro_metadata_translator import ObservationInfo

41from astropy.coordinates import SkyCoord, AltAz

42from astropy.coordinates.earth import EarthLocation

43import astropy.units as u

45from .astrometry.utils import genericCameraHeaderToWcs

47__all__ = ["SIGMATOFWHM",

48 "FWHMTOSIGMA",

49 "EFD_CLIENT_MISSING_MSG",

50 "GOOGLE_CLOUD_MISSING_MSG",

51 "AUXTEL_LOCATION",

52 "countPixels",

53 "quickSmooth",

54 "argMax2d",

55 "getImageStats",

56 "detectObjectsInExp",

57 "humanNameForCelestialObject",

58 "getFocusFromHeader",

59 "dayObsIntToString",

60 "dayObsSeqNumToVisitId",

61 "setupLogging",

62 "getCurrentDayObs_datetime",

63 "getCurrentDayObs_int",

64 "getCurrentDayObs_humanStr",

65 "getSite",

66 "getExpPositionOffset",

67 "starTrackerFileToExposure",

68 ]

71SIGMATOFWHM = 2.0*np.sqrt(2.0*np.log(2.0))

72FWHMTOSIGMA = 1/SIGMATOFWHM

74EFD_CLIENT_MISSING_MSG = ('ImportError: lsst_efd_client not found. Please install with:\n'

75 ' pip install lsst-efd-client')

77GOOGLE_CLOUD_MISSING_MSG = ('ImportError: Google cloud storage not found. Please install with:\n'

78 ' pip install google-cloud-storage')

81def countPixels(maskedImage, maskPlane):

82 """Count the number of pixels in an image with a given mask bit set.

84 Parameters

85 ----------

86 maskedImage : `lsst.afw.image.MaskedImage`

87 The masked image,

88 maskPlane : `str`

89 The name of the bitmask.

91 Returns

92 -------

93 count : `int``

94 The number of pixels in with the selected mask bit

95 """

96 bit = maskedImage.mask.getPlaneBitMask(maskPlane)

97 return len(np.where(np.bitwise_and(maskedImage.mask.array, bit))[0])

100def quickSmooth(data, sigma=2):

101 """Perform a quick smoothing of the image.

102

103 Not to be used for scientific purposes, but improves the stretch and

104 visual rendering of low SNR against the sky background in cutouts.

105

106 Parameters

107 ----------

108 data : `np.array`

109 The image data to smooth

110 sigma : `float`, optional

111 The size of the smoothing kernel.

112

113 Returns

114 -------

115 smoothData : `np.array`

116 The smoothed data

117 """

118 kernel = [sigma, sigma]

119 smoothData = gaussian_filter(data, kernel, mode='constant')

120 return smoothData

121

122

123def argMax2d(array):

124 """Get the index of the max value of an array and whether it's unique.

125

126 If its not unique, returns a list of the other locations containing the

127 maximum value, e.g. returns

128

129 (12, 34), False, [(56,78), (910, 1112)]

130

131 Parameters

132 ----------

133 array : `np.array`

134 The data

135

136 Returns

137 -------

138 maxLocation : `tuple`

139 The coords of the first instance of the max value

140 unique : `bool`

141 Whether it's the only location

142 otherLocations : `list` of `tuple`

143 List of the other max values' locations, empty if False

144 """

145 uniqueMaximum = False

146 maxCoords = np.where(array == np.max(array))

147 maxCoords = [coord for coord in zip(*maxCoords)] # list of coords as tuples

148 if len(maxCoords) == 1: # single unambiguous value

149 uniqueMaximum = True

150

151 return maxCoords[0], uniqueMaximum, maxCoords[1:]

152

153

154def dayObsIntToString(dayObs):

155 """Convert an integer dayObs to a dash-delimited string.

156

157 e.g. convert the hard to read 20210101 to 2021-01-01

158

159 Parameters

160 ----------

161 dayObs : `int`

162 The dayObs.

163

164 Returns

165 -------

166 dayObs : `str`

167 The dayObs as a string.

168 """

169 assert isinstance(dayObs, int)

170 dStr = str(dayObs)

171 assert len(dStr) == 8

172 return '-'.join([dStr[0:4], dStr[4:6], dStr[6:8]])

173

174

175def dayObsSeqNumToVisitId(dayObs, seqNum):

176 """Get the visit id for a given dayObs/seqNum.

177

178 Parameters

179 ----------

180 dayObs : `int`

181 The dayObs.

182 seqNum : `int`

183 The seqNum.

184

185 Returns

186 -------

187 visitId : `int`

188 The visitId.

189

190 Notes

191 -----

192 TODO: Remove this horrible hack once DM-30948 makes this possible

193 programatically/via the butler.

194 """

195 if dayObs < 19700101 or dayObs > 35000101:

196 raise ValueError(f'dayObs value {dayObs} outside plausible range')

197 return int(f"{dayObs}{seqNum:05}")

198

199

200def getImageStats(exp):

201 """Calculate a grab-bag of stats for an image. Must remain fast.

202

203 Parameters

204 ----------

205 exp : `lsst.afw.image.Exposure`

206 The input exposure.

207

208 Returns

209 -------

210 stats : `lsst.pipe.base.Struct`

211 A container with attributes containing measurements and statistics

212 for the image.

213 """

214 result = pipeBase.Struct()

215

216 vi = exp.visitInfo

217 expTime = vi.exposureTime

218 md = exp.getMetadata()

219

220 obj = vi.object

221 mjd = vi.getDate().get()

222 result.object = obj

223 result.mjd = mjd

224

225 fullFilterString = exp.filter.physicalLabel

226 filt = fullFilterString.split(FILTER_DELIMITER)[0]

227 grating = fullFilterString.split(FILTER_DELIMITER)[1]

228

229 airmass = vi.getBoresightAirmass()

230 rotangle = vi.getBoresightRotAngle().asDegrees()

231

232 azAlt = vi.getBoresightAzAlt()

233 az = azAlt[0].asDegrees()

234 el = azAlt[1].asDegrees()

235

236 result.expTime = expTime

237 result.filter = filt

238 result.grating = grating

239 result.airmass = airmass

240 result.rotangle = rotangle

241 result.az = az

242 result.el = el

243 result.focus = md.get('FOCUSZ')

244

245 data = exp.image.array

246 result.maxValue = np.max(data)

247

248 peak, uniquePeak, otherPeaks = argMax2d(data)

249 result.maxPixelLocation = peak

250 result.multipleMaxPixels = uniquePeak

251

252 result.nBadPixels = countPixels(exp.maskedImage, 'BAD')

253 result.nSatPixels = countPixels(exp.maskedImage, 'SAT')

254 result.percentile99 = np.percentile(data, 99)

255 result.percentile9999 = np.percentile(data, 99.99)

256

257 sctrl = afwMath.StatisticsControl()

258 sctrl.setNumSigmaClip(5)

259 sctrl.setNumIter(2)

260 statTypes = afwMath.MEANCLIP | afwMath.STDEVCLIP

261 stats = afwMath.makeStatistics(exp.maskedImage, statTypes, sctrl)

262 std, stderr = stats.getResult(afwMath.STDEVCLIP)

263 mean, meanerr = stats.getResult(afwMath.MEANCLIP)

264

265 result.clippedMean = mean

266 result.clippedStddev = std

267

268 return result

269

270

271def detectObjectsInExp(exp, nSigma=10, nPixMin=10, grow=0):

272 """Quick and dirty object detection for an expsure.

273

274 Return the footPrintSet for the objects in a preferably-postISR exposure.

275

276 Parameters

277 ----------

278 exp : `lsst.afw.image.Exposure`

279 The exposure to detect objects in.

280 nSigma : `float`

281 The number of sigma for detection.

282 nPixMin : `int`

283 The minimum number of pixels in an object for detection.

284 grow : `int`

285 The number of pixels to grow the footprint by after detection.

286

287 Returns

288 -------

289 footPrintSet : `lsst.afw.detection.FootprintSet`

290 The set of footprints in the image.

291 """

292 median = np.nanmedian(exp.image.array)

293 exp.image -= median

294

295 threshold = afwDetect.Threshold(nSigma, afwDetect.Threshold.STDEV)

296 footPrintSet = afwDetect.FootprintSet(exp.getMaskedImage(), threshold, "DETECTED", nPixMin)

297 if grow > 0:

298 isotropic = True

299 footPrintSet = afwDetect.FootprintSet(footPrintSet, grow, isotropic)

300

301 exp.image += median # add back in to leave background unchanged

302 return footPrintSet

303

304

305def humanNameForCelestialObject(objName):

306 """Returns a list of all human names for obj, or [] if none are found.

307

308 Parameters

309 ----------

310 objName : `str`

311 The/a name of the object.

312

313 Returns

314 -------

315 names : `list` of `str`

316 The names found for the object

317 """

318 from astroquery.simbad import Simbad

319 results = []

320 try:

321 simbadResult = Simbad.query_objectids(objName)

322 for row in simbadResult:

323 if row['ID'].startswith('NAME'):

324 results.append(row['ID'].replace('NAME ', ''))

325 return results

326 except Exception:

327 return [] # same behavior as for found but un-named objects

328

329

330def _getAltAzZenithsFromSeqNum(butler, dayObs, seqNumList):

331 """Get the alt, az and zenith angle for the seqNums of a given dayObs.

332

333 Parameters

334 ----------

335 butler : `lsst.daf.butler.Butler`

336 The butler to query.

337 dayObs : `int`

338 The dayObs.

339 seqNumList : `list` of `int`

340 The seqNums for which to return the alt, az and zenith

341

342 Returns

343 -------

344 azimuths : `list` of `float`

345 List of the azimuths for each seqNum

346 elevations : `list` of `float`

347 List of the elevations for each seqNum

348 zeniths : `list` of `float`

349 List of the zenith angles for each seqNum

350 """

351 azimuths, elevations, zeniths = [], [], []

352 for seqNum in seqNumList:

353 md = butler.get('raw.metadata', day_obs=dayObs, seq_num=seqNum, detector=0)

354 obsInfo = ObservationInfo(md)

355 alt = obsInfo.altaz_begin.alt.value

356 az = obsInfo.altaz_begin.az.value

357 elevations.append(alt)

358 zeniths.append(90-alt)

359 azimuths.append(az)

360 return azimuths, elevations, zeniths

361

362

363def getFocusFromHeader(exp):

364 """Get the raw focus value from the header.

365

366 Parameters

367 ----------

368 exp : `lsst.afw.image.exposure`

369 The exposure.

370

371 Returns

372 -------

373 focus : `float` or `None`

374 The focus value if found, else ``None``.

375 """

376 md = exp.getMetadata()

377 if 'FOCUSZ' in md:

378 return md['FOCUSZ']

379 return None

380

381

382def checkStackSetup():

383 """Check which weekly tag is being used and which local packages are setup.

384

385 Designed primarily for use in notbooks/observing, this prints the weekly

386 tag(s) are setup for lsst_distrib, and lists any locally setup packages and

387 the path to each.

388

389 Notes

390 -----

391 Uses print() instead of logger messages as this should simply print them

392 without being vulnerable to any log messages potentially being diverted.

393 """

394 packages = packageUtils.getEnvironmentPackages(include_all=True)

395

396 lsstDistribHashAndTags = packages['lsst_distrib'] # looks something like 'g4eae7cb9+1418867f (w_2022_13)'

397 lsstDistribTags = lsstDistribHashAndTags.split()[1]

398 if len(lsstDistribTags.split()) == 1:

399 tag = lsstDistribTags.replace('(', '')

400 tag = tag.replace(')', '')

401 print(f"You are running {tag} of lsst_distrib")

402 else: # multiple weekly tags found for lsst_distrib!

403 print(f'The version of lsst_distrib you have is compatible with: {lsstDistribTags}')

404

405 localPackages = []

406 localPaths = []

407 for package, tags in packages.items():

408 if tags.startswith('LOCAL:'):

409 path = tags.split('LOCAL:')[1]

410 path = path.split('@')[0] # don't need the git SHA etc

411 localPaths.append(path)

412 localPackages.append(package)

413

414 if localPackages:

415 print("\nLocally setup packages:")

416 print("-----------------------")

417 maxLen = max(len(package) for package in localPackages)

418 for package, path in zip(localPackages, localPaths):

419 print(f"{package:<{maxLen}s} at {path}")

420 else:

421 print("\nNo locally setup packages (using a vanilla stack)")

422

423

424def setupLogging(longlog=False):

425 """Setup logging in the same way as one would get from pipetask run.

426

427 Code that isn't run through the butler CLI defaults to WARNING level

428 messages and no logger names. This sets the behaviour to follow whatever

429 the pipeline default is, currently

430 <logger_name> <level>: <message> e.g.

431 lsst.isr INFO: Masking defects.

432 """

433 CliLog.initLog(longlog=longlog)

434

435

436def getCurrentDayObs_datetime():

437 """Get the current day_obs - the observatory rolls the date over at UTC-12

438

439 Returned as datetime.date(2022, 4, 28)

440 """

441 utc = gettz("UTC")

442 nowUtc = datetime.datetime.now().astimezone(utc)

443 offset = datetime.timedelta(hours=-12)

444 dayObs = (nowUtc + offset).date()

445 return dayObs

446

447

448def getCurrentDayObs_int():

449 """Return the current dayObs as an int in the form 20220428

450 """

451 return int(getCurrentDayObs_datetime().strftime("%Y%m%d"))

452

453

454def getCurrentDayObs_humanStr():

455 """Return the current dayObs as a string in the form '2022-04-28'

456 """

457 return dayObsIntToString(getCurrentDayObs_int())

458

459

460def getSite():

461 """Returns where the code is running.

462

463 Returns

464 -------

465 location : `str`

466 One of ['tucson', 'summit', 'base', 'staff-rsp', 'rubin-devl']

467

468 Raises

469 ------

470 ValueError

471 Raised if location cannot be determined.

472 """

473 # All nublado instances guarantee that EXTERNAL_URL is set and uniquely

474 # identifies it.

475 location = os.getenv('EXTERNAL_URL', "")

476 if location == "https://tucson-teststand.lsst.codes": 476 ↛ 477line 476 didn't jump to line 477, because the condition on line 476 was never true

477 return 'tucson'

478 elif location == "https://summit-lsp.lsst.codes": 478 ↛ 479line 478 didn't jump to line 479, because the condition on line 478 was never true

479 return 'summit'

480 elif location == "https://base-lsp.lsst.codes": 480 ↛ 481line 480 didn't jump to line 481, because the condition on line 480 was never true

481 return 'base'

482 elif location == "https://usdf-rsp.slac.stanford.edu": 482 ↛ 483line 482 didn't jump to line 483, because the condition on line 482 was never true

483 return 'staff-rsp'

484

485 # if no EXTERNAL_URL, try HOSTNAME to see if we're on the dev nodes

486 # it is expected that this will be extensible to SLAC

487 hostname = os.getenv('HOSTNAME', "")

488 if hostname.startswith('sdfrome'): 488 ↛ 489line 488 didn't jump to line 489, because the condition on line 488 was never true

489 return 'rubin-devl'

490

491 # we have failed

492 raise ValueError('Location could not be determined')

493

494

495def getAltAzFromSkyPosition(skyPos, visitInfo):

496 """Get the alt/az from the position on the sky and the time and location

497 of the observation.

498

499 Parameters

500 ----------

501 skyPos : `lsst.geom.SpherePoint`

502 The position on the sky.

503 visitInfo : `lsst.afw.image.VisitInfo`

504 The visit info containing the time of the observation.

505

506 Returns

507 -------

508 alt : `lsst.geom.Angle`

509 The altitude.

510 az : `lsst.geom.Angle`

511 The azimuth.

512 """

513 skyLocation = SkyCoord(skyPos.getRa().asRadians(), skyPos.getDec().asRadians(), unit=u.rad)

514 long = visitInfo.observatory.getLongitude()

515 lat = visitInfo.observatory.getLatitude()

516 ele = visitInfo.observatory.getElevation()

517 earthLocation = EarthLocation.from_geodetic(long.asDegrees(), lat.asDegrees(), ele)

518 altAz = AltAz(obstime=visitInfo.date.toPython(), location=earthLocation)

519 obsAltAz = skyLocation.transform_to(altAz)

520 alt = geom.Angle(obsAltAz.alt.degree, geom.degrees)

521 az = geom.Angle(obsAltAz.az.degree, geom.degrees)

522

523 return alt, az

524

525

526def getExpPositionOffset(exp1, exp2, useWcs=True, allowDifferentPlateScales=False):

527 """Get the change in sky position between two exposures.

528

529 Given two exposures, calculate the offset on the sky between the images.

530 If useWcs then use the (fitted or unfitted) skyOrigin from their WCSs, and

531 calculate the alt/az from the observation times, otherwise use the nominal

532 values in the exposures' visitInfos. Note that if using the visitInfo

533 values that for a given pointing the ra/dec will be ~identical, regardless

534 of whether astrometric fitting has been performed.

535

536 Values are given as exp1-exp2.

537

538 Parameters

539 ----------

540 exp1 : `lsst.afw.image.Exposure`

541 The first exposure.

542 exp2 : `lsst.afw.image.Exposure`

543 The second exposure.

544 useWcs : `bool`

545 Use the WCS for the ra/dec and alt/az if True, else use the nominal/

546 boresight values from the exposures' visitInfos.

547 allowDifferentPlateScales : `bool`, optional

548 Use to disable checking that plate scales are the same. Generally,

549 differing plate scales would indicate an error, but where blind-solving

550 has been undertaken during commissioning plate scales can be different

551 enough to warrant setting this to ``True``.

552

553 Returns

554 -------

555 offsets : `lsst.pipe.base.Struct`

556 A struct containing the offsets:

557 ``deltaRa``

558 The diference in ra (`lsst.geom.Angle`)

559 ``deltaDec``

560 The diference in dec (`lsst.geom.Angle`)

561 ``deltaAlt``

562 The diference in alt (`lsst.geom.Angle`)

563 ``deltaAz``

564 The diference in az (`lsst.geom.Angle`)

565 ``deltaPixels``

566 The diference in pixels (`float`)

567 """

568

569 wcs1 = exp1.getWcs()

570 wcs2 = exp2.getWcs()

571 pixScaleArcSec = wcs1.getPixelScale().asArcseconds()

572 if not allowDifferentPlateScales:

573 assert np.isclose(pixScaleArcSec, wcs2.getPixelScale().asArcseconds()), \

574 "Pixel scales in the exposures differ."

575

576 if useWcs:

577 p1 = wcs1.getSkyOrigin()

578 p2 = wcs2.getSkyOrigin()

579 alt1, az1 = getAltAzFromSkyPosition(p1, exp1.getInfo().getVisitInfo())

580 alt2, az2 = getAltAzFromSkyPosition(p2, exp2.getInfo().getVisitInfo())

581 ra1 = p1[0]

582 ra2 = p2[0]

583 dec1 = p1[1]

584 dec2 = p2[1]

585 else:

586 az1 = exp1.visitInfo.boresightAzAlt[0]

587 az2 = exp2.visitInfo.boresightAzAlt[0]

588 alt1 = exp1.visitInfo.boresightAzAlt[1]

589 alt2 = exp2.visitInfo.boresightAzAlt[1]

590

591 ra1 = exp1.visitInfo.boresightRaDec[0]

592 ra2 = exp2.visitInfo.boresightRaDec[0]

593 dec1 = exp1.visitInfo.boresightRaDec[1]

594 dec2 = exp2.visitInfo.boresightRaDec[1]

595

596 p1 = exp1.visitInfo.boresightRaDec

597 p2 = exp2.visitInfo.boresightRaDec

598

599 angular_offset = p1.separation(p2).asArcseconds()

600 deltaPixels = angular_offset / pixScaleArcSec

601

602 ret = pipeBase.Struct(deltaRa=(ra1-ra2).wrapNear(geom.Angle(0.0)),

603 deltaDec=dec1-dec2,

604 deltaAlt=alt1-alt2,

605 deltaAz=(az1-az2).wrapNear(geom.Angle(0.0)),

606 deltaPixels=deltaPixels

607 )

608

609 return ret

610

611

612def starTrackerFileToExposure(filename, logger=None):

613 """Read the exposure from the file and set the wcs from the header.

614

615 Parameters

616 ----------

617 filename : `str`

618 The full path to the file.

619 logger : `logging.Logger`, optional

620 The logger to use for errors, created if not supplied.

621

622 Returns

623 -------

624 exp : `lsst.afw.image.Exposure`

625 The exposure.

626 """

627 if not logger:

628 logger = logging.getLogger(__name__)

629 exp = afwImage.ExposureF(filename)

630 try:

631 wcs = genericCameraHeaderToWcs(exp)

632 exp.setWcs(wcs)

633 except Exception as e:

634 logger.warning(f"Failed to set wcs from header: {e}")

635

636 # for some reason the date isn't being set correctly

637 # DATE-OBS is present in the original header, but it's being

638 # stripped out and somehow not set (plus it doesn't give the midpoint

639 # of the exposure), so set it manually from the midpoint here

640 try:

641 md = exp.getMetadata()

642 begin = datetime.datetime.fromisoformat(md['DATE-BEG'])

643 end = datetime.datetime.fromisoformat(md['DATE-END'])

644 duration = end - begin

645 mid = begin + duration/2

646 newTime = dafBase.DateTime(mid.isoformat(), dafBase.DateTime.Timescale.TAI)

647 newVi = exp.visitInfo.copyWith(date=newTime)

648 exp.info.setVisitInfo(newVi)

649 except Exception as e:

650 logger.warning(f"Failed to set date from header: {e}")

651

652 return exp

653

654

655def obsInfoToDict(obsInfo):

656 """Convert an ObservationInfo to a dict.

657

658 Parameters

659 ----------

660 obsInfo : `astro_metadata_translator.ObservationInfo`

661 The ObservationInfo to convert.

662

663 Returns

664 -------

665 obsInfoDict : `dict`

666 The ObservationInfo as a dict.

667 """

668 return {prop: getattr(obsInfo, prop) for prop in obsInfo.all_properties.keys()}

669

670

671def getFieldNameAndTileNumber(field, warn=True, logger=None):

672 """Get the tile name and number of an observed field.

673

674 It is assumed to always be appended, with an underscore, to the rest of the

675 field name. Returns the name and number as a tuple, or the name unchanged

676 if no tile number is found.

677

678 Parameters

679 ----------

680 field : `str`

681 The name of the field

682

683 Returns

684 -------

685 fieldName : `str`

686 The name of the field without the trailing tile number, if present.

687 tileNum : `int`

688 The number of the tile, as an integer, or ``None`` if not found.

689 """

690 if warn and not logger:

691 logger = logging.getLogger('lsst.summit.utils.utils.getFieldNameAndTileNumber')

692

693 if '_' not in field:

694 if warn:

695 logger.warning(f"Field {field} does not contain an underscore,"

696 " so cannot determine the tile number.")

697 return field, None

698

699 try:

700 fieldParts = field.split("_")

701 fieldNum = int(fieldParts[-1])

702 except ValueError:

703 if warn:

704 logger.warning(f"Field {field} does not contain only an integer after the final underscore"

705 " so cannot determine the tile number.")

706 return field, None

707

708 return "_".join(fieldParts[:-1]), fieldNum