Coverage for python/lsst/pipe/tasks/processBrightStars.py: 23%

177 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 10:20 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Extract bright star cutouts; normalize and warp to the same pixel grid.""" 

23 

24__all__ = ["ProcessBrightStarsTask"] 

25 

26import astropy.units as u 

27import numpy as np 

28from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS 

29from lsst.afw.detection import FootprintSet, Threshold 

30from lsst.afw.geom.transformFactory import makeIdentityTransform, makeTransform 

31from lsst.afw.image import Exposure, ExposureF, MaskedImageF 

32from lsst.afw.math import ( 

33 StatisticsControl, 

34 WarpingControl, 

35 rotateImageBy90, 

36 stringToStatisticsProperty, 

37 warpImage, 

38) 

39from lsst.geom import AffineTransform, Box2I, Extent2I, Point2D, Point2I, SpherePoint, radians 

40from lsst.meas.algorithms import LoadReferenceObjectsConfig, ReferenceObjectLoader 

41from lsst.meas.algorithms.brightStarStamps import BrightStarStamp, BrightStarStamps 

42from lsst.pex.config import ChoiceField, ConfigField, Field, ListField 

43from lsst.pex.exceptions import InvalidParameterError 

44from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct 

45from lsst.pipe.base.connectionTypes import Input, Output, PrerequisiteInput 

46from lsst.utils.timer import timeMethod 

47 

48 

49class ProcessBrightStarsConnections(PipelineTaskConnections, dimensions=("instrument", "visit", "detector")): 

50 """Connections for ProcessBrightStarsTask.""" 

51 

52 inputExposure = Input( 

53 doc="Input exposure from which to extract bright star stamps", 

54 name="calexp", 

55 storageClass="ExposureF", 

56 dimensions=("visit", "detector"), 

57 ) 

58 skyCorr = Input( 

59 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

60 name="skyCorr", 

61 storageClass="Background", 

62 dimensions=("instrument", "visit", "detector"), 

63 ) 

64 refCat = PrerequisiteInput( 

65 doc="Reference catalog that contains bright star positions", 

66 name="gaia_dr2_20200414", 

67 storageClass="SimpleCatalog", 

68 dimensions=("skypix",), 

69 multiple=True, 

70 deferLoad=True, 

71 ) 

72 brightStarStamps = Output( 

73 doc="Set of preprocessed postage stamps, each centered on a single bright star.", 

74 name="brightStarStamps", 

75 storageClass="BrightStarStamps", 

76 dimensions=("visit", "detector"), 

77 ) 

78 

79 def __init__(self, *, config=None): 

80 super().__init__(config=config) 

81 if not config.doApplySkyCorr: 

82 self.inputs.remove("skyCorr") 

83 

84 

85class ProcessBrightStarsConfig(PipelineTaskConfig, pipelineConnections=ProcessBrightStarsConnections): 

86 """Configuration parameters for ProcessBrightStarsTask.""" 

87 

88 magLimit = Field( 

89 dtype=float, 

90 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed.", 

91 default=18, 

92 ) 

93 stampSize = ListField( 

94 dtype=int, 

95 doc="Size of the stamps to be extracted, in pixels.", 

96 default=(250, 250), 

97 ) 

98 modelStampBuffer = Field( 

99 dtype=float, 

100 doc=( 

101 "'Buffer' factor to be applied to determine the size of the stamp the processed stars will be " 

102 "saved in. This will also be the size of the extended PSF model." 

103 ), 

104 default=1.1, 

105 ) 

106 doRemoveDetected = Field( 

107 dtype=bool, 

108 doc="Whether DETECTION footprints, other than that for the central object, should be changed to BAD.", 

109 default=True, 

110 ) 

111 doApplyTransform = Field( 

112 dtype=bool, 

113 doc="Apply transform to bright star stamps to correct for optical distortions?", 

114 default=True, 

115 ) 

116 warpingKernelName = ChoiceField( 

117 dtype=str, 

118 doc="Warping kernel", 

119 default="lanczos5", 

120 allowed={ 

121 "bilinear": "bilinear interpolation", 

122 "lanczos3": "Lanczos kernel of order 3", 

123 "lanczos4": "Lanczos kernel of order 4", 

124 "lanczos5": "Lanczos kernel of order 5", 

125 }, 

126 ) 

127 annularFluxRadii = ListField( 

128 dtype=int, 

129 doc="Inner and outer radii of the annulus used to compute AnnularFlux for normalization, in pixels.", 

130 default=(70, 80), 

131 ) 

132 annularFluxStatistic = ChoiceField( 

133 dtype=str, 

134 doc="Type of statistic to use to compute annular flux.", 

135 default="MEANCLIP", 

136 allowed={ 

137 "MEAN": "mean", 

138 "MEDIAN": "median", 

139 "MEANCLIP": "clipped mean", 

140 }, 

141 ) 

142 numSigmaClip = Field( 

143 dtype=float, 

144 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

145 default=4, 

146 ) 

147 numIter = Field( 

148 dtype=int, 

149 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

150 default=3, 

151 ) 

152 badMaskPlanes = ListField( 

153 dtype=str, 

154 doc="Mask planes that identify pixels to not include in the computation of the annular flux.", 

155 default=("BAD", "CR", "CROSSTALK", "EDGE", "NO_DATA", "SAT", "SUSPECT", "UNMASKEDNAN"), 

156 ) 

157 minValidAnnulusFraction = Field( 

158 dtype=float, 

159 doc="Minumum number of valid pixels that must fall within the annulus for the bright star to be " 

160 "saved for subsequent generation of a PSF.", 

161 default=0.0, 

162 ) 

163 doApplySkyCorr = Field( 

164 dtype=bool, 

165 doc="Apply full focal plane sky correction before extracting stars?", 

166 default=True, 

167 ) 

168 discardNanFluxStars = Field( 

169 dtype=bool, 

170 doc="Should stars with NaN annular flux be discarded?", 

171 default=False, 

172 ) 

173 refObjLoader = ConfigField( 

174 dtype=LoadReferenceObjectsConfig, 

175 doc="Reference object loader for astrometric calibration.", 

176 ) 

177 

178 

179class ProcessBrightStarsTask(PipelineTask): 

180 """The description of the parameters for this Task are detailed in 

181 :lsst-task:`~lsst.pipe.base.PipelineTask`. 

182 

183 Parameters 

184 ---------- 

185 initInputs : `Unknown` 

186 *args 

187 Additional positional arguments. 

188 **kwargs 

189 Additional keyword arguments. 

190 

191 Notes 

192 ----- 

193 `ProcessBrightStarsTask` is used to extract, process, and store small 

194 image cut-outs (or "postage stamps") around bright stars. It relies on 

195 three methods, called in succession: 

196 

197 `extractStamps` 

198 Find bright stars within the exposure using a reference catalog and 

199 extract a stamp centered on each. 

200 `warpStamps` 

201 Shift and warp each stamp to remove optical distortions and sample all 

202 stars on the same pixel grid. 

203 `measureAndNormalize` 

204 Compute the flux of an object in an annulus and normalize it. This is 

205 required to normalize each bright star stamp as their central pixels 

206 are likely saturated and/or contain ghosts, and cannot be used. 

207 """ 

208 

209 ConfigClass = ProcessBrightStarsConfig 

210 _DefaultName = "processBrightStars" 

211 

212 def __init__(self, initInputs=None, *args, **kwargs): 

213 super().__init__(*args, **kwargs) 

214 # Compute (model) stamp size depending on provided "buffer" value 

215 self.modelStampSize = [ 

216 int(self.config.stampSize[0] * self.config.modelStampBuffer), 

217 int(self.config.stampSize[1] * self.config.modelStampBuffer), 

218 ] 

219 # force it to be odd-sized so we have a central pixel 

220 if not self.modelStampSize[0] % 2: 

221 self.modelStampSize[0] += 1 

222 if not self.modelStampSize[1] % 2: 

223 self.modelStampSize[1] += 1 

224 # central pixel 

225 self.modelCenter = self.modelStampSize[0] // 2, self.modelStampSize[1] // 2 

226 

227 def applySkyCorr(self, calexp, skyCorr): 

228 """Apply correction to the sky background level. 

229 

230 Sky corrections can be generated using the ``SkyCorrectionTask``. 

231 As the sky model generated there extends over the full focal plane, 

232 this should produce a more optimal sky subtraction solution. 

233 

234 Parameters 

235 ---------- 

236 calexp : `~lsst.afw.image.Exposure` or `~lsst.afw.image.MaskedImage` 

237 Calibrated exposure. 

238 skyCorr : `~lsst.afw.math.backgroundList.BackgroundList`, optional 

239 Full focal plane sky correction from ``SkyCorrectionTask``. 

240 

241 Notes 

242 ----- 

243 This method modifies the input ``calexp`` in-place. 

244 """ 

245 if isinstance(calexp, Exposure): 

246 calexp = calexp.getMaskedImage() 

247 calexp -= skyCorr.getImage() 

248 

249 def extractStamps(self, inputExposure, refObjLoader=None): 

250 """Read the position of bright stars within an input exposure using a 

251 refCat and extract them. 

252 

253 Parameters 

254 ---------- 

255 inputExposure : `~lsst.afw.image.ExposureF` 

256 The image from which bright star stamps should be extracted. 

257 refObjLoader : `~lsst.meas.algorithms.ReferenceObjectLoader`, optional 

258 Loader to find objects within a reference catalog. 

259 

260 Returns 

261 ------- 

262 result : `~lsst.pipe.base.Struct` 

263 Results as a struct with attributes: 

264 

265 ``starIms`` 

266 Postage stamps (`list`). 

267 ``pixCenters`` 

268 Corresponding coords to each star's center, in pixels (`list`). 

269 ``GMags`` 

270 Corresponding (Gaia) G magnitudes (`list`). 

271 ``gaiaIds`` 

272 Corresponding unique Gaia identifiers (`np.ndarray`). 

273 """ 

274 if refObjLoader is None: 

275 refObjLoader = self.refObjLoader 

276 starIms = [] 

277 pixCenters = [] 

278 GMags = [] 

279 ids = [] 

280 wcs = inputExposure.getWcs() 

281 # select stars within, or close enough to input exposure from refcat 

282 inputIm = inputExposure.maskedImage 

283 inputExpBBox = inputExposure.getBBox() 

284 # Attempt to include stars that are outside of the exposure but their 

285 # stamps overlap with the exposure. 

286 dilatationExtent = Extent2I(np.array(self.config.stampSize) // 2) 

287 # TODO (DM-25894): handle catalog with stars missing from Gaia 

288 withinCalexp = refObjLoader.loadPixelBox( 

289 inputExpBBox.dilatedBy(dilatationExtent), 

290 wcs, 

291 filterName="phot_g_mean", 

292 ) 

293 refCat = withinCalexp.refCat 

294 # keep bright objects 

295 fluxLimit = ((self.config.magLimit * u.ABmag).to(u.nJy)).to_value() 

296 GFluxes = np.array(refCat["phot_g_mean_flux"]) 

297 bright = GFluxes > fluxLimit 

298 # convert to AB magnitudes 

299 allGMags = [((gFlux * u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]] 

300 allIds = refCat.columns.extract("id", where=bright)["id"] 

301 selectedColumns = refCat.columns.extract("coord_ra", "coord_dec", where=bright) 

302 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])): 

303 sp = SpherePoint(ra, dec, radians) 

304 cpix = wcs.skyToPixel(sp) 

305 try: 

306 starIm = inputExposure.getCutout(sp, Extent2I(self.config.stampSize)) 

307 except InvalidParameterError: 

308 # star is beyond boundary 

309 bboxCorner = np.array(cpix) - np.array(self.config.stampSize) / 2 

310 # compute bbox as it would be otherwise 

311 idealBBox = Box2I(Point2I(bboxCorner), Extent2I(self.config.stampSize)) 

312 clippedStarBBox = Box2I(idealBBox) 

313 clippedStarBBox.clip(inputExpBBox) 

314 if clippedStarBBox.getArea() > 0: 

315 # create full-sized stamp with all pixels 

316 # flagged as NO_DATA 

317 starIm = ExposureF(bbox=idealBBox) 

318 starIm.image[:] = np.nan 

319 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA")) 

320 # recover pixels from intersection with the exposure 

321 clippedIm = inputIm.Factory(inputIm, clippedStarBBox) 

322 starIm.maskedImage[clippedStarBBox] = clippedIm 

323 # set detector and wcs, used in warpStars 

324 starIm.setDetector(inputExposure.getDetector()) 

325 starIm.setWcs(inputExposure.getWcs()) 

326 else: 

327 continue 

328 if self.config.doRemoveDetected: 

329 # give detection footprint of other objects the BAD flag 

330 detThreshold = Threshold(starIm.mask.getPlaneBitMask("DETECTED"), Threshold.BITMASK) 

331 omask = FootprintSet(starIm.mask, detThreshold) 

332 allFootprints = omask.getFootprints() 

333 otherFootprints = [] 

334 for fs in allFootprints: 

335 if not fs.contains(Point2I(cpix)): 

336 otherFootprints.append(fs) 

337 nbMatchingFootprints = len(allFootprints) - len(otherFootprints) 

338 if not nbMatchingFootprints == 1: 

339 self.log.warning( 

340 "Failed to uniquely identify central DETECTION footprint for star " 

341 "%s; found %d footprints instead.", 

342 allIds[j], 

343 nbMatchingFootprints, 

344 ) 

345 omask.setFootprints(otherFootprints) 

346 omask.setMask(starIm.mask, "BAD") 

347 starIms.append(starIm) 

348 pixCenters.append(cpix) 

349 GMags.append(allGMags[j]) 

350 ids.append(allIds[j]) 

351 return Struct(starIms=starIms, pixCenters=pixCenters, GMags=GMags, gaiaIds=ids) 

352 

353 def warpStamps(self, stamps, pixCenters): 

354 """Warps and shifts all given stamps so they are sampled on the same 

355 pixel grid and centered on the central pixel. This includes rotating 

356 the stamp depending on detector orientation. 

357 

358 Parameters 

359 ---------- 

360 stamps : `Sequence` [`~lsst.afw.image.ExposureF`] 

361 Image cutouts centered on a single object. 

362 pixCenters : `Sequence` [`~lsst.geom.Point2D`] 

363 Positions of each object's center (from the refCat) in pixels. 

364 

365 Returns 

366 ------- 

367 result : `~lsst.pipe.base.Struct` 

368 Results as a struct with attributes: 

369 

370 ``warpedStars`` 

371 Stamps of warped stars. 

372 (`list` [`~lsst.afw.image.MaskedImage`]) 

373 ``warpTransforms`` 

374 The corresponding Transform from the initial star stamp 

375 to the common model grid. 

376 (`list` [`~lsst.afw.geom.TransformPoint2ToPoint2`]) 

377 ``xy0s`` 

378 Coordinates of the bottom-left pixels of each stamp, 

379 before rotation. 

380 (`list` [`~lsst.geom.Point2I`]) 

381 ``nb90Rots`` 

382 The number of 90 degrees rotations required to compensate for 

383 detector orientation. 

384 (`int`) 

385 """ 

386 # warping control; only contains shiftingALg provided in config 

387 warpCont = WarpingControl(self.config.warpingKernelName) 

388 # Compare model to star stamp sizes 

389 bufferPix = ( 

390 self.modelStampSize[0] - self.config.stampSize[0], 

391 self.modelStampSize[1] - self.config.stampSize[1], 

392 ) 

393 # Initialize detector instance (note all stars were extracted from an 

394 # exposure from the same detector) 

395 det = stamps[0].getDetector() 

396 # Define correction for optical distortions 

397 if self.config.doApplyTransform: 

398 pixToTan = det.getTransform(PIXELS, TAN_PIXELS) 

399 else: 

400 pixToTan = makeIdentityTransform() 

401 # Array of all possible rotations for detector orientation: 

402 possibleRots = np.array([k * np.pi / 2 for k in range(4)]) 

403 # determine how many, if any, rotations are required 

404 yaw = det.getOrientation().getYaw() 

405 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw))) 

406 

407 # apply transformation to each star 

408 warpedStars, warpTransforms, xy0s = [], [], [] 

409 for star, cent in zip(stamps, pixCenters): 

410 # (re)create empty destination image 

411 destImage = MaskedImageF(*self.modelStampSize) 

412 bottomLeft = Point2D(star.image.getXY0()) 

413 newBottomLeft = pixToTan.applyForward(bottomLeft) 

414 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0] / 2) 

415 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1] / 2) 

416 # Convert to int 

417 newBottomLeft = Point2I(newBottomLeft) 

418 # Set origin and save it 

419 destImage.setXY0(newBottomLeft) 

420 xy0s.append(newBottomLeft) 

421 

422 # Define linear shifting to recenter stamps 

423 newCenter = pixToTan.applyForward(cent) # center of warped star 

424 shift = ( 

425 self.modelCenter[0] + newBottomLeft[0] - newCenter[0], 

426 self.modelCenter[1] + newBottomLeft[1] - newCenter[1], 

427 ) 

428 affineShift = AffineTransform(shift) 

429 shiftTransform = makeTransform(affineShift) 

430 

431 # Define full transform (warp and shift) 

432 starWarper = pixToTan.then(shiftTransform) 

433 

434 # Apply it 

435 goodPix = warpImage(destImage, star.getMaskedImage(), starWarper, warpCont) 

436 if not goodPix: 

437 self.log.debug("Warping of a star failed: no good pixel in output") 

438 

439 # Arbitrarily set origin of shifted star to 0 

440 destImage.setXY0(0, 0) 

441 

442 # Apply rotation if appropriate 

443 if nb90Rots: 

444 destImage = rotateImageBy90(destImage, nb90Rots) 

445 warpedStars.append(destImage.clone()) 

446 warpTransforms.append(starWarper) 

447 return Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s, nb90Rots=nb90Rots) 

448 

449 @timeMethod 

450 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None): 

451 """Identify bright stars within an exposure using a reference catalog, 

452 extract stamps around each, then preprocess them. The preprocessing 

453 steps are: shifting, warping and potentially rotating them to the same 

454 pixel grid; computing their annular flux and normalizing them. 

455 

456 Parameters 

457 ---------- 

458 inputExposure : `~lsst.afw.image.ExposureF` 

459 The image from which bright star stamps should be extracted. 

460 refObjLoader : `~lsst.meas.algorithms.ReferenceObjectLoader`, optional 

461 Loader to find objects within a reference catalog. 

462 dataId : `dict` or `~lsst.daf.butler.DataCoordinate` 

463 The dataId of the exposure (and detector) bright stars should be 

464 extracted from. 

465 skyCorr : `~lsst.afw.math.backgroundList.BackgroundList`, optional 

466 Full focal plane sky correction obtained by `SkyCorrectionTask`. 

467 

468 Returns 

469 ------- 

470 result : `~lsst.pipe.base.Struct` 

471 Results as a struct with attributes: 

472 

473 ``brightStarStamps`` 

474 (`~lsst.meas.algorithms.brightStarStamps.BrightStarStamps`) 

475 """ 

476 if self.config.doApplySkyCorr: 

477 self.log.info( 

478 "Applying sky correction to exposure %s (exposure will be modified in-place).", dataId 

479 ) 

480 self.applySkyCorr(inputExposure, skyCorr) 

481 self.log.info("Extracting bright stars from exposure %s", dataId) 

482 # Extract stamps around bright stars 

483 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader) 

484 if not extractedStamps.starIms: 

485 self.log.info("No suitable bright star found.") 

486 return None 

487 # Warp (and shift, and potentially rotate) them 

488 self.log.info( 

489 "Applying warp and/or shift to %i star stamps from exposure %s.", 

490 len(extractedStamps.starIms), 

491 dataId, 

492 ) 

493 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters) 

494 warpedStars = warpOutputs.warpedStars 

495 xy0s = warpOutputs.xy0s 

496 brightStarList = [ 

497 BrightStarStamp( 

498 stamp_im=warp, 

499 archive_element=transform, 

500 position=xy0s[j], 

501 gaiaGMag=extractedStamps.GMags[j], 

502 gaiaId=extractedStamps.gaiaIds[j], 

503 minValidAnnulusFraction=self.config.minValidAnnulusFraction, 

504 ) 

505 for j, (warp, transform) in enumerate(zip(warpedStars, warpOutputs.warpTransforms)) 

506 ] 

507 # Compute annularFlux and normalize 

508 self.log.info( 

509 "Computing annular flux and normalizing %i bright stars from exposure %s.", 

510 len(warpedStars), 

511 dataId, 

512 ) 

513 # annularFlux statistic set-up, excluding mask planes 

514 statsControl = StatisticsControl() 

515 statsControl.setNumSigmaClip(self.config.numSigmaClip) 

516 statsControl.setNumIter(self.config.numIter) 

517 innerRadius, outerRadius = self.config.annularFluxRadii 

518 statsFlag = stringToStatisticsProperty(self.config.annularFluxStatistic) 

519 brightStarStamps = BrightStarStamps.initAndNormalize( 

520 brightStarList, 

521 innerRadius=innerRadius, 

522 outerRadius=outerRadius, 

523 nb90Rots=warpOutputs.nb90Rots, 

524 imCenter=self.modelCenter, 

525 use_archive=True, 

526 statsControl=statsControl, 

527 statsFlag=statsFlag, 

528 badMaskPlanes=self.config.badMaskPlanes, 

529 discardNanFluxObjects=(self.config.discardNanFluxStars), 

530 ) 

531 return Struct(brightStarStamps=brightStarStamps) 

532 

533 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

534 inputs = butlerQC.get(inputRefs) 

535 inputs["dataId"] = str(butlerQC.quantum.dataId) 

536 refObjLoader = ReferenceObjectLoader( 

537 dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], 

538 refCats=inputs.pop("refCat"), 

539 name=self.config.connections.refCat, 

540 config=self.config.refObjLoader, 

541 ) 

542 output = self.run(**inputs, refObjLoader=refObjLoader) 

543 if output: 

544 butlerQC.put(output, outputRefs)