Coverage for python/lsst/pipe/tasks/processBrightStars.py: 22%

180 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-20 10:28 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Extract small cutouts around bright stars, normalize and warp them to the 

23same arbitrary pixel grid. 

24""" 

25 

26__all__ = ["ProcessBrightStarsTask"] 

27 

28import numpy as np 

29import astropy.units as u 

30 

31from lsst import geom 

32from lsst.afw import math as afwMath 

33from lsst.afw import image as afwImage 

34from lsst.afw import detection as afwDetect 

35from lsst.afw import cameraGeom as cg 

36from lsst.afw.geom import transformFactory as tFactory 

37import lsst.pex.config as pexConfig 

38from lsst.pipe import base as pipeBase 

39from lsst.pipe.base import connectionTypes as cT 

40from lsst.pex.exceptions import InvalidParameterError 

41from lsst.meas.algorithms import LoadReferenceObjectsConfig 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43from lsst.meas.algorithms import brightStarStamps as bSS 

44from lsst.utils.timer import timeMethod 

45 

46 

47class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "visit", "detector")): 

49 inputExposure = cT.Input( 

50 doc="Input exposure from which to extract bright star stamps", 

51 name="calexp", 

52 storageClass="ExposureF", 

53 dimensions=("visit", "detector") 

54 ) 

55 skyCorr = cT.Input( 

56 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

57 name="skyCorr", 

58 storageClass="Background", 

59 dimensions=("instrument", "visit", "detector") 

60 ) 

61 refCat = cT.PrerequisiteInput( 

62 doc="Reference catalog that contains bright star positions", 

63 name="gaia_dr2_20200414", 

64 storageClass="SimpleCatalog", 

65 dimensions=("skypix",), 

66 multiple=True, 

67 deferLoad=True 

68 ) 

69 brightStarStamps = cT.Output( 

70 doc="Set of preprocessed postage stamps, each centered on a single bright star.", 

71 name="brightStarStamps", 

72 storageClass="BrightStarStamps", 

73 dimensions=("visit", "detector") 

74 ) 

75 

76 def __init__(self, *, config=None): 

77 super().__init__(config=config) 

78 if not config.doApplySkyCorr: 

79 self.inputs.remove("skyCorr") 

80 

81 

82class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=ProcessBrightStarsConnections): 

84 """Configuration parameters for ProcessBrightStarsTask. 

85 """ 

86 

87 magLimit = pexConfig.Field( 

88 dtype=float, 

89 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed", 

90 default=18 

91 ) 

92 stampSize = pexConfig.ListField( 

93 dtype=int, 

94 doc="Size of the stamps to be extracted, in pixels", 

95 default=(250, 250) 

96 ) 

97 modelStampBuffer = pexConfig.Field( 

98 dtype=float, 

99 doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will " 

100 "be saved in. This will also be the size of the extended PSF model.", 

101 default=1.1 

102 ) 

103 doRemoveDetected = pexConfig.Field( 

104 dtype=bool, 

105 doc="Whether DETECTION footprints, other than that for the central object, should be changed to " 

106 "BAD", 

107 default=True 

108 ) 

109 doApplyTransform = pexConfig.Field( 

110 dtype=bool, 

111 doc="Apply transform to bright star stamps to correct for optical distortions?", 

112 default=True 

113 ) 

114 warpingKernelName = pexConfig.ChoiceField( 

115 dtype=str, 

116 doc="Warping kernel", 

117 default="lanczos5", 

118 allowed={ 

119 "bilinear": "bilinear interpolation", 

120 "lanczos3": "Lanczos kernel of order 3", 

121 "lanczos4": "Lanczos kernel of order 4", 

122 "lanczos5": "Lanczos kernel of order 5", 

123 } 

124 ) 

125 annularFluxRadii = pexConfig.ListField( 

126 dtype=int, 

127 doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, " 

128 "in pixels.", 

129 default=(40, 50) 

130 ) 

131 annularFluxStatistic = pexConfig.ChoiceField( 

132 dtype=str, 

133 doc="Type of statistic to use to compute annular flux.", 

134 default="MEANCLIP", 

135 allowed={ 

136 "MEAN": "mean", 

137 "MEDIAN": "median", 

138 "MEANCLIP": "clipped mean", 

139 } 

140 ) 

141 numSigmaClip = pexConfig.Field( 

142 dtype=float, 

143 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

144 default=4 

145 ) 

146 numIter = pexConfig.Field( 

147 dtype=int, 

148 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

149 default=3 

150 ) 

151 badMaskPlanes = pexConfig.ListField( 

152 dtype=str, 

153 doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the" 

154 " annular flux.", 

155 default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN') 

156 ) 

157 minPixelsWithinFrame = pexConfig.Field( 

158 dtype=int, 

159 doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be" 

160 " saved when its center is beyond the exposure boundary.", 

161 default=50 

162 ) 

163 doApplySkyCorr = pexConfig.Field( 

164 dtype=bool, 

165 doc="Apply full focal plane sky correction before extracting stars?", 

166 default=True 

167 ) 

168 discardNanFluxStars = pexConfig.Field( 

169 dtype=bool, 

170 doc="Should stars with NaN annular flux be discarded?", 

171 default=False 

172 ) 

173 refObjLoader = pexConfig.ConfigField( 

174 dtype=LoadReferenceObjectsConfig, 

175 doc="Reference object loader for astrometric calibration.", 

176 ) 

177 

178 

179class ProcessBrightStarsTask(pipeBase.PipelineTask): 

180 """The description of the parameters for this Task are detailed in 

181 :lsst-task:`~lsst.pipe.base.PipelineTask`. 

182 

183 Parameters 

184 ---------- 

185 initInputs : `Unknown` 

186 *args 

187 Additional positional arguments. 

188 **kwargs 

189 Additional keyword arguments. 

190 

191 Notes 

192 ----- 

193 `ProcessBrightStarsTask` is used to extract, process, and store small 

194 image cut-outs (or "postage stamps") around bright stars. It relies on 

195 three methods, called in succession: 

196 

197 `extractStamps` 

198 Find bright stars within the exposure using a reference catalog and 

199 extract a stamp centered on each. 

200 `warpStamps` 

201 Shift and warp each stamp to remove optical distortions and sample all 

202 stars on the same pixel grid. 

203 `measureAndNormalize` 

204 Compute the flux of an object in an annulus and normalize it. This is 

205 required to normalize each bright star stamp as their central pixels 

206 are likely saturated and/or contain ghosts, and cannot be used. 

207 """ 

208 ConfigClass = ProcessBrightStarsConfig 

209 _DefaultName = "processBrightStars" 

210 

211 def __init__(self, butler=None, initInputs=None, *args, **kwargs): 

212 super().__init__(*args, **kwargs) 

213 # Compute (model) stamp size depending on provided "buffer" value 

214 self.modelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer), 

215 int(self.config.stampSize[1]*self.config.modelStampBuffer)] 

216 # force it to be odd-sized so we have a central pixel 

217 if not self.modelStampSize[0] % 2: 

218 self.modelStampSize[0] += 1 

219 if not self.modelStampSize[1] % 2: 

220 self.modelStampSize[1] += 1 

221 # central pixel 

222 self.modelCenter = self.modelStampSize[0]//2, self.modelStampSize[1]//2 

223 # configure Gaia refcat 

224 if butler is not None: 

225 self.makeSubtask('refObjLoader', butler=butler) 

226 

227 def applySkyCorr(self, calexp, skyCorr): 

228 """Apply correction to the sky background level. 

229 

230 Sky corrections can be generated with the 'skyCorrection.py' 

231 executable in pipe_drivers. Because the sky model used by that 

232 code extends over the entire focal plane, this can produce 

233 better sky subtraction. 

234 The calexp is updated in-place. 

235 

236 Parameters 

237 ---------- 

238 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

239 Calibrated exposure. 

240 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or `None`, 

241 optional 

242 Full focal plane sky correction, obtained by running 

243 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

244 """ 

245 if isinstance(calexp, afwImage.Exposure): 

246 calexp = calexp.getMaskedImage() 

247 calexp -= skyCorr.getImage() 

248 

249 def extractStamps(self, inputExposure, refObjLoader=None): 

250 """ Read position of bright stars within `inputExposure` from refCat 

251 and extract them. 

252 

253 Parameters 

254 ---------- 

255 inputExposure : `afwImage.exposure.exposure.ExposureF` 

256 The image from which bright star stamps should be extracted. 

257 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

258 Loader to find objects within a reference catalog. 

259 

260 Returns 

261 ------- 

262 result : `lsst.pipe.base.Struct` 

263 Results as a struct with attributes: 

264 

265 ``starIms`` 

266 List of stamps (`list`). 

267 ``pixCenters`` 

268 List of corresponding coordinates to each star's center, in pixels (`list`). 

269 ``GMags`` 

270 List of corresponding (Gaia) G magnitudes (`list`). 

271 ``gaiaIds`` 

272 Array of corresponding unique Gaia identifiers (`np.ndarray`). 

273 """ 

274 if refObjLoader is None: 

275 refObjLoader = self.refObjLoader 

276 starIms = [] 

277 pixCenters = [] 

278 GMags = [] 

279 ids = [] 

280 wcs = inputExposure.getWcs() 

281 # select stars within, or close enough to input exposure from refcat 

282 inputIm = inputExposure.maskedImage 

283 inputExpBBox = inputExposure.getBBox() 

284 dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame) 

285 # TODO (DM-25894): handle catalog with stars missing from Gaia 

286 withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs, 

287 filterName="phot_g_mean") 

288 refCat = withinCalexp.refCat 

289 # keep bright objects 

290 fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value() 

291 GFluxes = np.array(refCat['phot_g_mean_flux']) 

292 bright = GFluxes > fluxLimit 

293 # convert to AB magnitudes 

294 allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]] 

295 allIds = refCat.columns.extract("id", where=bright)["id"] 

296 selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright) 

297 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])): 

298 sp = geom.SpherePoint(ra, dec, geom.radians) 

299 cpix = wcs.skyToPixel(sp) 

300 try: 

301 starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize)) 

302 except InvalidParameterError: 

303 # star is beyond boundary 

304 bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2 

305 # compute bbox as it would be otherwise 

306 idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize)) 

307 clippedStarBBox = geom.Box2I(idealBBox) 

308 clippedStarBBox.clip(inputExpBBox) 

309 if clippedStarBBox.getArea() > 0: 

310 # create full-sized stamp with all pixels 

311 # flagged as NO_DATA 

312 starIm = afwImage.ExposureF(bbox=idealBBox) 

313 starIm.image[:] = np.nan 

314 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA")) 

315 # recover pixels from intersection with the exposure 

316 clippedIm = inputIm.Factory(inputIm, clippedStarBBox) 

317 starIm.maskedImage[clippedStarBBox] = clippedIm 

318 # set detector and wcs, used in warpStars 

319 starIm.setDetector(inputExposure.getDetector()) 

320 starIm.setWcs(inputExposure.getWcs()) 

321 else: 

322 continue 

323 if self.config.doRemoveDetected: 

324 # give detection footprint of other objects the BAD flag 

325 detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"), 

326 afwDetect.Threshold.BITMASK) 

327 omask = afwDetect.FootprintSet(starIm.mask, detThreshold) 

328 allFootprints = omask.getFootprints() 

329 otherFootprints = [] 

330 for fs in allFootprints: 

331 if not fs.contains(geom.Point2I(cpix)): 

332 otherFootprints.append(fs) 

333 nbMatchingFootprints = len(allFootprints) - len(otherFootprints) 

334 if not nbMatchingFootprints == 1: 

335 self.log.warning("Failed to uniquely identify central DETECTION footprint for star " 

336 "%s; found %d footprints instead.", 

337 allIds[j], nbMatchingFootprints) 

338 omask.setFootprints(otherFootprints) 

339 omask.setMask(starIm.mask, "BAD") 

340 starIms.append(starIm) 

341 pixCenters.append(cpix) 

342 GMags.append(allGMags[j]) 

343 ids.append(allIds[j]) 

344 return pipeBase.Struct(starIms=starIms, 

345 pixCenters=pixCenters, 

346 GMags=GMags, 

347 gaiaIds=ids) 

348 

349 def warpStamps(self, stamps, pixCenters): 

350 """Warps and shifts all given stamps so they are sampled on the same 

351 pixel grid and centered on the central pixel. This includes rotating 

352 the stamp depending on detector orientation. 

353 

354 Parameters 

355 ---------- 

356 stamps : `collections.abc.Sequence` 

357 [`afwImage.exposure.exposure.ExposureF`] 

358 Image cutouts centered on a single object. 

359 pixCenters : `collections.abc.Sequence` [`geom.Point2D`] 

360 Positions of each object's center (as obtained from the refCat), 

361 in pixels. 

362 

363 Returns 

364 ------- 

365 result : `lsst.pipe.base.Struct` 

366 Results as a struct with attributes: 

367 

368 ``warpedStars`` 

369 List of stamps of warped stars (`list` of `afwImage.maskedImage.maskedImage.MaskedImage`). 

370 ``warpTransforms`` 

371 List of the corresponding Transform from the initial star stamp 

372 to the common model grid (`list` of `afwGeom.TransformPoint2ToPoint2`). 

373 ``xy0s`` 

374 List of coordinates of the bottom-left 

375 pixels of each stamp, before rotation (`list` of `geom.Point2I`). 

376 ``nb90Rots`` 

377 The number of 90 degrees rotations required 

378 to compensate for detector orientation (`int`). 

379 """ 

380 # warping control; only contains shiftingALg provided in config 

381 warpCont = afwMath.WarpingControl(self.config.warpingKernelName) 

382 # Compare model to star stamp sizes 

383 bufferPix = (self.modelStampSize[0] - self.config.stampSize[0], 

384 self.modelStampSize[1] - self.config.stampSize[1]) 

385 # Initialize detector instance (note all stars were extracted from an 

386 # exposure from the same detector) 

387 det = stamps[0].getDetector() 

388 # Define correction for optical distortions 

389 if self.config.doApplyTransform: 

390 pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS) 

391 else: 

392 pixToTan = tFactory.makeIdentityTransform() 

393 # Array of all possible rotations for detector orientation: 

394 possibleRots = np.array([k*np.pi/2 for k in range(4)]) 

395 # determine how many, if any, rotations are required 

396 yaw = det.getOrientation().getYaw() 

397 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw))) 

398 

399 # apply transformation to each star 

400 warpedStars, warpTransforms, xy0s = [], [], [] 

401 for star, cent in zip(stamps, pixCenters): 

402 # (re)create empty destination image 

403 destImage = afwImage.MaskedImageF(*self.modelStampSize) 

404 bottomLeft = geom.Point2D(star.image.getXY0()) 

405 newBottomLeft = pixToTan.applyForward(bottomLeft) 

406 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2) 

407 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2) 

408 # Convert to int 

409 newBottomLeft = geom.Point2I(newBottomLeft) 

410 # Set origin and save it 

411 destImage.setXY0(newBottomLeft) 

412 xy0s.append(newBottomLeft) 

413 

414 # Define linear shifting to recenter stamps 

415 newCenter = pixToTan.applyForward(cent) # center of warped star 

416 shift = self.modelCenter[0] + newBottomLeft[0] - newCenter[0],\ 

417 self.modelCenter[1] + newBottomLeft[1] - newCenter[1] 

418 affineShift = geom.AffineTransform(shift) 

419 shiftTransform = tFactory.makeTransform(affineShift) 

420 

421 # Define full transform (warp and shift) 

422 starWarper = pixToTan.then(shiftTransform) 

423 

424 # Apply it 

425 goodPix = afwMath.warpImage(destImage, star.getMaskedImage(), 

426 starWarper, warpCont) 

427 if not goodPix: 

428 self.log.debug("Warping of a star failed: no good pixel in output") 

429 

430 # Arbitrarily set origin of shifted star to 0 

431 destImage.setXY0(0, 0) 

432 

433 # Apply rotation if appropriate 

434 if nb90Rots: 

435 destImage = afwMath.rotateImageBy90(destImage, nb90Rots) 

436 warpedStars.append(destImage.clone()) 

437 warpTransforms.append(starWarper) 

438 return pipeBase.Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s, 

439 nb90Rots=nb90Rots) 

440 

441 @timeMethod 

442 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None): 

443 """Identify bright stars within an exposure using a reference catalog, 

444 extract stamps around each, then preprocess them. The preprocessing 

445 steps are: shifting, warping and potentially rotating them to the same 

446 pixel grid; computing their annular flux and normalizing them. 

447 

448 Parameters 

449 ---------- 

450 inputExposure : `afwImage.exposure.exposure.ExposureF` 

451 The image from which bright star stamps should be extracted. 

452 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

453 Loader to find objects within a reference catalog. 

454 dataId : `dict` or `lsst.daf.butler.DataCoordinate` 

455 The dataId of the exposure (and detector) bright stars should be 

456 extracted from. 

457 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or `None`, 

458 optional 

459 Full focal plane sky correction, obtained by running 

460 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

461 

462 Returns 

463 ------- 

464 result : `lsst.pipe.base.Struct` 

465 Results as a struct with attributes: 

466 

467 ``brightStarStamps`` 

468 (`bSS.BrightStarStamps`) 

469 """ 

470 if self.config.doApplySkyCorr: 

471 self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).", 

472 dataId) 

473 self.applySkyCorr(inputExposure, skyCorr) 

474 self.log.info("Extracting bright stars from exposure %s", dataId) 

475 # Extract stamps around bright stars 

476 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader) 

477 if not extractedStamps.starIms: 

478 self.log.info("No suitable bright star found.") 

479 return None 

480 # Warp (and shift, and potentially rotate) them 

481 self.log.info("Applying warp and/or shift to %i star stamps from exposure %s", 

482 len(extractedStamps.starIms), dataId) 

483 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters) 

484 warpedStars = warpOutputs.warpedStars 

485 xy0s = warpOutputs.xy0s 

486 brightStarList = [bSS.BrightStarStamp(stamp_im=warp, 

487 archive_element=transform, 

488 position=xy0s[j], 

489 gaiaGMag=extractedStamps.GMags[j], 

490 gaiaId=extractedStamps.gaiaIds[j]) 

491 for j, (warp, transform) in 

492 enumerate(zip(warpedStars, warpOutputs.warpTransforms))] 

493 # Compute annularFlux and normalize 

494 self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s", 

495 len(warpedStars), dataId) 

496 # annularFlux statistic set-up, excluding mask planes 

497 statsControl = afwMath.StatisticsControl() 

498 statsControl.setNumSigmaClip(self.config.numSigmaClip) 

499 statsControl.setNumIter(self.config.numIter) 

500 innerRadius, outerRadius = self.config.annularFluxRadii 

501 statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic) 

502 brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList, 

503 innerRadius=innerRadius, 

504 outerRadius=outerRadius, 

505 nb90Rots=warpOutputs.nb90Rots, 

506 imCenter=self.modelCenter, 

507 use_archive=True, 

508 statsControl=statsControl, 

509 statsFlag=statsFlag, 

510 badMaskPlanes=self.config.badMaskPlanes, 

511 discardNanFluxObjects=( 

512 self.config.discardNanFluxStars)) 

513 return pipeBase.Struct(brightStarStamps=brightStarStamps) 

514 

515 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

516 inputs = butlerQC.get(inputRefs) 

517 inputs['dataId'] = str(butlerQC.quantum.dataId) 

518 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

519 for ref in inputRefs.refCat], 

520 refCats=inputs.pop("refCat"), 

521 name=self.config.connections.refCat, 

522 config=self.config.refObjLoader) 

523 output = self.run(**inputs, refObjLoader=refObjLoader) 

524 if output: 

525 butlerQC.put(output, outputRefs)