Coverage for python/lsst/pipe/tasks/processBrightStars.py: 24%

180 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-25 01:37 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Extract small cutouts around bright stars, normalize and warp them to the 

23same arbitrary pixel grid. 

24""" 

25 

26__all__ = ["ProcessBrightStarsTask"] 

27 

28import numpy as np 

29import astropy.units as u 

30 

31from lsst import geom 

32from lsst.afw import math as afwMath 

33from lsst.afw import image as afwImage 

34from lsst.afw import detection as afwDetect 

35from lsst.afw import cameraGeom as cg 

36from lsst.afw.geom import transformFactory as tFactory 

37import lsst.pex.config as pexConfig 

38from lsst.pipe import base as pipeBase 

39from lsst.pipe.base import connectionTypes as cT 

40from lsst.pex.exceptions import InvalidParameterError 

41from lsst.meas.algorithms import LoadReferenceObjectsConfig 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43from lsst.meas.algorithms import brightStarStamps as bSS 

44from lsst.utils.timer import timeMethod 

45 

46 

47class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "visit", "detector")): 

49 inputExposure = cT.Input( 

50 doc="Input exposure from which to extract bright star stamps", 

51 name="calexp", 

52 storageClass="ExposureF", 

53 dimensions=("visit", "detector") 

54 ) 

55 skyCorr = cT.Input( 

56 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

57 name="skyCorr", 

58 storageClass="Background", 

59 dimensions=("instrument", "visit", "detector") 

60 ) 

61 refCat = cT.PrerequisiteInput( 

62 doc="Reference catalog that contains bright star positions", 

63 name="gaia_dr2_20200414", 

64 storageClass="SimpleCatalog", 

65 dimensions=("skypix",), 

66 multiple=True, 

67 deferLoad=True 

68 ) 

69 brightStarStamps = cT.Output( 

70 doc="Set of preprocessed postage stamps, each centered on a single bright star.", 

71 name="brightStarStamps", 

72 storageClass="BrightStarStamps", 

73 dimensions=("visit", "detector") 

74 ) 

75 

76 def __init__(self, *, config=None): 

77 super().__init__(config=config) 

78 if not config.doApplySkyCorr: 

79 self.inputs.remove("skyCorr") 

80 

81 

82class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=ProcessBrightStarsConnections): 

84 """Configuration parameters for ProcessBrightStarsTask 

85 """ 

86 magLimit = pexConfig.Field( 

87 dtype=float, 

88 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed", 

89 default=18 

90 ) 

91 stampSize = pexConfig.ListField( 

92 dtype=int, 

93 doc="Size of the stamps to be extracted, in pixels", 

94 default=(250, 250) 

95 ) 

96 modelStampBuffer = pexConfig.Field( 

97 dtype=float, 

98 doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will " 

99 "be saved in. This will also be the size of the extended PSF model.", 

100 default=1.1 

101 ) 

102 doRemoveDetected = pexConfig.Field( 

103 dtype=bool, 

104 doc="Whether DETECTION footprints, other than that for the central object, should be changed to " 

105 "BAD", 

106 default=True 

107 ) 

108 doApplyTransform = pexConfig.Field( 

109 dtype=bool, 

110 doc="Apply transform to bright star stamps to correct for optical distortions?", 

111 default=True 

112 ) 

113 warpingKernelName = pexConfig.ChoiceField( 

114 dtype=str, 

115 doc="Warping kernel", 

116 default="lanczos5", 

117 allowed={ 

118 "bilinear": "bilinear interpolation", 

119 "lanczos3": "Lanczos kernel of order 3", 

120 "lanczos4": "Lanczos kernel of order 4", 

121 "lanczos5": "Lanczos kernel of order 5", 

122 } 

123 ) 

124 annularFluxRadii = pexConfig.ListField( 

125 dtype=int, 

126 doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, " 

127 "in pixels.", 

128 default=(40, 50) 

129 ) 

130 annularFluxStatistic = pexConfig.ChoiceField( 

131 dtype=str, 

132 doc="Type of statistic to use to compute annular flux.", 

133 default="MEANCLIP", 

134 allowed={ 

135 "MEAN": "mean", 

136 "MEDIAN": "median", 

137 "MEANCLIP": "clipped mean", 

138 } 

139 ) 

140 numSigmaClip = pexConfig.Field( 

141 dtype=float, 

142 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

143 default=4 

144 ) 

145 numIter = pexConfig.Field( 

146 dtype=int, 

147 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

148 default=3 

149 ) 

150 badMaskPlanes = pexConfig.ListField( 

151 dtype=str, 

152 doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the" 

153 " annular flux.", 

154 default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN') 

155 ) 

156 minPixelsWithinFrame = pexConfig.Field( 

157 dtype=int, 

158 doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be" 

159 " saved when its center is beyond the exposure boundary.", 

160 default=50 

161 ) 

162 doApplySkyCorr = pexConfig.Field( 

163 dtype=bool, 

164 doc="Apply full focal plane sky correction before extracting stars?", 

165 default=True 

166 ) 

167 discardNanFluxStars = pexConfig.Field( 

168 dtype=bool, 

169 doc="Should stars with NaN annular flux be discarded?", 

170 default=False 

171 ) 

172 refObjLoader = pexConfig.ConfigField( 

173 dtype=LoadReferenceObjectsConfig, 

174 doc="Reference object loader for astrometric calibration.", 

175 ) 

176 

177 

178class ProcessBrightStarsTask(pipeBase.PipelineTask): 

179 """The description of the parameters for this Task are detailed in 

180 :lsst-task:`~lsst.pipe.base.PipelineTask`. 

181 

182 Notes 

183 ----- 

184 `ProcessBrightStarsTask` is used to extract, process, and store small 

185 image cut-outs (or "postage stamps") around bright stars. It relies on 

186 three methods, called in succession: 

187 

188 `extractStamps` 

189 Find bright stars within the exposure using a reference catalog and 

190 extract a stamp centered on each. 

191 `warpStamps` 

192 Shift and warp each stamp to remove optical distortions and sample all 

193 stars on the same pixel grid. 

194 `measureAndNormalize` 

195 Compute the flux of an object in an annulus and normalize it. This is 

196 required to normalize each bright star stamp as their central pixels 

197 are likely saturated and/or contain ghosts, and cannot be used. 

198 """ 

199 ConfigClass = ProcessBrightStarsConfig 

200 _DefaultName = "processBrightStars" 

201 

202 def __init__(self, butler=None, initInputs=None, *args, **kwargs): 

203 super().__init__(*args, **kwargs) 

204 # Compute (model) stamp size depending on provided "buffer" value 

205 self.modelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer), 

206 int(self.config.stampSize[1]*self.config.modelStampBuffer)] 

207 # force it to be odd-sized so we have a central pixel 

208 if not self.modelStampSize[0] % 2: 

209 self.modelStampSize[0] += 1 

210 if not self.modelStampSize[1] % 2: 

211 self.modelStampSize[1] += 1 

212 # central pixel 

213 self.modelCenter = self.modelStampSize[0]//2, self.modelStampSize[1]//2 

214 # configure Gaia refcat 

215 if butler is not None: 

216 self.makeSubtask('refObjLoader', butler=butler) 

217 

218 def applySkyCorr(self, calexp, skyCorr): 

219 """Apply correction to the sky background level. 

220 

221 Sky corrections can be generated with the 'skyCorrection.py' 

222 executable in pipe_drivers. Because the sky model used by that 

223 code extends over the entire focal plane, this can produce 

224 better sky subtraction. 

225 The calexp is updated in-place. 

226 

227 Parameters 

228 ---------- 

229 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

230 Calibrated exposure. 

231 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None, 

232 optional 

233 Full focal plane sky correction, obtained by running 

234 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

235 """ 

236 if isinstance(calexp, afwImage.Exposure): 

237 calexp = calexp.getMaskedImage() 

238 calexp -= skyCorr.getImage() 

239 

240 def extractStamps(self, inputExposure, refObjLoader=None): 

241 """ Read position of bright stars within `inputExposure` from refCat 

242 and extract them. 

243 

244 Parameters 

245 ---------- 

246 inputExposure : `afwImage.exposure.exposure.ExposureF` 

247 The image from which bright star stamps should be extracted. 

248 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

249 Loader to find objects within a reference catalog. 

250 

251 Returns 

252 ------- 

253 result : `lsst.pipe.base.Struct` 

254 Result struct with components: 

255 

256 - ``starIms``: `list` of stamps 

257 - ``pixCenters``: `list` of corresponding coordinates to each 

258 star's center, in pixels. 

259 - ``GMags``: `list` of corresponding (Gaia) G magnitudes. 

260 - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia 

261 identifiers. 

262 """ 

263 if refObjLoader is None: 

264 refObjLoader = self.refObjLoader 

265 starIms = [] 

266 pixCenters = [] 

267 GMags = [] 

268 ids = [] 

269 wcs = inputExposure.getWcs() 

270 # select stars within, or close enough to input exposure from refcat 

271 inputIm = inputExposure.maskedImage 

272 inputExpBBox = inputExposure.getBBox() 

273 dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame) 

274 # TODO (DM-25894): handle catalog with stars missing from Gaia 

275 withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs, 

276 filterName="phot_g_mean") 

277 refCat = withinCalexp.refCat 

278 # keep bright objects 

279 fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value() 

280 GFluxes = np.array(refCat['phot_g_mean_flux']) 

281 bright = GFluxes > fluxLimit 

282 # convert to AB magnitudes 

283 allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]] 

284 allIds = refCat.columns.extract("id", where=bright)["id"] 

285 selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright) 

286 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])): 

287 sp = geom.SpherePoint(ra, dec, geom.radians) 

288 cpix = wcs.skyToPixel(sp) 

289 try: 

290 starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize)) 

291 except InvalidParameterError: 

292 # star is beyond boundary 

293 bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2 

294 # compute bbox as it would be otherwise 

295 idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize)) 

296 clippedStarBBox = geom.Box2I(idealBBox) 

297 clippedStarBBox.clip(inputExpBBox) 

298 if clippedStarBBox.getArea() > 0: 

299 # create full-sized stamp with all pixels 

300 # flagged as NO_DATA 

301 starIm = afwImage.ExposureF(bbox=idealBBox) 

302 starIm.image[:] = np.nan 

303 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA")) 

304 # recover pixels from intersection with the exposure 

305 clippedIm = inputIm.Factory(inputIm, clippedStarBBox) 

306 starIm.maskedImage[clippedStarBBox] = clippedIm 

307 # set detector and wcs, used in warpStars 

308 starIm.setDetector(inputExposure.getDetector()) 

309 starIm.setWcs(inputExposure.getWcs()) 

310 else: 

311 continue 

312 if self.config.doRemoveDetected: 

313 # give detection footprint of other objects the BAD flag 

314 detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"), 

315 afwDetect.Threshold.BITMASK) 

316 omask = afwDetect.FootprintSet(starIm.mask, detThreshold) 

317 allFootprints = omask.getFootprints() 

318 otherFootprints = [] 

319 for fs in allFootprints: 

320 if not fs.contains(geom.Point2I(cpix)): 

321 otherFootprints.append(fs) 

322 nbMatchingFootprints = len(allFootprints) - len(otherFootprints) 

323 if not nbMatchingFootprints == 1: 

324 self.log.warning("Failed to uniquely identify central DETECTION footprint for star " 

325 "%s; found %d footprints instead.", 

326 allIds[j], nbMatchingFootprints) 

327 omask.setFootprints(otherFootprints) 

328 omask.setMask(starIm.mask, "BAD") 

329 starIms.append(starIm) 

330 pixCenters.append(cpix) 

331 GMags.append(allGMags[j]) 

332 ids.append(allIds[j]) 

333 return pipeBase.Struct(starIms=starIms, 

334 pixCenters=pixCenters, 

335 GMags=GMags, 

336 gaiaIds=ids) 

337 

338 def warpStamps(self, stamps, pixCenters): 

339 """Warps and shifts all given stamps so they are sampled on the same 

340 pixel grid and centered on the central pixel. This includes rotating 

341 the stamp depending on detector orientation. 

342 

343 Parameters 

344 ---------- 

345 stamps : `collections.abc.Sequence` 

346 [`afwImage.exposure.exposure.ExposureF`] 

347 Image cutouts centered on a single object. 

348 pixCenters : `collections.abc.Sequence` [`geom.Point2D`] 

349 Positions of each object's center (as obtained from the refCat), 

350 in pixels. 

351 

352 Returns 

353 ------- 

354 result : `lsst.pipe.base.Struct` 

355 Result struct with components: 

356 

357 - ``warpedStars``: 

358 `list` [`afwImage.maskedImage.maskedImage.MaskedImage`] of 

359 stamps of warped stars 

360 - ``warpTransforms``: 

361 `list` [`afwGeom.TransformPoint2ToPoint2`] of 

362 the corresponding Transform from the initial star stamp to 

363 the common model grid 

364 - ``xy0s``: 

365 `list` [`geom.Point2I`] of coordinates of the bottom-left 

366 pixels of each stamp, before rotation 

367 - ``nb90Rots``: `int`, the number of 90 degrees rotations required 

368 to compensate for detector orientation 

369 """ 

370 # warping control; only contains shiftingALg provided in config 

371 warpCont = afwMath.WarpingControl(self.config.warpingKernelName) 

372 # Compare model to star stamp sizes 

373 bufferPix = (self.modelStampSize[0] - self.config.stampSize[0], 

374 self.modelStampSize[1] - self.config.stampSize[1]) 

375 # Initialize detector instance (note all stars were extracted from an 

376 # exposure from the same detector) 

377 det = stamps[0].getDetector() 

378 # Define correction for optical distortions 

379 if self.config.doApplyTransform: 

380 pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS) 

381 else: 

382 pixToTan = tFactory.makeIdentityTransform() 

383 # Array of all possible rotations for detector orientation: 

384 possibleRots = np.array([k*np.pi/2 for k in range(4)]) 

385 # determine how many, if any, rotations are required 

386 yaw = det.getOrientation().getYaw() 

387 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw))) 

388 

389 # apply transformation to each star 

390 warpedStars, warpTransforms, xy0s = [], [], [] 

391 for star, cent in zip(stamps, pixCenters): 

392 # (re)create empty destination image 

393 destImage = afwImage.MaskedImageF(*self.modelStampSize) 

394 bottomLeft = geom.Point2D(star.image.getXY0()) 

395 newBottomLeft = pixToTan.applyForward(bottomLeft) 

396 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2) 

397 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2) 

398 # Convert to int 

399 newBottomLeft = geom.Point2I(newBottomLeft) 

400 # Set origin and save it 

401 destImage.setXY0(newBottomLeft) 

402 xy0s.append(newBottomLeft) 

403 

404 # Define linear shifting to recenter stamps 

405 newCenter = pixToTan.applyForward(cent) # center of warped star 

406 shift = self.modelCenter[0] + newBottomLeft[0] - newCenter[0],\ 

407 self.modelCenter[1] + newBottomLeft[1] - newCenter[1] 

408 affineShift = geom.AffineTransform(shift) 

409 shiftTransform = tFactory.makeTransform(affineShift) 

410 

411 # Define full transform (warp and shift) 

412 starWarper = pixToTan.then(shiftTransform) 

413 

414 # Apply it 

415 goodPix = afwMath.warpImage(destImage, star.getMaskedImage(), 

416 starWarper, warpCont) 

417 if not goodPix: 

418 self.log.debug("Warping of a star failed: no good pixel in output") 

419 

420 # Arbitrarily set origin of shifted star to 0 

421 destImage.setXY0(0, 0) 

422 

423 # Apply rotation if appropriate 

424 if nb90Rots: 

425 destImage = afwMath.rotateImageBy90(destImage, nb90Rots) 

426 warpedStars.append(destImage.clone()) 

427 warpTransforms.append(starWarper) 

428 return pipeBase.Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s, 

429 nb90Rots=nb90Rots) 

430 

431 @timeMethod 

432 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None): 

433 """Identify bright stars within an exposure using a reference catalog, 

434 extract stamps around each, then preprocess them. The preprocessing 

435 steps are: shifting, warping and potentially rotating them to the same 

436 pixel grid; computing their annular flux and normalizing them. 

437 

438 Parameters 

439 ---------- 

440 inputExposure : `afwImage.exposure.exposure.ExposureF` 

441 The image from which bright star stamps should be extracted. 

442 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

443 Loader to find objects within a reference catalog. 

444 dataId : `dict` or `lsst.daf.butler.DataCoordinate` 

445 The dataId of the exposure (and detector) bright stars should be 

446 extracted from. 

447 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``, 

448 optional 

449 Full focal plane sky correction, obtained by running 

450 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

451 

452 Returns 

453 ------- 

454 result : `lsst.pipe.base.Struct` 

455 Result struct with component: 

456 

457 - ``brightStarStamps``: ``bSS.BrightStarStamps`` 

458 """ 

459 if self.config.doApplySkyCorr: 

460 self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).", 

461 dataId) 

462 self.applySkyCorr(inputExposure, skyCorr) 

463 self.log.info("Extracting bright stars from exposure %s", dataId) 

464 # Extract stamps around bright stars 

465 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader) 

466 if not extractedStamps.starIms: 

467 self.log.info("No suitable bright star found.") 

468 return None 

469 # Warp (and shift, and potentially rotate) them 

470 self.log.info("Applying warp and/or shift to %i star stamps from exposure %s", 

471 len(extractedStamps.starIms), dataId) 

472 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters) 

473 warpedStars = warpOutputs.warpedStars 

474 xy0s = warpOutputs.xy0s 

475 brightStarList = [bSS.BrightStarStamp(stamp_im=warp, 

476 archive_element=transform, 

477 position=xy0s[j], 

478 gaiaGMag=extractedStamps.GMags[j], 

479 gaiaId=extractedStamps.gaiaIds[j]) 

480 for j, (warp, transform) in 

481 enumerate(zip(warpedStars, warpOutputs.warpTransforms))] 

482 # Compute annularFlux and normalize 

483 self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s", 

484 len(warpedStars), dataId) 

485 # annularFlux statistic set-up, excluding mask planes 

486 statsControl = afwMath.StatisticsControl() 

487 statsControl.setNumSigmaClip(self.config.numSigmaClip) 

488 statsControl.setNumIter(self.config.numIter) 

489 innerRadius, outerRadius = self.config.annularFluxRadii 

490 statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic) 

491 brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList, 

492 innerRadius=innerRadius, 

493 outerRadius=outerRadius, 

494 nb90Rots=warpOutputs.nb90Rots, 

495 imCenter=self.modelCenter, 

496 use_archive=True, 

497 statsControl=statsControl, 

498 statsFlag=statsFlag, 

499 badMaskPlanes=self.config.badMaskPlanes, 

500 discardNanFluxObjects=( 

501 self.config.discardNanFluxStars)) 

502 return pipeBase.Struct(brightStarStamps=brightStarStamps) 

503 

504 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

505 inputs = butlerQC.get(inputRefs) 

506 inputs['dataId'] = str(butlerQC.quantum.dataId) 

507 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

508 for ref in inputRefs.refCat], 

509 refCats=inputs.pop("refCat"), 

510 name=self.config.connections.refCat, 

511 config=self.config.refObjLoader) 

512 output = self.run(**inputs, refObjLoader=refObjLoader) 

513 if output: 

514 butlerQC.put(output, outputRefs)