Coverage for python/lsst/pipe/tasks/processBrightStars.py: 24%

182 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-19 13:00 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Extract small cutouts around bright stars, normalize and warp them to the 

23same arbitrary pixel grid. 

24""" 

25 

26__all__ = ["ProcessBrightStarsTask"] 

27 

28import numpy as np 

29import astropy.units as u 

30 

31from lsst import geom 

32from lsst.afw import math as afwMath 

33from lsst.afw import image as afwImage 

34from lsst.afw import detection as afwDetect 

35from lsst.afw import cameraGeom as cg 

36from lsst.afw.geom import transformFactory as tFactory 

37import lsst.pex.config as pexConfig 

38from lsst.pipe import base as pipeBase 

39from lsst.pipe.base import connectionTypes as cT 

40from lsst.pex.exceptions import InvalidParameterError 

41from lsst.meas.algorithms import LoadReferenceObjectsConfig 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43from lsst.meas.algorithms import brightStarStamps as bSS 

44from lsst.utils.timer import timeMethod 

45 

46 

47class ProcessBrightStarsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "visit", "detector")): 

49 inputExposure = cT.Input( 

50 doc="Input exposure from which to extract bright star stamps", 

51 name="calexp", 

52 storageClass="ExposureF", 

53 dimensions=("visit", "detector") 

54 ) 

55 skyCorr = cT.Input( 

56 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", 

57 name="skyCorr", 

58 storageClass="Background", 

59 dimensions=("instrument", "visit", "detector") 

60 ) 

61 refCat = cT.PrerequisiteInput( 

62 doc="Reference catalog that contains bright star positions", 

63 name="gaia_dr2_20200414", 

64 storageClass="SimpleCatalog", 

65 dimensions=("skypix",), 

66 multiple=True, 

67 deferLoad=True 

68 ) 

69 brightStarStamps = cT.Output( 

70 doc="Set of preprocessed postage stamps, each centered on a single bright star.", 

71 name="brightStarStamps", 

72 storageClass="BrightStarStamps", 

73 dimensions=("visit", "detector") 

74 ) 

75 

76 def __init__(self, *, config=None): 

77 super().__init__(config=config) 

78 if not config.doApplySkyCorr: 

79 self.inputs.remove("skyCorr") 

80 

81 

82class ProcessBrightStarsConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=ProcessBrightStarsConnections): 

84 """Configuration parameters for ProcessBrightStarsTask 

85 """ 

86 magLimit = pexConfig.Field( 

87 dtype=float, 

88 doc="Magnitude limit, in Gaia G; all stars brighter than this value will be processed", 

89 default=18 

90 ) 

91 stampSize = pexConfig.ListField( 

92 dtype=int, 

93 doc="Size of the stamps to be extracted, in pixels", 

94 default=(250, 250) 

95 ) 

96 modelStampBuffer = pexConfig.Field( 

97 dtype=float, 

98 doc="'Buffer' factor to be applied to determine the size of the stamp the processed stars will " 

99 "be saved in. This will also be the size of the extended PSF model.", 

100 default=1.1 

101 ) 

102 doRemoveDetected = pexConfig.Field( 

103 dtype=bool, 

104 doc="Whether DETECTION footprints, other than that for the central object, should be changed to " 

105 "BAD", 

106 default=True 

107 ) 

108 doApplyTransform = pexConfig.Field( 

109 dtype=bool, 

110 doc="Apply transform to bright star stamps to correct for optical distortions?", 

111 default=True 

112 ) 

113 warpingKernelName = pexConfig.ChoiceField( 

114 dtype=str, 

115 doc="Warping kernel", 

116 default="lanczos5", 

117 allowed={ 

118 "bilinear": "bilinear interpolation", 

119 "lanczos3": "Lanczos kernel of order 3", 

120 "lanczos4": "Lanczos kernel of order 4", 

121 "lanczos5": "Lanczos kernel of order 5", 

122 } 

123 ) 

124 annularFluxRadii = pexConfig.ListField( 

125 dtype=int, 

126 doc="Inner and outer radii of the annulus used to compute the AnnularFlux for normalization, " 

127 "in pixels.", 

128 default=(40, 50) 

129 ) 

130 annularFluxStatistic = pexConfig.ChoiceField( 

131 dtype=str, 

132 doc="Type of statistic to use to compute annular flux.", 

133 default="MEANCLIP", 

134 allowed={ 

135 "MEAN": "mean", 

136 "MEDIAN": "median", 

137 "MEANCLIP": "clipped mean", 

138 } 

139 ) 

140 numSigmaClip = pexConfig.Field( 

141 dtype=float, 

142 doc="Sigma for outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

143 default=4 

144 ) 

145 numIter = pexConfig.Field( 

146 dtype=int, 

147 doc="Number of iterations of outlier rejection; ignored if annularFluxStatistic != 'MEANCLIP'.", 

148 default=3 

149 ) 

150 badMaskPlanes = pexConfig.ListField( 

151 dtype=str, 

152 doc="Mask planes that, if set, lead to associated pixels not being included in the computation of the" 

153 " annular flux.", 

154 default=('BAD', 'CR', 'CROSSTALK', 'EDGE', 'NO_DATA', 'SAT', 'SUSPECT', 'UNMASKEDNAN') 

155 ) 

156 minPixelsWithinFrame = pexConfig.Field( 

157 dtype=int, 

158 doc="Minimum number of pixels that must fall within the stamp boundary for the bright star to be" 

159 " saved when its center is beyond the exposure boundary.", 

160 default=50 

161 ) 

162 doApplySkyCorr = pexConfig.Field( 

163 dtype=bool, 

164 doc="Apply full focal plane sky correction before extracting stars?", 

165 default=True 

166 ) 

167 discardNanFluxStars = pexConfig.Field( 

168 dtype=bool, 

169 doc="Should stars with NaN annular flux be discarded?", 

170 default=False 

171 ) 

172 refObjLoader = pexConfig.ConfigField( 

173 dtype=LoadReferenceObjectsConfig, 

174 doc="Reference object loader for astrometric calibration.", 

175 ) 

176 

177 def setDefaults(self): 

178 self.refObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

179 

180 

181class ProcessBrightStarsTask(pipeBase.PipelineTask): 

182 """The description of the parameters for this Task are detailed in 

183 :lsst-task:`~lsst.pipe.base.PipelineTask`. 

184 

185 Notes 

186 ----- 

187 `ProcessBrightStarsTask` is used to extract, process, and store small 

188 image cut-outs (or "postage stamps") around bright stars. It relies on 

189 three methods, called in succession: 

190 

191 `extractStamps` 

192 Find bright stars within the exposure using a reference catalog and 

193 extract a stamp centered on each. 

194 `warpStamps` 

195 Shift and warp each stamp to remove optical distortions and sample all 

196 stars on the same pixel grid. 

197 `measureAndNormalize` 

198 Compute the flux of an object in an annulus and normalize it. This is 

199 required to normalize each bright star stamp as their central pixels 

200 are likely saturated and/or contain ghosts, and cannot be used. 

201 """ 

202 ConfigClass = ProcessBrightStarsConfig 

203 _DefaultName = "processBrightStars" 

204 

205 def __init__(self, butler=None, initInputs=None, *args, **kwargs): 

206 super().__init__(*args, **kwargs) 

207 # Compute (model) stamp size depending on provided "buffer" value 

208 self.modelStampSize = [int(self.config.stampSize[0]*self.config.modelStampBuffer), 

209 int(self.config.stampSize[1]*self.config.modelStampBuffer)] 

210 # force it to be odd-sized so we have a central pixel 

211 if not self.modelStampSize[0] % 2: 

212 self.modelStampSize[0] += 1 

213 if not self.modelStampSize[1] % 2: 

214 self.modelStampSize[1] += 1 

215 # central pixel 

216 self.modelCenter = self.modelStampSize[0]//2, self.modelStampSize[1]//2 

217 # configure Gaia refcat 

218 if butler is not None: 

219 self.makeSubtask('refObjLoader', butler=butler) 

220 

221 def applySkyCorr(self, calexp, skyCorr): 

222 """Apply correction to the sky background level. 

223 

224 Sky corrections can be generated with the 'skyCorrection.py' 

225 executable in pipe_drivers. Because the sky model used by that 

226 code extends over the entire focal plane, this can produce 

227 better sky subtraction. 

228 The calexp is updated in-place. 

229 

230 Parameters 

231 ---------- 

232 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 

233 Calibrated exposure. 

234 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or None, 

235 optional 

236 Full focal plane sky correction, obtained by running 

237 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

238 """ 

239 if isinstance(calexp, afwImage.Exposure): 

240 calexp = calexp.getMaskedImage() 

241 calexp -= skyCorr.getImage() 

242 

243 def extractStamps(self, inputExposure, refObjLoader=None): 

244 """ Read position of bright stars within `inputExposure` from refCat 

245 and extract them. 

246 

247 Parameters 

248 ---------- 

249 inputExposure : `afwImage.exposure.exposure.ExposureF` 

250 The image from which bright star stamps should be extracted. 

251 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

252 Loader to find objects within a reference catalog. 

253 

254 Returns 

255 ------- 

256 result : `lsst.pipe.base.Struct` 

257 Result struct with components: 

258 

259 - ``starIms``: `list` of stamps 

260 - ``pixCenters``: `list` of corresponding coordinates to each 

261 star's center, in pixels. 

262 - ``GMags``: `list` of corresponding (Gaia) G magnitudes. 

263 - ``gaiaIds``: `np.ndarray` of corresponding unique Gaia 

264 identifiers. 

265 """ 

266 if refObjLoader is None: 

267 refObjLoader = self.refObjLoader 

268 starIms = [] 

269 pixCenters = [] 

270 GMags = [] 

271 ids = [] 

272 wcs = inputExposure.getWcs() 

273 # select stars within, or close enough to input exposure from refcat 

274 inputIm = inputExposure.maskedImage 

275 inputExpBBox = inputExposure.getBBox() 

276 dilatationExtent = geom.Extent2I(np.array(self.config.stampSize) - self.config.minPixelsWithinFrame) 

277 # TODO (DM-25894): handle catalog with stars missing from Gaia 

278 withinCalexp = refObjLoader.loadPixelBox(inputExpBBox.dilatedBy(dilatationExtent), wcs, 

279 filterName="phot_g_mean") 

280 refCat = withinCalexp.refCat 

281 # keep bright objects 

282 fluxLimit = ((self.config.magLimit*u.ABmag).to(u.nJy)).to_value() 

283 GFluxes = np.array(refCat['phot_g_mean_flux']) 

284 bright = GFluxes > fluxLimit 

285 # convert to AB magnitudes 

286 allGMags = [((gFlux*u.nJy).to(u.ABmag)).to_value() for gFlux in GFluxes[bright]] 

287 allIds = refCat.columns.extract("id", where=bright)["id"] 

288 selectedColumns = refCat.columns.extract('coord_ra', 'coord_dec', where=bright) 

289 for j, (ra, dec) in enumerate(zip(selectedColumns["coord_ra"], selectedColumns["coord_dec"])): 

290 sp = geom.SpherePoint(ra, dec, geom.radians) 

291 cpix = wcs.skyToPixel(sp) 

292 try: 

293 starIm = inputExposure.getCutout(sp, geom.Extent2I(self.config.stampSize)) 

294 except InvalidParameterError: 

295 # star is beyond boundary 

296 bboxCorner = np.array(cpix) - np.array(self.config.stampSize)/2 

297 # compute bbox as it would be otherwise 

298 idealBBox = geom.Box2I(geom.Point2I(bboxCorner), geom.Extent2I(self.config.stampSize)) 

299 clippedStarBBox = geom.Box2I(idealBBox) 

300 clippedStarBBox.clip(inputExpBBox) 

301 if clippedStarBBox.getArea() > 0: 

302 # create full-sized stamp with all pixels 

303 # flagged as NO_DATA 

304 starIm = afwImage.ExposureF(bbox=idealBBox) 

305 starIm.image[:] = np.nan 

306 starIm.mask.set(inputExposure.mask.getPlaneBitMask("NO_DATA")) 

307 # recover pixels from intersection with the exposure 

308 clippedIm = inputIm.Factory(inputIm, clippedStarBBox) 

309 starIm.maskedImage[clippedStarBBox] = clippedIm 

310 # set detector and wcs, used in warpStars 

311 starIm.setDetector(inputExposure.getDetector()) 

312 starIm.setWcs(inputExposure.getWcs()) 

313 else: 

314 continue 

315 if self.config.doRemoveDetected: 

316 # give detection footprint of other objects the BAD flag 

317 detThreshold = afwDetect.Threshold(starIm.mask.getPlaneBitMask("DETECTED"), 

318 afwDetect.Threshold.BITMASK) 

319 omask = afwDetect.FootprintSet(starIm.mask, detThreshold) 

320 allFootprints = omask.getFootprints() 

321 otherFootprints = [] 

322 for fs in allFootprints: 

323 if not fs.contains(geom.Point2I(cpix)): 

324 otherFootprints.append(fs) 

325 nbMatchingFootprints = len(allFootprints) - len(otherFootprints) 

326 if not nbMatchingFootprints == 1: 

327 self.log.warning("Failed to uniquely identify central DETECTION footprint for star " 

328 "%s; found %d footprints instead.", 

329 allIds[j], nbMatchingFootprints) 

330 omask.setFootprints(otherFootprints) 

331 omask.setMask(starIm.mask, "BAD") 

332 starIms.append(starIm) 

333 pixCenters.append(cpix) 

334 GMags.append(allGMags[j]) 

335 ids.append(allIds[j]) 

336 return pipeBase.Struct(starIms=starIms, 

337 pixCenters=pixCenters, 

338 GMags=GMags, 

339 gaiaIds=ids) 

340 

341 def warpStamps(self, stamps, pixCenters): 

342 """Warps and shifts all given stamps so they are sampled on the same 

343 pixel grid and centered on the central pixel. This includes rotating 

344 the stamp depending on detector orientation. 

345 

346 Parameters 

347 ---------- 

348 stamps : `collections.abc.Sequence` 

349 [`afwImage.exposure.exposure.ExposureF`] 

350 Image cutouts centered on a single object. 

351 pixCenters : `collections.abc.Sequence` [`geom.Point2D`] 

352 Positions of each object's center (as obtained from the refCat), 

353 in pixels. 

354 

355 Returns 

356 ------- 

357 result : `lsst.pipe.base.Struct` 

358 Result struct with components: 

359 

360 - ``warpedStars``: 

361 `list` [`afwImage.maskedImage.maskedImage.MaskedImage`] of 

362 stamps of warped stars 

363 - ``warpTransforms``: 

364 `list` [`afwGeom.TransformPoint2ToPoint2`] of 

365 the corresponding Transform from the initial star stamp to 

366 the common model grid 

367 - ``xy0s``: 

368 `list` [`geom.Point2I`] of coordinates of the bottom-left 

369 pixels of each stamp, before rotation 

370 - ``nb90Rots``: `int`, the number of 90 degrees rotations required 

371 to compensate for detector orientation 

372 """ 

373 # warping control; only contains shiftingALg provided in config 

374 warpCont = afwMath.WarpingControl(self.config.warpingKernelName) 

375 # Compare model to star stamp sizes 

376 bufferPix = (self.modelStampSize[0] - self.config.stampSize[0], 

377 self.modelStampSize[1] - self.config.stampSize[1]) 

378 # Initialize detector instance (note all stars were extracted from an 

379 # exposure from the same detector) 

380 det = stamps[0].getDetector() 

381 # Define correction for optical distortions 

382 if self.config.doApplyTransform: 

383 pixToTan = det.getTransform(cg.PIXELS, cg.TAN_PIXELS) 

384 else: 

385 pixToTan = tFactory.makeIdentityTransform() 

386 # Array of all possible rotations for detector orientation: 

387 possibleRots = np.array([k*np.pi/2 for k in range(4)]) 

388 # determine how many, if any, rotations are required 

389 yaw = det.getOrientation().getYaw() 

390 nb90Rots = np.argmin(np.abs(possibleRots - float(yaw))) 

391 

392 # apply transformation to each star 

393 warpedStars, warpTransforms, xy0s = [], [], [] 

394 for star, cent in zip(stamps, pixCenters): 

395 # (re)create empty destination image 

396 destImage = afwImage.MaskedImageF(*self.modelStampSize) 

397 bottomLeft = geom.Point2D(star.image.getXY0()) 

398 newBottomLeft = pixToTan.applyForward(bottomLeft) 

399 newBottomLeft.setX(newBottomLeft.getX() - bufferPix[0]/2) 

400 newBottomLeft.setY(newBottomLeft.getY() - bufferPix[1]/2) 

401 # Convert to int 

402 newBottomLeft = geom.Point2I(newBottomLeft) 

403 # Set origin and save it 

404 destImage.setXY0(newBottomLeft) 

405 xy0s.append(newBottomLeft) 

406 

407 # Define linear shifting to recenter stamps 

408 newCenter = pixToTan.applyForward(cent) # center of warped star 

409 shift = self.modelCenter[0] + newBottomLeft[0] - newCenter[0],\ 

410 self.modelCenter[1] + newBottomLeft[1] - newCenter[1] 

411 affineShift = geom.AffineTransform(shift) 

412 shiftTransform = tFactory.makeTransform(affineShift) 

413 

414 # Define full transform (warp and shift) 

415 starWarper = pixToTan.then(shiftTransform) 

416 

417 # Apply it 

418 goodPix = afwMath.warpImage(destImage, star.getMaskedImage(), 

419 starWarper, warpCont) 

420 if not goodPix: 

421 self.log.debug("Warping of a star failed: no good pixel in output") 

422 

423 # Arbitrarily set origin of shifted star to 0 

424 destImage.setXY0(0, 0) 

425 

426 # Apply rotation if appropriate 

427 if nb90Rots: 

428 destImage = afwMath.rotateImageBy90(destImage, nb90Rots) 

429 warpedStars.append(destImage.clone()) 

430 warpTransforms.append(starWarper) 

431 return pipeBase.Struct(warpedStars=warpedStars, warpTransforms=warpTransforms, xy0s=xy0s, 

432 nb90Rots=nb90Rots) 

433 

434 @timeMethod 

435 def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None): 

436 """Identify bright stars within an exposure using a reference catalog, 

437 extract stamps around each, then preprocess them. The preprocessing 

438 steps are: shifting, warping and potentially rotating them to the same 

439 pixel grid; computing their annular flux and normalizing them. 

440 

441 Parameters 

442 ---------- 

443 inputExposure : `afwImage.exposure.exposure.ExposureF` 

444 The image from which bright star stamps should be extracted. 

445 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

446 Loader to find objects within a reference catalog. 

447 dataId : `dict` or `lsst.daf.butler.DataCoordinate` 

448 The dataId of the exposure (and detector) bright stars should be 

449 extracted from. 

450 skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``, 

451 optional 

452 Full focal plane sky correction, obtained by running 

453 `lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`. 

454 

455 Returns 

456 ------- 

457 result : `lsst.pipe.base.Struct` 

458 Result struct with component: 

459 

460 - ``brightStarStamps``: ``bSS.BrightStarStamps`` 

461 """ 

462 if self.config.doApplySkyCorr: 

463 self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).", 

464 dataId) 

465 self.applySkyCorr(inputExposure, skyCorr) 

466 self.log.info("Extracting bright stars from exposure %s", dataId) 

467 # Extract stamps around bright stars 

468 extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader) 

469 if not extractedStamps.starIms: 

470 self.log.info("No suitable bright star found.") 

471 return None 

472 # Warp (and shift, and potentially rotate) them 

473 self.log.info("Applying warp and/or shift to %i star stamps from exposure %s", 

474 len(extractedStamps.starIms), dataId) 

475 warpOutputs = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters) 

476 warpedStars = warpOutputs.warpedStars 

477 xy0s = warpOutputs.xy0s 

478 brightStarList = [bSS.BrightStarStamp(stamp_im=warp, 

479 archive_element=transform, 

480 position=xy0s[j], 

481 gaiaGMag=extractedStamps.GMags[j], 

482 gaiaId=extractedStamps.gaiaIds[j]) 

483 for j, (warp, transform) in 

484 enumerate(zip(warpedStars, warpOutputs.warpTransforms))] 

485 # Compute annularFlux and normalize 

486 self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s", 

487 len(warpedStars), dataId) 

488 # annularFlux statistic set-up, excluding mask planes 

489 statsControl = afwMath.StatisticsControl() 

490 statsControl.setNumSigmaClip(self.config.numSigmaClip) 

491 statsControl.setNumIter(self.config.numIter) 

492 innerRadius, outerRadius = self.config.annularFluxRadii 

493 statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic) 

494 brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList, 

495 innerRadius=innerRadius, 

496 outerRadius=outerRadius, 

497 nb90Rots=warpOutputs.nb90Rots, 

498 imCenter=self.modelCenter, 

499 use_archive=True, 

500 statsControl=statsControl, 

501 statsFlag=statsFlag, 

502 badMaskPlanes=self.config.badMaskPlanes, 

503 discardNanFluxObjects=( 

504 self.config.discardNanFluxStars)) 

505 return pipeBase.Struct(brightStarStamps=brightStarStamps) 

506 

507 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

508 inputs = butlerQC.get(inputRefs) 

509 inputs['dataId'] = str(butlerQC.quantum.dataId) 

510 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

511 for ref in inputRefs.refCat], 

512 refCats=inputs.pop("refCat"), 

513 config=self.config.refObjLoader) 

514 output = self.run(**inputs, refObjLoader=refObjLoader) 

515 if output: 

516 butlerQC.put(output, outputRefs)