Coverage for python/lsst/meas/extensions/psfex/psfexPsfDeterminer.py: 11%

224 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-16 09:42 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import numpy as np 

24 

25import lsst.daf.base as dafBase 

26import lsst.pex.config as pexConfig 

27import lsst.pex.exceptions as pexExcept 

28import lsst.geom as geom 

29import lsst.afw.geom.ellipses as afwEll 

30import lsst.afw.display as afwDisplay 

31import lsst.afw.image as afwImage 

32import lsst.afw.math as afwMath 

33import lsst.meas.algorithms as measAlg 

34import lsst.meas.algorithms.utils as maUtils 

35import lsst.meas.extensions.psfex as psfex 

36 

37 

38class PsfexPsfDeterminerConfig(measAlg.BasePsfDeterminerConfig): 

39 spatialOrder = pexConfig.Field[int]( 39 ↛ exitline 39 didn't jump to the function exit

40 doc="specify spatial order for PSF kernel creation", 

41 default=2, 

42 check=lambda x: x >= 1, 

43 ) 

44 sizeCellX = pexConfig.Field[int]( 44 ↛ exitline 44 didn't jump to the function exit

45 doc="size of cell used to determine PSF (pixels, column direction)", 

46 default=256, 

47 # minValue = 10, 

48 check=lambda x: x >= 10, 

49 ) 

50 sizeCellY = pexConfig.Field[int]( 50 ↛ exitline 50 didn't jump to the function exit

51 doc="size of cell used to determine PSF (pixels, row direction)", 

52 default=sizeCellX.default, 

53 # minValue = 10, 

54 check=lambda x: x >= 10, 

55 ) 

56 samplingSize = pexConfig.Field[float]( 

57 doc="Resolution of the internal PSF model relative to the pixel size; " 

58 "e.g. 0.5 is equal to 2x oversampling", 

59 default=0.5, 

60 ) 

61 badMaskBits = pexConfig.ListField[str]( 

62 doc="List of mask bits which cause a source to be rejected as bad " 

63 "N.b. INTRP is used specially in PsfCandidateSet; it means \"Contaminated by neighbour\"", 

64 default=["INTRP", "SAT"], 

65 ) 

66 psfexBasis = pexConfig.ChoiceField[str]( 

67 doc="BASIS value given to psfex. PIXEL_AUTO will use the requested samplingSize only if " 

68 "the FWHM < 3 pixels. Otherwise, it will use samplingSize=1. PIXEL will always use the " 

69 "requested samplingSize", 

70 allowed={ 

71 "PIXEL": "Always use requested samplingSize", 

72 "PIXEL_AUTO": "Only use requested samplingSize when FWHM < 3", 

73 }, 

74 default='PIXEL_AUTO', 

75 optional=False, 

76 ) 

77 tolerance = pexConfig.Field[float]( 

78 doc="tolerance of spatial fitting", 

79 default=1e-2, 

80 ) 

81 lam = pexConfig.Field[float]( 

82 doc="floor for variance is lam*data", 

83 default=0.05, 

84 ) 

85 reducedChi2ForPsfCandidates = pexConfig.Field[float]( 

86 doc="for psf candidate evaluation", 

87 default=2.0, 

88 ) 

89 spatialReject = pexConfig.Field[float]( 

90 doc="Rejection threshold (stdev) for candidates based on spatial fit", 

91 default=3.0, 

92 ) 

93 recentroid = pexConfig.Field[bool]( 

94 doc="Should PSFEX be permitted to recentroid PSF candidates?", 

95 default=False, 

96 ) 

97 kernelSize = pexConfig.Field[int]( 

98 doc=("Size of the postage stamp around each star that is extracted for fitting." 

99 "Note: this reflects the oversampling setting of the psf, set by `samplingSize`;" 

100 "e.g. `samplingSize=0.5` would require this value to be 2x what you expect."), 

101 default=None, 

102 optional=True, 

103 deprecated="'kernelSize' is deprecated and will be removed in v25. Use `stampSize` instead.", 

104 ) 

105 

106 def setDefaults(self): 

107 super().setDefaults() 

108 self.stampSize = 41 

109 

110 

111class PsfexPsfDeterminerTask(measAlg.BasePsfDeterminerTask): 

112 ConfigClass = PsfexPsfDeterminerConfig 

113 

114 def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): 

115 """Determine a PSFEX PSF model for an exposure given a list of PSF 

116 candidates. 

117 

118 Parameters 

119 ---------- 

120 exposure: `lsst.afw.image.Exposure` 

121 Exposure containing the PSF candidates. 

122 psfCandidateList: iterable of `lsst.meas.algorithms.PsfCandidate` 

123 Sequence of PSF candidates typically obtained by detecting sources 

124 and then running them through a star selector. 

125 metadata: metadata, optional 

126 A home for interesting tidbits of information. 

127 flagKey: `lsst.afw.table.Key`, optional 

128 Schema key used to mark sources actually used in PSF determination. 

129 

130 Returns 

131 ------- 

132 psf: `lsst.meas.extensions.psfex.PsfexPsf` 

133 The determined PSF. 

134 """ 

135 

136 import lsstDebug 

137 display = lsstDebug.Info(__name__).display 

138 displayExposure = display and \ 

139 lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

140 displayPsfComponents = display and \ 

141 lsstDebug.Info(__name__).displayPsfComponents # show the basis functions 

142 showBadCandidates = display and \ 

143 lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks) 

144 displayResiduals = display and \ 

145 lsstDebug.Info(__name__).displayResiduals # show residuals 

146 displayPsfMosaic = display and \ 

147 lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) 

148 normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals 

149 afwDisplay.setDefaultMaskTransparency(75) 

150 # Normalise residuals by object amplitude 

151 

152 mi = exposure.getMaskedImage() 

153 

154 nCand = len(psfCandidateList) 

155 if nCand == 0: 

156 raise RuntimeError("No PSF candidates supplied.") 

157 # 

158 # How big should our PSF models be? 

159 # 

160 if display: # only needed for debug plots 

161 # construct and populate a spatial cell set 

162 bbox = mi.getBBox(afwImage.PARENT) 

163 psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) 

164 else: 

165 psfCellSet = None 

166 

167 sizes = np.empty(nCand) 

168 for i, psfCandidate in enumerate(psfCandidateList): 

169 try: 

170 if psfCellSet: 

171 psfCellSet.insertCandidate(psfCandidate) 

172 except Exception as e: 

173 self.log.error("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) 

174 continue 

175 

176 source = psfCandidate.getSource() 

177 quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) 

178 rmsSize = quad.getTraceRadius() 

179 sizes[i] = rmsSize 

180 

181 # TODO: Keep only the if block and remove the else blocks in DM-36311 

182 if self.config.stampSize: 

183 pixKernelSize = self.config.stampSize 

184 actualKernelSize = int(2*np.floor(0.5*pixKernelSize/self.config.samplingSize) + 1) 

185 elif self.config.kernelSize >= 15: 

186 self.log.warning("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value") 

187 actualKernelSize = self.config.kernelSize 

188 pixKernelSize = int(actualKernelSize*self.config.samplingSize) 

189 if pixKernelSize % 2 == 0: 

190 pixKernelSize += 1 

191 else: 

192 actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1 

193 # TODO: DM-36311 Remove deprecated kernelSizeMin and kernelSizeMax 

194 if actualKernelSize < self.config.kernelSizeMin: 

195 actualKernelSize = self.config.kernelSizeMin 

196 if actualKernelSize > self.config.kernelSizeMax: 

197 actualKernelSize = self.config.kernelSizeMax 

198 

199 pixKernelSize = int(actualKernelSize*self.config.samplingSize) 

200 if pixKernelSize % 2 == 0: 

201 pixKernelSize += 1 

202 

203 if display: 

204 rms = np.median(sizes) 

205 self.log.debug("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)", 

206 rms, 2*np.sqrt(2*np.log(2))*rms) 

207 

208 self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize) 

209 

210 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX 

211 # 

212 # Insert the good candidates into the set 

213 # 

214 defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex") 

215 args_md = dafBase.PropertySet() 

216 args_md.set("BASIS_TYPE", str(self.config.psfexBasis)) 

217 args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder)) 

218 args_md.set("PSF_SIZE", str(actualKernelSize)) 

219 args_md.set("PSF_SAMPLING", str(self.config.samplingSize)) 

220 prefs = psfex.Prefs(defaultsFile, args_md) 

221 prefs.setCommandLine([]) 

222 prefs.addCatalog("psfexPsfDeterminer") 

223 

224 prefs.use() 

225 principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN) 

226 if False else psfex.Context.KEEPHIDDEN) 

227 context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(), 

228 prefs.getGroupDeg(), principalComponentExclusionFlag) 

229 psfSet = psfex.Set(context) 

230 psfSet.setVigSize(pixKernelSize, pixKernelSize) 

231 psfSet.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes)) 

232 psfSet.setRecentroid(self.config.recentroid) 

233 

234 catindex, ext = 0, 0 

235 backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() 

236 ccd = exposure.getDetector() 

237 if ccd: 

238 gain = np.mean(np.array([a.getGain() for a in ccd])) 

239 else: 

240 gain = 1.0 

241 self.log.warning("Setting gain to %g", gain) 

242 

243 contextvalp = [] 

244 for i, key in enumerate(context.getName()): 

245 if key[0] == ':': 

246 try: 

247 contextvalp.append(exposure.getMetadata().getScalar(key[1:])) 

248 except KeyError as e: 

249 raise RuntimeError("%s parameter not found in the header of %s" % 

250 (key[1:], prefs.getContextName())) from e 

251 else: 

252 try: 

253 contextvalp.append(np.array([psfCandidateList[_].getSource().get(key) 

254 for _ in range(nCand)])) 

255 except KeyError as e: 

256 raise RuntimeError("%s parameter not found" % (key,)) from e 

257 psfSet.setContextname(i, key) 

258 

259 if display: 

260 frame = 0 

261 if displayExposure: 

262 disp = afwDisplay.Display(frame=frame) 

263 disp.mtv(exposure, title="psf determination") 

264 

265 badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits) 

266 fluxName = prefs.getPhotfluxRkey() 

267 fluxFlagName = "base_" + fluxName + "_flag" 

268 

269 xpos, ypos = [], [] 

270 for i, psfCandidate in enumerate(psfCandidateList): 

271 source = psfCandidate.getSource() 

272 

273 # skip sources with bad centroids 

274 xc, yc = source.getX(), source.getY() 

275 if not np.isfinite(xc) or not np.isfinite(yc): 

276 continue 

277 # skip flagged sources 

278 if fluxFlagName in source.schema and source.get(fluxFlagName): 

279 continue 

280 # skip nonfinite and negative sources 

281 flux = source.get(fluxName) 

282 if flux < 0 or not np.isfinite(flux): 

283 continue 

284 

285 try: 

286 pstamp = psfCandidate.getMaskedImage(pixKernelSize, pixKernelSize).clone() 

287 except pexExcept.LengthError: 

288 self.log.warning("Could not get stamp image for psfCandidate: %s with kernel size: %s", 

289 psfCandidate, pixKernelSize) 

290 continue 

291 

292 # From this point, we're configuring the "sample" (PSFEx's version 

293 # of a PSF candidate). 

294 # Having created the sample, we must proceed to configure it, and 

295 # then fini (finalize), or it will be malformed. 

296 try: 

297 sample = psfSet.newSample() 

298 sample.setCatindex(catindex) 

299 sample.setExtindex(ext) 

300 sample.setObjindex(i) 

301 

302 imArray = pstamp.getImage().getArray() 

303 imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \ 

304 -2*psfex.BIG 

305 sample.setVig(imArray) 

306 

307 sample.setNorm(flux) 

308 sample.setBacknoise2(backnoise2) 

309 sample.setGain(gain) 

310 sample.setX(xc) 

311 sample.setY(yc) 

312 sample.setFluxrad(sizes[i]) 

313 

314 for j in range(psfSet.getNcontext()): 

315 sample.setContext(j, float(contextvalp[j][i])) 

316 except Exception as e: 

317 self.log.error("Exception when processing sample at (%f,%f): %s", xc, yc, e) 

318 continue 

319 else: 

320 psfSet.finiSample(sample) 

321 

322 xpos.append(xc) # for QA 

323 ypos.append(yc) 

324 

325 if displayExposure: 

326 with disp.Buffering(): 

327 disp.dot("o", xc, yc, ctype=afwDisplay.CYAN, size=4) 

328 

329 if psfSet.getNsample() == 0: 

330 raise RuntimeError("No good PSF candidates to pass to PSFEx") 

331 

332 # ---- Update min and max and then the scaling 

333 for i in range(psfSet.getNcontext()): 

334 cmin = contextvalp[i].min() 

335 cmax = contextvalp[i].max() 

336 psfSet.setContextScale(i, cmax - cmin) 

337 psfSet.setContextOffset(i, (cmin + cmax)/2.0) 

338 

339 # Don't waste memory! 

340 psfSet.trimMemory() 

341 

342 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX 

343 # 

344 # Do a PSFEX decomposition of those PSF candidates 

345 # 

346 fields = [] 

347 field = psfex.Field("Unknown") 

348 field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), psfSet.getNsample()) 

349 field.finalize() 

350 

351 fields.append(field) 

352 

353 sets = [] 

354 sets.append(psfSet) 

355 

356 psfex.makeit(fields, sets) 

357 psfs = field.getPsfs() 

358 

359 # Flag which objects were actually used in psfex by 

360 good_indices = [] 

361 for i in range(sets[0].getNsample()): 

362 index = sets[0].getSample(i).getObjindex() 

363 if index > -1: 

364 good_indices.append(index) 

365 

366 if flagKey is not None: 

367 for i, psfCandidate in enumerate(psfCandidateList): 

368 source = psfCandidate.getSource() 

369 if i in good_indices: 

370 source.set(flagKey, True) 

371 

372 xpos = np.array(xpos) 

373 ypos = np.array(ypos) 

374 numGoodStars = len(good_indices) 

375 avgX, avgY = np.mean(xpos), np.mean(ypos) 

376 

377 psf = psfex.PsfexPsf(psfs[0], geom.Point2D(avgX, avgY)) 

378 

379 # If there are too few stars, the PSFEx psf model will reduce the order 

380 # to 0, which the Science Pipelines code cannot handle (see 

381 # https://github.com/lsst/meas_extensions_psfex/blob/f0d5218b5446faf5e39edc30e31d2e6f673ef294/src/PsfexPsf.cc#L118 

382 # ). The easiest way to test for this condition is trying to compute 

383 # the PSF kernel and checking for an InvalidParameterError. 

384 try: 

385 _ = psf.getKernel(psf.getAveragePosition()) 

386 except pexExcept.InvalidParameterError: 

387 raise RuntimeError("Failed to determine psfex psf: too few good stars.") 

388 

389 # 

390 # Display code for debugging 

391 # 

392 if display: 

393 assert psfCellSet is not None 

394 

395 if displayExposure: 

396 maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True, 

397 symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, 

398 size=8, display=disp) 

399 if displayResiduals: 

400 disp4 = afwDisplay.Display(frame=4) 

401 maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, 

402 normalize=normalizeResiduals, 

403 showBadCandidates=showBadCandidates) 

404 if displayPsfComponents: 

405 disp6 = afwDisplay.Display(frame=6) 

406 maUtils.showPsf(psf, display=disp6) 

407 if displayPsfMosaic: 

408 disp7 = afwDisplay.Display(frame=7) 

409 maUtils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) 

410 disp.scale('linear', 0, 1) 

411 # 

412 # Generate some QA information 

413 # 

414 # Count PSF stars 

415 # 

416 if metadata is not None: 

417 metadata["spatialFitChi2"] = np.nan 

418 metadata["numAvailStars"] = nCand 

419 metadata["numGoodStars"] = numGoodStars 

420 metadata["avgX"] = avgX 

421 metadata["avgY"] = avgY 

422 

423 return psf, psfCellSet 

424 

425 

426measAlg.psfDeterminerRegistry.register("psfex", PsfexPsfDeterminerTask)