Coverage for python/lsst/meas/extensions/psfex/psfexPsfDeterminer.py: 12%

209 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-22 10:31 +0000

1# This file is part of meas_extensions_psfex. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ("PsfexPsfDeterminerConfig", "PsfexPsfDeterminerTask") 

23 

24import os 

25import numpy as np 

26 

27import lsst.daf.base as dafBase 

28import lsst.pex.config as pexConfig 

29import lsst.pex.exceptions as pexExcept 

30import lsst.geom as geom 

31import lsst.afw.geom.ellipses as afwEll 

32import lsst.afw.display as afwDisplay 

33import lsst.afw.image as afwImage 

34import lsst.afw.math as afwMath 

35import lsst.meas.algorithms as measAlg 

36import lsst.meas.algorithms.utils as maUtils 

37import lsst.meas.extensions.psfex as psfex 

38 

39 

40class PsfexPsfDeterminerConfig(measAlg.BasePsfDeterminerConfig): 

41 spatialOrder = pexConfig.Field[int]( 41 ↛ exitline 41 didn't jump to the function exit

42 doc="specify spatial order for PSF kernel creation", 

43 default=2, 

44 check=lambda x: x >= 1, 

45 ) 

46 sizeCellX = pexConfig.Field[int]( 46 ↛ exitline 46 didn't jump to the function exit

47 doc="size of cell used to determine PSF (pixels, column direction)", 

48 default=256, 

49 # minValue = 10, 

50 check=lambda x: x >= 10, 

51 ) 

52 sizeCellY = pexConfig.Field[int]( 52 ↛ exitline 52 didn't jump to the function exit

53 doc="size of cell used to determine PSF (pixels, row direction)", 

54 default=sizeCellX.default, 

55 # minValue = 10, 

56 check=lambda x: x >= 10, 

57 ) 

58 samplingSize = pexConfig.Field[float]( 

59 doc="Resolution of the internal PSF model relative to the pixel size; " 

60 "e.g. 0.5 is equal to 2x oversampling", 

61 default=0.5, 

62 ) 

63 badMaskBits = pexConfig.ListField[str]( 

64 doc="List of mask bits which cause a source to be rejected as bad " 

65 "N.b. INTRP is used specially in PsfCandidateSet; it means \"Contaminated by neighbour\"", 

66 default=["INTRP", "SAT"], 

67 ) 

68 psfexBasis = pexConfig.ChoiceField[str]( 

69 doc="BASIS value given to psfex. PIXEL_AUTO will use the requested samplingSize only if " 

70 "the FWHM < 3 pixels. Otherwise, it will use samplingSize=1. PIXEL will always use the " 

71 "requested samplingSize", 

72 allowed={ 

73 "PIXEL": "Always use requested samplingSize", 

74 "PIXEL_AUTO": "Only use requested samplingSize when FWHM < 3", 

75 }, 

76 default='PIXEL_AUTO', 

77 optional=False, 

78 ) 

79 tolerance = pexConfig.Field[float]( 

80 doc="tolerance of spatial fitting", 

81 default=1e-2, 

82 ) 

83 lam = pexConfig.Field[float]( 

84 doc="floor for variance is lam*data", 

85 default=0.05, 

86 ) 

87 reducedChi2ForPsfCandidates = pexConfig.Field[float]( 

88 doc="for psf candidate evaluation", 

89 default=2.0, 

90 ) 

91 spatialReject = pexConfig.Field[float]( 

92 doc="Rejection threshold (stdev) for candidates based on spatial fit", 

93 default=3.0, 

94 ) 

95 recentroid = pexConfig.Field[bool]( 

96 doc="Should PSFEX be permitted to recentroid PSF candidates?", 

97 default=False, 

98 ) 

99 

100 def setDefaults(self): 

101 super().setDefaults() 

102 self.stampSize = 41 

103 

104 

105class PsfexPsfDeterminerTask(measAlg.BasePsfDeterminerTask): 

106 ConfigClass = PsfexPsfDeterminerConfig 

107 

108 def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): 

109 """Determine a PSFEX PSF model for an exposure given a list of PSF 

110 candidates. 

111 

112 Parameters 

113 ---------- 

114 exposure: `lsst.afw.image.Exposure` 

115 Exposure containing the PSF candidates. 

116 psfCandidateList: iterable of `lsst.meas.algorithms.PsfCandidate` 

117 Sequence of PSF candidates typically obtained by detecting sources 

118 and then running them through a star selector. 

119 metadata: metadata, optional 

120 A home for interesting tidbits of information. 

121 flagKey: `lsst.afw.table.Key`, optional 

122 Schema key used to mark sources actually used in PSF determination. 

123 

124 Returns 

125 ------- 

126 psf: `lsst.meas.extensions.psfex.PsfexPsf` 

127 The determined PSF. 

128 """ 

129 

130 import lsstDebug 

131 display = lsstDebug.Info(__name__).display 

132 displayExposure = display and \ 

133 lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells 

134 displayPsfComponents = display and \ 

135 lsstDebug.Info(__name__).displayPsfComponents # show the basis functions 

136 showBadCandidates = display and \ 

137 lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks) 

138 displayResiduals = display and \ 

139 lsstDebug.Info(__name__).displayResiduals # show residuals 

140 displayPsfMosaic = display and \ 

141 lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) 

142 normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals 

143 afwDisplay.setDefaultMaskTransparency(75) 

144 # Normalise residuals by object amplitude 

145 

146 mi = exposure.getMaskedImage() 

147 

148 nCand = len(psfCandidateList) 

149 if nCand == 0: 

150 raise RuntimeError("No PSF candidates supplied.") 

151 # 

152 # How big should our PSF models be? 

153 # 

154 if display: # only needed for debug plots 

155 # construct and populate a spatial cell set 

156 bbox = mi.getBBox(afwImage.PARENT) 

157 psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) 

158 else: 

159 psfCellSet = None 

160 

161 sizes = np.empty(nCand) 

162 for i, psfCandidate in enumerate(psfCandidateList): 

163 try: 

164 if psfCellSet: 

165 psfCellSet.insertCandidate(psfCandidate) 

166 except Exception as e: 

167 self.log.error("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) 

168 continue 

169 

170 source = psfCandidate.getSource() 

171 quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) 

172 rmsSize = quad.getTraceRadius() 

173 sizes[i] = rmsSize 

174 

175 pixKernelSize = self.config.stampSize 

176 actualKernelSize = int(2*np.floor(0.5*pixKernelSize/self.config.samplingSize) + 1) 

177 

178 if display: 

179 rms = np.median(sizes) 

180 self.log.debug("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)", 

181 rms, 2*np.sqrt(2*np.log(2))*rms) 

182 

183 self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize) 

184 

185 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX 

186 # 

187 # Insert the good candidates into the set 

188 # 

189 defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex") 

190 args_md = dafBase.PropertySet() 

191 args_md.set("BASIS_TYPE", str(self.config.psfexBasis)) 

192 args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder)) 

193 args_md.set("PSF_SIZE", str(actualKernelSize)) 

194 args_md.set("PSF_SAMPLING", str(self.config.samplingSize)) 

195 prefs = psfex.Prefs(defaultsFile, args_md) 

196 prefs.setCommandLine([]) 

197 prefs.addCatalog("psfexPsfDeterminer") 

198 

199 prefs.use() 

200 principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN) 

201 if False else psfex.Context.KEEPHIDDEN) 

202 context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(), 

203 prefs.getGroupDeg(), principalComponentExclusionFlag) 

204 psfSet = psfex.Set(context) 

205 psfSet.setVigSize(pixKernelSize, pixKernelSize) 

206 psfSet.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes)) 

207 psfSet.setRecentroid(self.config.recentroid) 

208 

209 catindex, ext = 0, 0 

210 backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() 

211 ccd = exposure.getDetector() 

212 if ccd: 

213 gain = np.mean(np.array([a.getGain() for a in ccd])) 

214 else: 

215 gain = 1.0 

216 self.log.warning("Setting gain to %g", gain) 

217 

218 contextvalp = [] 

219 for i, key in enumerate(context.getName()): 

220 if key[0] == ':': 

221 try: 

222 contextvalp.append(exposure.getMetadata().getScalar(key[1:])) 

223 except KeyError as e: 

224 raise RuntimeError("%s parameter not found in the header of %s" % 

225 (key[1:], prefs.getContextName())) from e 

226 else: 

227 try: 

228 contextvalp.append(np.array([psfCandidateList[_].getSource().get(key) 

229 for _ in range(nCand)])) 

230 except KeyError as e: 

231 raise RuntimeError("%s parameter not found" % (key,)) from e 

232 psfSet.setContextname(i, key) 

233 

234 if display: 

235 frame = 0 

236 if displayExposure: 

237 disp = afwDisplay.Display(frame=frame) 

238 disp.mtv(exposure, title="psf determination") 

239 

240 badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits) 

241 fluxName = prefs.getPhotfluxRkey() 

242 fluxFlagName = "base_" + fluxName + "_flag" 

243 

244 xpos, ypos = [], [] 

245 for i, psfCandidate in enumerate(psfCandidateList): 

246 source = psfCandidate.getSource() 

247 

248 # skip sources with bad centroids 

249 xc, yc = source.getX(), source.getY() 

250 if not np.isfinite(xc) or not np.isfinite(yc): 

251 continue 

252 # skip flagged sources 

253 if fluxFlagName in source.schema and source.get(fluxFlagName): 

254 continue 

255 # skip nonfinite and negative sources 

256 flux = source.get(fluxName) 

257 if flux < 0 or not np.isfinite(flux): 

258 continue 

259 

260 try: 

261 pstamp = psfCandidate.getMaskedImage(pixKernelSize, pixKernelSize).clone() 

262 except pexExcept.LengthError: 

263 self.log.warning("Could not get stamp image for psfCandidate: %s with kernel size: %s", 

264 psfCandidate, pixKernelSize) 

265 continue 

266 

267 # From this point, we're configuring the "sample" (PSFEx's version 

268 # of a PSF candidate). 

269 # Having created the sample, we must proceed to configure it, and 

270 # then fini (finalize), or it will be malformed. 

271 try: 

272 sample = psfSet.newSample() 

273 sample.setCatindex(catindex) 

274 sample.setExtindex(ext) 

275 sample.setObjindex(i) 

276 

277 imArray = pstamp.getImage().getArray() 

278 imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \ 

279 -2*psfex.BIG 

280 sample.setVig(imArray) 

281 

282 sample.setNorm(flux) 

283 sample.setBacknoise2(backnoise2) 

284 sample.setGain(gain) 

285 sample.setX(xc) 

286 sample.setY(yc) 

287 sample.setFluxrad(sizes[i]) 

288 

289 for j in range(psfSet.getNcontext()): 

290 sample.setContext(j, float(contextvalp[j][i])) 

291 except Exception as e: 

292 self.log.error("Exception when processing sample at (%f,%f): %s", xc, yc, e) 

293 continue 

294 else: 

295 psfSet.finiSample(sample) 

296 

297 xpos.append(xc) # for QA 

298 ypos.append(yc) 

299 

300 if displayExposure: 

301 with disp.Buffering(): 

302 disp.dot("o", xc, yc, ctype=afwDisplay.CYAN, size=4) 

303 

304 if psfSet.getNsample() == 0: 

305 raise RuntimeError("No good PSF candidates to pass to PSFEx") 

306 

307 # ---- Update min and max and then the scaling 

308 for i in range(psfSet.getNcontext()): 

309 cmin = contextvalp[i].min() 

310 cmax = contextvalp[i].max() 

311 psfSet.setContextScale(i, cmax - cmin) 

312 psfSet.setContextOffset(i, (cmin + cmax)/2.0) 

313 

314 # Don't waste memory! 

315 psfSet.trimMemory() 

316 

317 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX 

318 # 

319 # Do a PSFEX decomposition of those PSF candidates 

320 # 

321 fields = [] 

322 field = psfex.Field("Unknown") 

323 field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), psfSet.getNsample()) 

324 field.finalize() 

325 

326 fields.append(field) 

327 

328 sets = [] 

329 sets.append(psfSet) 

330 

331 psfex.makeit(fields, sets) 

332 psfs = field.getPsfs() 

333 

334 # Flag which objects were actually used in psfex by 

335 good_indices = [] 

336 for i in range(sets[0].getNsample()): 

337 index = sets[0].getSample(i).getObjindex() 

338 if index > -1: 

339 good_indices.append(index) 

340 

341 if flagKey is not None: 

342 for i, psfCandidate in enumerate(psfCandidateList): 

343 source = psfCandidate.getSource() 

344 if i in good_indices: 

345 source.set(flagKey, True) 

346 

347 xpos = np.array(xpos) 

348 ypos = np.array(ypos) 

349 numGoodStars = len(good_indices) 

350 avgX, avgY = np.mean(xpos), np.mean(ypos) 

351 

352 psf = psfex.PsfexPsf(psfs[0], geom.Point2D(avgX, avgY)) 

353 

354 # If there are too few stars, the PSFEx psf model will reduce the order 

355 # to 0, which the Science Pipelines code cannot handle (see 

356 # https://github.com/lsst/meas_extensions_psfex/blob/f0d5218b5446faf5e39edc30e31d2e6f673ef294/src/PsfexPsf.cc#L118 

357 # ). The easiest way to test for this condition is trying to compute 

358 # the PSF kernel and checking for an InvalidParameterError. 

359 try: 

360 _ = psf.getKernel(psf.getAveragePosition()) 

361 except pexExcept.InvalidParameterError: 

362 raise RuntimeError("Failed to determine psfex psf: too few good stars.") 

363 

364 # 

365 # Display code for debugging 

366 # 

367 if display: 

368 assert psfCellSet is not None 

369 

370 if displayExposure: 

371 maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True, 

372 symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, 

373 size=8, display=disp) 

374 if displayResiduals: 

375 disp4 = afwDisplay.Display(frame=4) 

376 maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, 

377 normalize=normalizeResiduals, 

378 showBadCandidates=showBadCandidates) 

379 if displayPsfComponents: 

380 disp6 = afwDisplay.Display(frame=6) 

381 maUtils.showPsf(psf, display=disp6) 

382 if displayPsfMosaic: 

383 disp7 = afwDisplay.Display(frame=7) 

384 maUtils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) 

385 disp.scale('linear', 0, 1) 

386 # 

387 # Generate some QA information 

388 # 

389 # Count PSF stars 

390 # 

391 if metadata is not None: 

392 metadata["spatialFitChi2"] = np.nan 

393 metadata["numAvailStars"] = nCand 

394 metadata["numGoodStars"] = numGoodStars 

395 metadata["avgX"] = avgX 

396 metadata["avgY"] = avgY 

397 

398 return psf, psfCellSet 

399 

400 

401measAlg.psfDeterminerRegistry.register("psfex", PsfexPsfDeterminerTask)