lsst.meas.extensions.psfex  20.0.0-2-g8177d33+f2ab36af31
psfexPsfDeterminer.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy as np
24 
25 import lsst.daf.base as dafBase
26 import lsst.pex.config as pexConfig
27 import lsst.geom as geom
28 import lsst.afw.geom.ellipses as afwEll
29 import lsst.afw.display as afwDisplay
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.meas.algorithms as measAlg
33 import lsst.meas.algorithms.utils as maUtils
34 import lsst.meas.extensions.psfex as psfex
35 
36 
37 class PsfexPsfDeterminerConfig(measAlg.BasePsfDeterminerConfig):
38  spatialOrder = pexConfig.Field(
39  doc="specify spatial order for PSF kernel creation",
40  dtype=int,
41  default=2,
42  check=lambda x: x >= 0,
43  )
44  sizeCellX = pexConfig.Field(
45  doc="size of cell used to determine PSF (pixels, column direction)",
46  dtype=int,
47  default=256,
48  # minValue = 10,
49  check=lambda x: x >= 10,
50  )
51  sizeCellY = pexConfig.Field(
52  doc="size of cell used to determine PSF (pixels, row direction)",
53  dtype=int,
54  default=sizeCellX.default,
55  # minValue = 10,
56  check=lambda x: x >= 10,
57  )
58  samplingSize = pexConfig.Field(
59  doc="Resolution of the internal PSF model relative to the pixel size; "
60  "e.g. 0.5 is equal to 2x oversampling",
61  dtype=float,
62  default=1,
63  )
64  badMaskBits = pexConfig.ListField(
65  doc="List of mask bits which cause a source to be rejected as bad "
66  "N.b. INTRP is used specially in PsfCandidateSet; it means \"Contaminated by neighbour\"",
67  dtype=str,
68  default=["INTRP", "SAT"],
69  )
70  psfexBasis = pexConfig.ChoiceField(
71  doc="BASIS value given to psfex. PIXEL_AUTO will use the requested samplingSize only if "
72  "the FWHM < 3 pixels. Otherwise, it will use samplingSize=1. PIXEL will always use the "
73  "requested samplingSize",
74  dtype=str,
75  allowed={
76  "PIXEL": "Always use requested samplingSize",
77  "PIXEL_AUTO": "Only use requested samplingSize when FWHM < 3",
78  },
79  default='PIXEL',
80  optional=False,
81  )
82  tolerance = pexConfig.Field(
83  doc="tolerance of spatial fitting",
84  dtype=float,
85  default=1e-2,
86  )
87  lam = pexConfig.Field(
88  doc="floor for variance is lam*data",
89  dtype=float,
90  default=0.05,
91  )
92  reducedChi2ForPsfCandidates = pexConfig.Field(
93  doc="for psf candidate evaluation",
94  dtype=float,
95  default=2.0,
96  )
97  spatialReject = pexConfig.Field(
98  doc="Rejection threshold (stdev) for candidates based on spatial fit",
99  dtype=float,
100  default=3.0,
101  )
102  recentroid = pexConfig.Field(
103  doc="Should PSFEX be permitted to recentroid PSF candidates?",
104  dtype=bool,
105  default=False,
106  )
107 
108  def setDefaults(self):
109  self.kernelSize = 41
110 
111 
112 class PsfexPsfDeterminerTask(measAlg.BasePsfDeterminerTask):
113  ConfigClass = PsfexPsfDeterminerConfig
114 
115  def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
116  """Determine a PSFEX PSF model for an exposure given a list of PSF
117  candidates.
118 
119  Parameters
120  ----------
121  exposure: `lsst.afw.image.Exposure`
122  Exposure containing the PSF candidates.
123  psfCandidateList: iterable of `lsst.meas.algorithms.PsfCandidate`
124  Sequence of PSF candidates typically obtained by detecting sources
125  and then running them through a star selector.
126  metadata: metadata, optional
127  A home for interesting tidbits of information.
128  flagKey: `lsst.afw.table.Key`, optional
129  Schema key used to mark sources actually used in PSF determination.
130 
131  Returns
132  -------
133  psf: `lsst.meas.extensions.psfex.PsfexPsf`
134  The determined PSF.
135  """
136 
137  import lsstDebug
138  display = lsstDebug.Info(__name__).display
139  displayExposure = display and \
140  lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
141  displayPsfComponents = display and \
142  lsstDebug.Info(__name__).displayPsfComponents # show the basis functions
143  showBadCandidates = display and \
144  lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks)
145  displayResiduals = display and \
146  lsstDebug.Info(__name__).displayResiduals # show residuals
147  displayPsfMosaic = display and \
148  lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
149  normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
150  afwDisplay.setDefaultMaskTransparency(75)
151  # Normalise residuals by object amplitude
152 
153  mi = exposure.getMaskedImage()
154 
155  nCand = len(psfCandidateList)
156  if nCand == 0:
157  raise RuntimeError("No PSF candidates supplied.")
158  #
159  # How big should our PSF models be?
160  #
161  if display: # only needed for debug plots
162  # construct and populate a spatial cell set
163  bbox = mi.getBBox(afwImage.PARENT)
164  psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
165  else:
166  psfCellSet = None
167 
168  sizes = np.empty(nCand)
169  for i, psfCandidate in enumerate(psfCandidateList):
170  try:
171  if psfCellSet:
172  psfCellSet.insertCandidate(psfCandidate)
173  except Exception as e:
174  self.log.error("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
175  continue
176 
177  source = psfCandidate.getSource()
178  quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
179  rmsSize = quad.getTraceRadius()
180  sizes[i] = rmsSize
181 
182  if self.config.kernelSize >= 15:
183  self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value")
184  actualKernelSize = int(self.config.kernelSize)
185  else:
186  actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1
187  if actualKernelSize < self.config.kernelSizeMin:
188  actualKernelSize = self.config.kernelSizeMin
189  if actualKernelSize > self.config.kernelSizeMax:
190  actualKernelSize = self.config.kernelSizeMax
191  if display:
192  rms = np.median(sizes)
193  msg = "Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms)
194  self.log.debug(msg)
195 
196  # If we manually set the resolution then we need the size in pixel
197  # units
198  pixKernelSize = actualKernelSize
199  if self.config.samplingSize > 0:
200  pixKernelSize = int(actualKernelSize*self.config.samplingSize)
201  if pixKernelSize % 2 == 0:
202  pixKernelSize += 1
203  self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize)
204  psfCandidateList[0].setHeight(pixKernelSize)
205  psfCandidateList[0].setWidth(pixKernelSize)
206 
207  # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
208  #
209  # Insert the good candidates into the set
210  #
211  defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex")
212  args_md = dafBase.PropertySet()
213  args_md.set("BASIS_TYPE", str(self.config.psfexBasis))
214  args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
215  args_md.set("PSF_SIZE", str(actualKernelSize))
216  args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
217  prefs = psfex.Prefs(defaultsFile, args_md)
218  prefs.setCommandLine([])
219  prefs.addCatalog("psfexPsfDeterminer")
220 
221  prefs.use()
222  principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN)
223  if False else psfex.Context.KEEPHIDDEN)
224  context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(),
225  prefs.getGroupDeg(), principalComponentExclusionFlag)
226  psfSet = psfex.Set(context)
227  psfSet.setVigSize(pixKernelSize, pixKernelSize)
228  psfSet.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes))
229  psfSet.setRecentroid(self.config.recentroid)
230 
231  catindex, ext = 0, 0
232  backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
233  ccd = exposure.getDetector()
234  if ccd:
235  gain = np.mean(np.array([a.getGain() for a in ccd]))
236  else:
237  gain = 1.0
238  self.log.warn("Setting gain to %g" % (gain,))
239 
240  contextvalp = []
241  for i, key in enumerate(context.getName()):
242  if key[0] == ':':
243  try:
244  contextvalp.append(exposure.getMetadata().getScalar(key[1:]))
245  except KeyError as e:
246  raise RuntimeError("%s parameter not found in the header of %s" %
247  (key[1:], prefs.getContextName())) from e
248  else:
249  try:
250  contextvalp.append(np.array([psfCandidateList[_].getSource().get(key)
251  for _ in range(nCand)]))
252  except KeyError as e:
253  raise RuntimeError("%s parameter not found" % (key,)) from e
254  psfSet.setContextname(i, key)
255 
256  if display:
257  frame = 0
258  if displayExposure:
259  disp = afwDisplay.Display(frame=frame)
260  disp.mtv(exposure, title="psf determination")
261 
262  badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
263  fluxName = prefs.getPhotfluxRkey()
264  fluxFlagName = "base_" + fluxName + "_flag"
265 
266  xpos, ypos = [], []
267  for i, psfCandidate in enumerate(psfCandidateList):
268  source = psfCandidate.getSource()
269 
270  # skip sources with bad centroids
271  xc, yc = source.getX(), source.getY()
272  if not np.isfinite(xc) or not np.isfinite(yc):
273  continue
274  # skip flagged sources
275  if fluxFlagName in source.schema and source.get(fluxFlagName):
276  continue
277  # skip nonfinite and negative sources
278  flux = source.get(fluxName)
279  if flux < 0 or not np.isfinite(flux):
280  continue
281 
282  pstamp = psfCandidate.getMaskedImage().clone()
283 
284  # From this point, we're configuring the "sample" (PSFEx's version
285  # of a PSF candidate).
286  # Having created the sample, we must proceed to configure it, and
287  # then fini (finalize), or it will be malformed.
288  try:
289  sample = psfSet.newSample()
290  sample.setCatindex(catindex)
291  sample.setExtindex(ext)
292  sample.setObjindex(i)
293 
294  imArray = pstamp.getImage().getArray()
295  imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
296  -2*psfex.BIG
297  sample.setVig(imArray)
298 
299  sample.setNorm(flux)
300  sample.setBacknoise2(backnoise2)
301  sample.setGain(gain)
302  sample.setX(xc)
303  sample.setY(yc)
304  sample.setFluxrad(sizes[i])
305 
306  for j in range(psfSet.getNcontext()):
307  sample.setContext(j, float(contextvalp[j][i]))
308  except Exception as e:
309  self.log.error("Exception when processing sample at (%f,%f): %s", xc, yc, e)
310  continue
311  else:
312  psfSet.finiSample(sample)
313 
314  xpos.append(xc) # for QA
315  ypos.append(yc)
316 
317  if displayExposure:
318  with disp.Buffering():
319  disp.dot("o", xc, yc, ctype=afwDisplay.CYAN, size=4)
320 
321  if psfSet.getNsample() == 0:
322  raise RuntimeError("No good PSF candidates to pass to PSFEx")
323 
324  # ---- Update min and max and then the scaling
325  for i in range(psfSet.getNcontext()):
326  cmin = contextvalp[i].min()
327  cmax = contextvalp[i].max()
328  psfSet.setContextScale(i, cmax - cmin)
329  psfSet.setContextOffset(i, (cmin + cmax)/2.0)
330 
331  # Don't waste memory!
332  psfSet.trimMemory()
333 
334  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
335  #
336  # Do a PSFEX decomposition of those PSF candidates
337  #
338  fields = []
339  field = psfex.Field("Unknown")
340  field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), psfSet.getNsample())
341  field.finalize()
342 
343  fields.append(field)
344 
345  sets = []
346  sets.append(psfSet)
347 
348  psfex.makeit(fields, sets)
349  psfs = field.getPsfs()
350 
351  # Flag which objects were actually used in psfex by
352  good_indices = []
353  for i in range(sets[0].getNsample()):
354  index = sets[0].getSample(i).getObjindex()
355  if index > -1:
356  good_indices.append(index)
357 
358  if flagKey is not None:
359  for i, psfCandidate in enumerate(psfCandidateList):
360  source = psfCandidate.getSource()
361  if i in good_indices:
362  source.set(flagKey, True)
363 
364  xpos = np.array(xpos)
365  ypos = np.array(ypos)
366  numGoodStars = len(good_indices)
367  avgX, avgY = np.mean(xpos), np.mean(ypos)
368 
369  psf = psfex.PsfexPsf(psfs[0], geom.Point2D(avgX, avgY))
370 
371  #
372  # Display code for debugging
373  #
374  if display:
375  assert psfCellSet is not None
376 
377  if displayExposure:
378  maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True,
379  symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
380  size=8, display=disp)
381  if displayResiduals:
382  disp4 = afwDisplay.Display(frame=4)
383  maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
384  normalize=normalizeResiduals,
385  showBadCandidates=showBadCandidates)
386  if displayPsfComponents:
387  disp6 = afwDisplay.Display(frame=6)
388  maUtils.showPsf(psf, display=disp6)
389  if displayPsfMosaic:
390  disp7 = afwDisplay.Display(frame=7)
391  maUtils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
392  disp.scale('linear', 0, 1)
393  #
394  # Generate some QA information
395  #
396  # Count PSF stars
397  #
398  if metadata is not None:
399  metadata.set("spatialFitChi2", np.nan)
400  metadata.set("numAvailStars", nCand)
401  metadata.set("numGoodStars", numGoodStars)
402  metadata.set("avgX", avgX)
403  metadata.set("avgY", avgY)
404 
405  return psf, psfCellSet
406 
407 
408 measAlg.psfDeterminerRegistry.register("psfex", PsfexPsfDeterminerTask)
lsst::meas::extensions::psfex.psfexPsfDeterminer.PsfexPsfDeterminerConfig.setDefaults
def setDefaults(self)
Definition: psfexPsfDeterminer.py:108
lsst::afw::image
lsst::afw::display
lsst::meas::extensions::psfex.psfexPsfDeterminer.PsfexPsfDeterminerTask
Definition: psfexPsfDeterminer.py:112
lsst::meas::extensions::psfex
Definition: PsfexPsf.h:31
lsstDebug::Info
lsst::afw::geom::ellipses
lsst::pex::config
lsst::meas::extensions::psfex.psfexPsfDeterminer.PsfexPsfDeterminerTask.determinePsf
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None)
Definition: psfexPsfDeterminer.py:115
lsst::meas::extensions::psfex.psfexPsfDeterminer.PsfexPsfDeterminerConfig.kernelSize
kernelSize
Definition: psfexPsfDeterminer.py:109
lsst::geom
lsst::daf::base
lsst::afw::math
Point< double, 2 >
lsst::meas::algorithms::utils
lsst::meas::extensions::psfex.psfexPsfDeterminer.PsfexPsfDeterminerConfig
Definition: psfexPsfDeterminer.py:37
lsst::meas::algorithms
lsst::meas::extensions::psfex::PsfexPsf
Represent a PSF as a linear combination of PSFEX (== Karhunen-Loeve) basis functions.
Definition: PsfexPsf.h:40