lsst.meas.extensions.psfex  14.0-3-g82223af+2
psfexPsfDeterminer.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from builtins import range
3 #
4 # LSST Data Management System
5 # Copyright 2008-2015 AURA/LSST.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <https://www.lsstcorp.org/LegalNotices/>.
23 #
24 import os
25 import numpy as np
26 
27 import lsst.daf.base as dafBase
28 import lsst.pex.config as pexConfig
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.geom.ellipses as afwEll
31 import lsst.afw.display.ds9 as ds9
32 import lsst.afw.image as afwImage
33 import lsst.afw.math as afwMath
34 import lsst.meas.algorithms as measAlg
35 import lsst.meas.algorithms.utils as maUtils
36 import lsst.meas.extensions.psfex as psfex
37 
38 
39 class PsfexPsfDeterminerConfig(measAlg.BasePsfDeterminerConfig):
40  __nEigenComponents = pexConfig.Field(
41  doc="number of eigen components for PSF kernel creation",
42  dtype=int,
43  default=4,
44  )
45  spatialOrder = pexConfig.Field(
46  doc="specify spatial order for PSF kernel creation",
47  dtype=int,
48  default=2,
49  check=lambda x: x >= 0,
50  )
51  sizeCellX = pexConfig.Field(
52  doc="size of cell used to determine PSF (pixels, column direction)",
53  dtype=int,
54  default=256,
55  # minValue = 10,
56  check=lambda x: x >= 10,
57  )
58  sizeCellY = pexConfig.Field(
59  doc="size of cell used to determine PSF (pixels, row direction)",
60  dtype=int,
61  default=sizeCellX.default,
62  # minValue = 10,
63  check=lambda x: x >= 10,
64  )
65  __nStarPerCell = pexConfig.Field(
66  doc="number of stars per psf cell for PSF kernel creation",
67  dtype=int,
68  default=3,
69  )
70  samplingSize = pexConfig.Field(
71  doc="Resolution of the internal PSF model relative to the pixel size; "
72  "e.g. 0.5 is equal to 2x oversampling",
73  dtype=float,
74  default=1,
75  )
76  badMaskBits = pexConfig.ListField(
77  doc="List of mask bits which cause a source to be rejected as bad "
78  "N.b. INTRP is used specially in PsfCandidateSet; it means \"Contaminated by neighbour\"",
79  dtype=str,
80  default=["INTRP", "SAT"],
81  )
82  psfexBasis = pexConfig.ChoiceField(
83  doc="BASIS value given to psfex. PIXEL_AUTO will use the requested samplingSize only if "
84  "the FWHM < 3 pixels. Otherwise, it will use samplingSize=1. PIXEL will always use the "
85  "requested samplingSize",
86  dtype=str,
87  allowed={
88  "PIXEL": "Always use requested samplingSize",
89  "PIXEL_AUTO": "Only use requested samplingSize when FWHM < 3",
90  },
91  default='PIXEL',
92  optional=False,
93  )
94  __borderWidth = pexConfig.Field(
95  doc="Number of pixels to ignore around the edge of PSF candidate postage stamps",
96  dtype=int,
97  default=0,
98  )
99  __nStarPerCellSpatialFit = pexConfig.Field(
100  doc="number of stars per psf Cell for spatial fitting",
101  dtype=int,
102  default=5,
103  )
104  __constantWeight = pexConfig.Field(
105  doc="Should each PSF candidate be given the same weight, independent of magnitude?",
106  dtype=bool,
107  default=True,
108  )
109  __nIterForPsf = pexConfig.Field(
110  doc="number of iterations of PSF candidate star list",
111  dtype=int,
112  default=3,
113  )
114  tolerance = pexConfig.Field(
115  doc="tolerance of spatial fitting",
116  dtype=float,
117  default=1e-2,
118  )
119  lam = pexConfig.Field(
120  doc="floor for variance is lam*data",
121  dtype=float,
122  default=0.05,
123  )
124  reducedChi2ForPsfCandidates = pexConfig.Field(
125  doc="for psf candidate evaluation",
126  dtype=float,
127  default=2.0,
128  )
129  spatialReject = pexConfig.Field(
130  doc="Rejection threshold (stdev) for candidates based on spatial fit",
131  dtype=float,
132  default=3.0,
133  )
134  recentroid = pexConfig.Field(
135  doc="Should PSFEX be permitted to recentroid PSF candidates?",
136  dtype=bool,
137  default=False,
138  )
139 
140  def setDefaults(self):
141  self.kernelSize = 41
142 
143 
144 class PsfexPsfDeterminerTask(measAlg.BasePsfDeterminerTask):
145  ConfigClass = PsfexPsfDeterminerConfig
146 
147  def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
148  """Determine a PSFEX PSF model for an exposure given a list of PSF candidates
149 
150  @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure)
151  @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
152  typically obtained by detecting sources and then running them through a star selector
153  @param[in,out] metadata a home for interesting tidbits of information
154  @param[in] flagKey: schema key used to mark sources actually used in PSF determination
155 
156  @return psf: a meas.extensions.psfex.PsfexPsf
157  """
158  import lsstDebug
159  display = lsstDebug.Info(__name__).display
160  displayExposure = display and \
161  lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
162  displayPsfComponents = display and \
163  lsstDebug.Info(__name__).displayPsfComponents # show the basis functions
164  showBadCandidates = display and \
165  lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks)
166  displayResiduals = display and \
167  lsstDebug.Info(__name__).displayResiduals # show residuals
168  displayPsfMosaic = display and \
169  lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
170  normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
171  # Normalise residuals by object amplitude
172 
173  mi = exposure.getMaskedImage()
174 
175  nCand = len(psfCandidateList)
176  if nCand == 0:
177  raise RuntimeError("No PSF candidates supplied.")
178  #
179  # How big should our PSF models be?
180  #
181  if display: # only needed for debug plots
182  # construct and populate a spatial cell set
183  bbox = mi.getBBox(afwImage.PARENT)
184  psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
185  else:
186  psfCellSet = None
187 
188  sizes = np.empty(nCand)
189  for i, psfCandidate in enumerate(psfCandidateList):
190  try:
191  if psfCellSet:
192  psfCellSet.insertCandidate(psfCandidate)
193  except Exception as e:
194  self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
195  continue
196 
197  source = psfCandidate.getSource()
198  quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
199  rmsSize = quad.getTraceRadius()
200  sizes[i] = rmsSize
201 
202  if self.config.kernelSize >= 15:
203  self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value")
204  actualKernelSize = int(self.config.kernelSize)
205  else:
206  actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1
207  if actualKernelSize < self.config.kernelSizeMin:
208  actualKernelSize = self.config.kernelSizeMin
209  if actualKernelSize > self.config.kernelSizeMax:
210  actualKernelSize = self.config.kernelSizeMax
211  if display:
212  rms = np.median(sizes)
213  print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms))
214 
215  # If we manually set the resolution then we need the size in pixel units
216  pixKernelSize = actualKernelSize
217  if self.config.samplingSize > 0:
218  pixKernelSize = int(actualKernelSize*self.config.samplingSize)
219  if pixKernelSize % 2 == 0:
220  pixKernelSize += 1
221  self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize)
222  psfCandidateList[0].setHeight(pixKernelSize)
223  psfCandidateList[0].setWidth(pixKernelSize)
224 
225  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
226  #
227  # Insert the good candidates into the set
228  #
229  defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex")
230  args_md = dafBase.PropertySet()
231  args_md.set("BASIS_TYPE", str(self.config.psfexBasis))
232  args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
233  args_md.set("PSF_SIZE", str(actualKernelSize))
234  args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
235  prefs = psfex.Prefs(defaultsFile, args_md)
236  prefs.setCommandLine([])
237  prefs.addCatalog("psfexPsfDeterminer")
238 
239  prefs.use()
240  principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN)
241  if False else psfex.Context.KEEPHIDDEN)
242  context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(),
243  prefs.getGroupDeg(), principalComponentExclusionFlag)
244  set = psfex.Set(context)
245  set.setVigSize(pixKernelSize, pixKernelSize)
246  set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes))
247  set.setRecentroid(self.config.recentroid)
248 
249  catindex, ext = 0, 0
250  backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
251  ccd = exposure.getDetector()
252  if ccd:
253  gain = np.mean(np.array([a.getGain() for a in ccd]))
254  else:
255  gain = 1.0
256  self.log.warn("Setting gain to %g" % (gain,))
257 
258  contextvalp = []
259  for i, key in enumerate(context.getName()):
260  if context.getPcflag(i):
261  contextvalp.append(pcval[pc])
262  pc += 1
263  elif key[0] == ':':
264  try:
265  contextvalp.append(exposure.getMetadata().get(key[1:]))
266  except KeyError:
267  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
268  (key[1:], prefs.getContextName()))
269  else:
270  try:
271  contextvalp.append(np.array([psfCandidateList[_].getSource().get(key)
272  for _ in range(nCand)]))
273  except KeyError:
274  raise RuntimeError("*Error*: %s parameter not found" % (key,))
275  set.setContextname(i, key)
276 
277  if display:
278  frame = 0
279  if displayExposure:
280  ds9.mtv(exposure, frame=frame, title="psf determination")
281 
282  badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
283  fluxName = prefs.getPhotfluxRkey()
284  fluxFlagName = "base_" + fluxName + "_flag"
285 
286  xpos, ypos = [], []
287  for i, psfCandidate in enumerate(psfCandidateList):
288  source = psfCandidate.getSource()
289  xc, yc = source.getX(), source.getY()
290  try:
291  int(xc), int(yc)
292  except ValueError:
293  continue
294 
295  try:
296  pstamp = psfCandidate.getMaskedImage().clone()
297  except Exception as e:
298  continue
299 
300  if fluxFlagName in source.schema and source.get(fluxFlagName):
301  continue
302 
303  flux = source.get(fluxName)
304  if flux < 0 or np.isnan(flux):
305  continue
306 
307  # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate).
308  # Having created the sample, we must proceed to configure it, and then fini (finalize),
309  # or it will be malformed.
310  try:
311  sample = set.newSample()
312  sample.setCatindex(catindex)
313  sample.setExtindex(ext)
314  sample.setObjindex(i)
315 
316  imArray = pstamp.getImage().getArray()
317  imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
318  -2*psfex.BIG
319  sample.setVig(imArray)
320 
321  sample.setNorm(flux)
322  sample.setBacknoise2(backnoise2)
323  sample.setGain(gain)
324  sample.setX(xc)
325  sample.setY(yc)
326  sample.setFluxrad(sizes[i])
327 
328  for j in range(set.getNcontext()):
329  sample.setContext(j, float(contextvalp[j][i]))
330  except Exception as e:
331  self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e)
332  continue
333  else:
334  set.finiSample(sample)
335 
336  xpos.append(xc) # for QA
337  ypos.append(yc)
338 
339  if displayExposure:
340  with ds9.Buffering():
341  ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame)
342 
343  if set.getNsample() == 0:
344  raise RuntimeError("No good PSF candidates to pass to PSFEx")
345 
346  #---- Update min and max and then the scaling
347  for i in range(set.getNcontext()):
348  cmin = contextvalp[i].min()
349  cmax = contextvalp[i].max()
350  set.setContextScale(i, cmax - cmin)
351  set.setContextOffset(i, (cmin + cmax)/2.0)
352 
353  # Don't waste memory!
354  set.trimMemory()
355 
356  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
357  #
358  # Do a PSFEX decomposition of those PSF candidates
359  #
360  fields = []
361  field = psfex.Field("Unknown")
362  field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample())
363  field.finalize()
364 
365  fields.append(field)
366 
367  sets = []
368  sets.append(set)
369 
370  psfex.makeit(fields, sets)
371  psfs = field.getPsfs()
372 
373  # Flag which objects were actually used in psfex by
374  good_indices = []
375  for i in range(sets[0].getNsample()):
376  index = sets[0].getSample(i).getObjindex()
377  if index > -1:
378  good_indices.append(index)
379 
380  if flagKey is not None:
381  for i, psfCandidate in enumerate(psfCandidateList):
382  source = psfCandidate.getSource()
383  if i in good_indices:
384  source.set(flagKey, True)
385 
386  xpos = np.array(xpos)
387  ypos = np.array(ypos)
388  numGoodStars = len(good_indices)
389  avgX, avgY = np.mean(xpos), np.mean(ypos)
390 
391  psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY))
392 
393  if False and (displayResiduals or displayPsfMosaic):
394  ext = 0
395  frame = 1
396  diagnostics = True
397  catDir = "."
398  title = "psfexPsfDeterminer"
399  psfex.psfex.showPsf(psfs, set, ext,
400  [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())],
401  nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir,
402  title=title)
403  #
404  # Display code for debugging
405  #
406  if display:
407  assert psfCellSet is not None
408 
409  if displayExposure:
410  maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True,
411  symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame)
412  if displayResiduals:
413  maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4,
414  normalize=normalizeResiduals,
415  showBadCandidates=showBadCandidates)
416  if displayPsfComponents:
417  maUtils.showPsf(psf, frame=6)
418  if displayPsfMosaic:
419  maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
420  ds9.scale('linear', 0, 1, frame=7)
421  #
422  # Generate some QA information
423  #
424  # Count PSF stars
425  #
426  if metadata is not None:
427  metadata.set("spatialFitChi2", np.nan)
428  metadata.set("numAvailStars", nCand)
429  metadata.set("numGoodStars", numGoodStars)
430  metadata.set("avgX", avgX)
431  metadata.set("avgY", avgY)
432 
433  psfCellSet = None
434  return psf, psfCellSet
435 
436 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
437 
438 measAlg.psfDeterminerRegistry.register("psfex", PsfexPsfDeterminerTask)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None)
Represent a PSF as a linear combination of PSFEX (== Karhunen-Loeve) basis functions.
Definition: PsfexPsf.h:38