lsst.meas.extensions.psfex  16.0-10-gc1446dd+17
psfexPsfDeterminer.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import os
23 import numpy as np
24 
25 import lsst.daf.base as dafBase
26 import lsst.pex.config as pexConfig
27 import lsst.afw.geom as afwGeom
28 import lsst.afw.geom.ellipses as afwEll
29 import lsst.afw.display.ds9 as ds9
30 import lsst.afw.image as afwImage
31 import lsst.afw.math as afwMath
32 import lsst.meas.algorithms as measAlg
33 import lsst.meas.algorithms.utils as maUtils
34 import lsst.meas.extensions.psfex as psfex
35 
36 
37 class PsfexPsfDeterminerConfig(measAlg.BasePsfDeterminerConfig):
38  __nEigenComponents = pexConfig.Field(
39  doc="number of eigen components for PSF kernel creation",
40  dtype=int,
41  default=4,
42  )
43  spatialOrder = pexConfig.Field(
44  doc="specify spatial order for PSF kernel creation",
45  dtype=int,
46  default=2,
47  check=lambda x: x >= 0,
48  )
49  sizeCellX = pexConfig.Field(
50  doc="size of cell used to determine PSF (pixels, column direction)",
51  dtype=int,
52  default=256,
53  # minValue = 10,
54  check=lambda x: x >= 10,
55  )
56  sizeCellY = pexConfig.Field(
57  doc="size of cell used to determine PSF (pixels, row direction)",
58  dtype=int,
59  default=sizeCellX.default,
60  # minValue = 10,
61  check=lambda x: x >= 10,
62  )
63  __nStarPerCell = pexConfig.Field(
64  doc="number of stars per psf cell for PSF kernel creation",
65  dtype=int,
66  default=3,
67  )
68  samplingSize = pexConfig.Field(
69  doc="Resolution of the internal PSF model relative to the pixel size; "
70  "e.g. 0.5 is equal to 2x oversampling",
71  dtype=float,
72  default=1,
73  )
74  badMaskBits = pexConfig.ListField(
75  doc="List of mask bits which cause a source to be rejected as bad "
76  "N.b. INTRP is used specially in PsfCandidateSet; it means \"Contaminated by neighbour\"",
77  dtype=str,
78  default=["INTRP", "SAT"],
79  )
80  psfexBasis = pexConfig.ChoiceField(
81  doc="BASIS value given to psfex. PIXEL_AUTO will use the requested samplingSize only if "
82  "the FWHM < 3 pixels. Otherwise, it will use samplingSize=1. PIXEL will always use the "
83  "requested samplingSize",
84  dtype=str,
85  allowed={
86  "PIXEL": "Always use requested samplingSize",
87  "PIXEL_AUTO": "Only use requested samplingSize when FWHM < 3",
88  },
89  default='PIXEL',
90  optional=False,
91  )
92  __borderWidth = pexConfig.Field(
93  doc="Number of pixels to ignore around the edge of PSF candidate postage stamps",
94  dtype=int,
95  default=0,
96  )
97  __nStarPerCellSpatialFit = pexConfig.Field(
98  doc="number of stars per psf Cell for spatial fitting",
99  dtype=int,
100  default=5,
101  )
102  __constantWeight = pexConfig.Field(
103  doc="Should each PSF candidate be given the same weight, independent of magnitude?",
104  dtype=bool,
105  default=True,
106  )
107  __nIterForPsf = pexConfig.Field(
108  doc="number of iterations of PSF candidate star list",
109  dtype=int,
110  default=3,
111  )
112  tolerance = pexConfig.Field(
113  doc="tolerance of spatial fitting",
114  dtype=float,
115  default=1e-2,
116  )
117  lam = pexConfig.Field(
118  doc="floor for variance is lam*data",
119  dtype=float,
120  default=0.05,
121  )
122  reducedChi2ForPsfCandidates = pexConfig.Field(
123  doc="for psf candidate evaluation",
124  dtype=float,
125  default=2.0,
126  )
127  spatialReject = pexConfig.Field(
128  doc="Rejection threshold (stdev) for candidates based on spatial fit",
129  dtype=float,
130  default=3.0,
131  )
132  recentroid = pexConfig.Field(
133  doc="Should PSFEX be permitted to recentroid PSF candidates?",
134  dtype=bool,
135  default=False,
136  )
137 
138  def setDefaults(self):
139  self.kernelSize = 41
140 
141 
142 class PsfexPsfDeterminerTask(measAlg.BasePsfDeterminerTask):
143  ConfigClass = PsfexPsfDeterminerConfig
144 
145  def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
146  """Determine a PSFEX PSF model for an exposure given a list of PSF candidates
147 
148  @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure)
149  @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
150  typically obtained by detecting sources and then running them through a star selector
151  @param[in,out] metadata a home for interesting tidbits of information
152  @param[in] flagKey: schema key used to mark sources actually used in PSF determination
153 
154  @return psf: a meas.extensions.psfex.PsfexPsf
155  """
156  import lsstDebug
157  display = lsstDebug.Info(__name__).display
158  displayExposure = display and \
159  lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
160  displayPsfComponents = display and \
161  lsstDebug.Info(__name__).displayPsfComponents # show the basis functions
162  showBadCandidates = display and \
163  lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks)
164  displayResiduals = display and \
165  lsstDebug.Info(__name__).displayResiduals # show residuals
166  displayPsfMosaic = display and \
167  lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
168  normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
169  # Normalise residuals by object amplitude
170 
171  mi = exposure.getMaskedImage()
172 
173  nCand = len(psfCandidateList)
174  if nCand == 0:
175  raise RuntimeError("No PSF candidates supplied.")
176  #
177  # How big should our PSF models be?
178  #
179  if display: # only needed for debug plots
180  # construct and populate a spatial cell set
181  bbox = mi.getBBox(afwImage.PARENT)
182  psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
183  else:
184  psfCellSet = None
185 
186  sizes = np.empty(nCand)
187  for i, psfCandidate in enumerate(psfCandidateList):
188  try:
189  if psfCellSet:
190  psfCellSet.insertCandidate(psfCandidate)
191  except Exception as e:
192  self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
193  continue
194 
195  source = psfCandidate.getSource()
196  quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
197  rmsSize = quad.getTraceRadius()
198  sizes[i] = rmsSize
199 
200  if self.config.kernelSize >= 15:
201  self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value")
202  actualKernelSize = int(self.config.kernelSize)
203  else:
204  actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1
205  if actualKernelSize < self.config.kernelSizeMin:
206  actualKernelSize = self.config.kernelSizeMin
207  if actualKernelSize > self.config.kernelSizeMax:
208  actualKernelSize = self.config.kernelSizeMax
209  if display:
210  rms = np.median(sizes)
211  print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms))
212 
213  # If we manually set the resolution then we need the size in pixel units
214  pixKernelSize = actualKernelSize
215  if self.config.samplingSize > 0:
216  pixKernelSize = int(actualKernelSize*self.config.samplingSize)
217  if pixKernelSize % 2 == 0:
218  pixKernelSize += 1
219  self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize)
220  psfCandidateList[0].setHeight(pixKernelSize)
221  psfCandidateList[0].setWidth(pixKernelSize)
222 
223  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
224  #
225  # Insert the good candidates into the set
226  #
227  defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex")
228  args_md = dafBase.PropertySet()
229  args_md.set("BASIS_TYPE", str(self.config.psfexBasis))
230  args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
231  args_md.set("PSF_SIZE", str(actualKernelSize))
232  args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
233  prefs = psfex.Prefs(defaultsFile, args_md)
234  prefs.setCommandLine([])
235  prefs.addCatalog("psfexPsfDeterminer")
236 
237  prefs.use()
238  principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN)
239  if False else psfex.Context.KEEPHIDDEN)
240  context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(),
241  prefs.getGroupDeg(), principalComponentExclusionFlag)
242  set = psfex.Set(context)
243  set.setVigSize(pixKernelSize, pixKernelSize)
244  set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes))
245  set.setRecentroid(self.config.recentroid)
246 
247  catindex, ext = 0, 0
248  backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
249  ccd = exposure.getDetector()
250  if ccd:
251  gain = np.mean(np.array([a.getGain() for a in ccd]))
252  else:
253  gain = 1.0
254  self.log.warn("Setting gain to %g" % (gain,))
255 
256  contextvalp = []
257  for i, key in enumerate(context.getName()):
258  if context.getPcflag(i):
259  contextvalp.append(pcval[pc])
260  pc += 1
261  elif key[0] == ':':
262  try:
263  contextvalp.append(exposure.getMetadata().getScalar(key[1:]))
264  except KeyError:
265  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
266  (key[1:], prefs.getContextName()))
267  else:
268  try:
269  contextvalp.append(np.array([psfCandidateList[_].getSource().get(key)
270  for _ in range(nCand)]))
271  except KeyError:
272  raise RuntimeError("*Error*: %s parameter not found" % (key,))
273  set.setContextname(i, key)
274 
275  if display:
276  frame = 0
277  if displayExposure:
278  ds9.mtv(exposure, frame=frame, title="psf determination")
279 
280  badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
281  fluxName = prefs.getPhotfluxRkey()
282  fluxFlagName = "base_" + fluxName + "_flag"
283 
284  xpos, ypos = [], []
285  for i, psfCandidate in enumerate(psfCandidateList):
286  source = psfCandidate.getSource()
287  xc, yc = source.getX(), source.getY()
288  try:
289  int(xc), int(yc)
290  except ValueError:
291  continue
292 
293  try:
294  pstamp = psfCandidate.getMaskedImage().clone()
295  except Exception as e:
296  continue
297 
298  if fluxFlagName in source.schema and source.get(fluxFlagName):
299  continue
300 
301  flux = source.get(fluxName)
302  if flux < 0 or np.isnan(flux):
303  continue
304 
305  # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate).
306  # Having created the sample, we must proceed to configure it, and then fini (finalize),
307  # or it will be malformed.
308  try:
309  sample = set.newSample()
310  sample.setCatindex(catindex)
311  sample.setExtindex(ext)
312  sample.setObjindex(i)
313 
314  imArray = pstamp.getImage().getArray()
315  imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
316  -2*psfex.BIG
317  sample.setVig(imArray)
318 
319  sample.setNorm(flux)
320  sample.setBacknoise2(backnoise2)
321  sample.setGain(gain)
322  sample.setX(xc)
323  sample.setY(yc)
324  sample.setFluxrad(sizes[i])
325 
326  for j in range(set.getNcontext()):
327  sample.setContext(j, float(contextvalp[j][i]))
328  except Exception as e:
329  self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e)
330  continue
331  else:
332  set.finiSample(sample)
333 
334  xpos.append(xc) # for QA
335  ypos.append(yc)
336 
337  if displayExposure:
338  with ds9.Buffering():
339  ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame)
340 
341  if set.getNsample() == 0:
342  raise RuntimeError("No good PSF candidates to pass to PSFEx")
343 
344  #---- Update min and max and then the scaling
345  for i in range(set.getNcontext()):
346  cmin = contextvalp[i].min()
347  cmax = contextvalp[i].max()
348  set.setContextScale(i, cmax - cmin)
349  set.setContextOffset(i, (cmin + cmax)/2.0)
350 
351  # Don't waste memory!
352  set.trimMemory()
353 
354  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
355  #
356  # Do a PSFEX decomposition of those PSF candidates
357  #
358  fields = []
359  field = psfex.Field("Unknown")
360  field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample())
361  field.finalize()
362 
363  fields.append(field)
364 
365  sets = []
366  sets.append(set)
367 
368  psfex.makeit(fields, sets)
369  psfs = field.getPsfs()
370 
371  # Flag which objects were actually used in psfex by
372  good_indices = []
373  for i in range(sets[0].getNsample()):
374  index = sets[0].getSample(i).getObjindex()
375  if index > -1:
376  good_indices.append(index)
377 
378  if flagKey is not None:
379  for i, psfCandidate in enumerate(psfCandidateList):
380  source = psfCandidate.getSource()
381  if i in good_indices:
382  source.set(flagKey, True)
383 
384  xpos = np.array(xpos)
385  ypos = np.array(ypos)
386  numGoodStars = len(good_indices)
387  avgX, avgY = np.mean(xpos), np.mean(ypos)
388 
389  psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY))
390 
391  if False and (displayResiduals or displayPsfMosaic):
392  ext = 0
393  frame = 1
394  diagnostics = True
395  catDir = "."
396  title = "psfexPsfDeterminer"
397  psfex.psfex.showPsf(psfs, set, ext,
398  [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())],
399  nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir,
400  title=title)
401  #
402  # Display code for debugging
403  #
404  if display:
405  assert psfCellSet is not None
406 
407  if displayExposure:
408  maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True,
409  symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame)
410  if displayResiduals:
411  maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4,
412  normalize=normalizeResiduals,
413  showBadCandidates=showBadCandidates)
414  if displayPsfComponents:
415  maUtils.showPsf(psf, frame=6)
416  if displayPsfMosaic:
417  maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
418  ds9.scale('linear', 0, 1, frame=7)
419  #
420  # Generate some QA information
421  #
422  # Count PSF stars
423  #
424  if metadata is not None:
425  metadata.set("spatialFitChi2", np.nan)
426  metadata.set("numAvailStars", nCand)
427  metadata.set("numGoodStars", numGoodStars)
428  metadata.set("avgX", avgX)
429  metadata.set("avgY", avgY)
430 
431  psfCellSet = None
432  return psf, psfCellSet
433 
434 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
435 
436 measAlg.psfDeterminerRegistry.register("psfex", PsfexPsfDeterminerTask)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None)
Represent a PSF as a linear combination of PSFEX (== Karhunen-Loeve) basis functions.
Definition: PsfexPsf.h:39