lsst.meas.extensions.psfex  14.0-1-g3019bb2+18
psfex.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 from __future__ import absolute_import, division, print_function
3 from builtins import zip
4 from builtins import input
5 from builtins import str
6 from builtins import range
7 from builtins import object
8 import os
9 import re
10 import sys
11 
12 import numpy as np
13 import pyfits
14 try:
15  import matplotlib.pyplot as plt
16 except ImportError:
17  plt = None
18 
19 import lsst.afw.coord as afwCoord
20 import lsst.afw.geom as afwGeom
21 import lsst.afw.image as afwImage
22 import lsst.afw.table as afwTable
23 import lsst.afw.display.ds9 as ds9
24 from lsst.daf.base import PropertySet
25 from . import psfexLib
26 
27 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
28 
29 
30 def splitFitsCard(line):
31  """Split a fits header, returning (key, value)"""
32  try:
33  k, v = re.search(r"(\S+)\s*=\s*'?((?:\S+|'))", line).groups()
34  except AttributeError:
35  raise
36 
37  try:
38  v = int(v)
39  except ValueError:
40  try:
41  v = float(v)
42  except ValueError:
43  pass
44 
45  return k, v
46 
47 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
48 
49 
50 def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False)):
51  """
52  PURPOSE Compute the FWHM range associated to a series of FWHM measurements.
53  INPUT Pointer to an array of FWHMs,
54  maximum allowed FWHM variation,
55  minimum allowed FWHM,
56  maximum allowed FWHM,
57 
58  OUTPUT FWHM mode, lower FWHM range, upper FWHM range
59  NOTES -.
60  AUTHOR E. Bertin (IAP, Leiden observatory & ESO)
61  VERSION 20/03/2008
62  """
63  nfwhm = len(fwhm)
64  fwhm.sort()
65 
66  # Find the mode
67  nw = nfwhm//4
68  if nw < 4:
69  nw = 1
70  dfmin = psfexLib.BIG
71  fmin = 0.0
72  for i in range(nfwhm - nw):
73  df = fwhm[i + nw] - fwhm[i]
74  if df < dfmin:
75  dfmin = df
76  fmin = (fwhm[i + nw] + fwhm[i])/2.0
77 
78  if nfwhm < 2:
79  fmin = fwhm[0]
80 
81  dfmin = (maxvar + 1.0)**0.3333333
82  minout = fmin/dfmin if dfmin > 0.0 else 0.0
83  if minout < minin:
84  minout = minin
85 
86  maxout = fmin*dfmin**2
87  if maxout > maxin:
88  maxout = maxin
89 
90  if plt and plot.get("fwhmHistogram"):
91  plt.clf()
92  plt.hist(fwhm, nfwhm//10 + 1, normed=1, facecolor='g', alpha=0.75)
93  plt.xlabel("FWHM")
94  plt.axvline(fmin, color='red')
95  [plt.axvline(_, color='blue') for _ in (minout, maxout)]
96 
97  input("Continue? ")
98 
99  return fmin, minout, maxout
100 
101 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
102 
103 
104 def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
105  plot=dict(showFlags=False, showRejection=False)):
106  # allocate a new set iff set is None
107  if not set:
108  set = psfexLib.Set(context)
109 
110  cmin, cmax = None, None
111  if set.getNcontext():
112  cmin = np.empty(set.getNcontext())
113  cmax = np.empty(set.getNcontext())
114  for i in range(set.getNcontext()):
115  if set.getNsample():
116  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
117  cmax[i] = cmin[i] + set.getContextScale(i)
118  else:
119  cmin[i] = psfexLib.BIG
120  cmax[i] = -psfexLib.BIG
121  #
122  # Read data
123  #
124  with pyfits.open(filename) as cat:
125  extCtr = -1
126  for tab in cat:
127  if tab.name == "LDAC_IMHEAD":
128  extCtr += 1
129 
130  if extCtr < ext:
131  continue
132  elif extCtr > ext:
133  break
134 
135  if tab.name == "PRIMARY":
136  pass
137  elif tab.name == "LDAC_IMHEAD":
138  hdr = tab.data[0][0] # the fits header from the original fits image
139  foundCards = 0 # how many of our desired cards have we found?
140 
141  for line in hdr:
142  try:
143  k, v = splitFitsCard(line)
144  except AttributeError:
145  continue
146 
147  if k == "SEXBKDEV":
148  backnoise2 = v**2
149  foundCards += 1
150  elif k == "SEXGAIN":
151  gain = v
152  foundCards += 1
153 
154  if foundCards == 2:
155  break
156  elif tab.name == "LDAC_OBJECTS":
157  xm = tab.data[prefs.getCenterKey(0)]
158  ym = tab.data[prefs.getCenterKey(1)]
159  fluxrad = tab.data["FLUX_RADIUS"]
160  flux = tab.data[prefs.getPhotfluxRkey()]
161  fluxerr = tab.data[prefs.getPhotfluxerrRkey()]
162  elong = tab.data["ELONGATION"]
163  flags = tab.data["FLAGS"]
164 
165  n = prefs.getPhotfluxNum() - 1
166  if n:
167  assert False, "Code to handle e.g. FLUX_APER(3) isn't yet converted"
168  if key.naxis == 1 and n < key.naxisn[0]:
169  flux += n
170  else:
171  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
172  (prefs.getPhotfluxRkey(), filename), file=sys.stderr)
173 
174  n = prefs.getPhotfluxerrNum() - 1
175  if n:
176  if key.naxis == 1 and n < key.naxisn[0]:
177  fluxerr += n
178  else:
179  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
180  (prefs.getPhotfluxerrRkey(), filename), file=sys.stderr)
181  #
182  # Now the VIGNET data
183  #
184  vignet = tab.data["VIGNET"]
185 
186  try:
187  vigw, vigh = vignet[0].shape
188  except ValueError:
189  raise RuntimeError("*Error*: VIGNET should be a 2D vector; saw %s" % str(vignet[0].shape))
190 
191  if set.empty():
192  set.setVigSize(vigw, vigh)
193 
194  # Try to load the set of context keys
195  pc = 0
196  contextvalp = []
197  for i, key in enumerate(context.getName()):
198  if context.getPcflag(i):
199  contextvalp.append(pcval[pc])
200  pc += 1
201  elif key[0] == ':':
202  try:
203  contextvalp.append(tab.header[key[1:]])
204  except KeyError:
205  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
206  (key[1:], filename))
207  else:
208  try:
209  contextvalp.append(tab.data[key])
210  except KeyError:
211  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
212  (key, filename))
213  set.setContextname(i, key)
214 
215  # Now examine each vector of the shipment
216  good = select_candidates(set, prefs, frmin, frmax,
217  flags, flux, fluxerr, fluxrad, elong, vignet,
218  plot=plot, title="%s[%d]" % (filename, ext + 1))
219  #
220  # Insert the good candidates into the set
221  #
222  if not vignet.dtype.isnative:
223  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
224  vignet = vignet.byteswap()
225 
226  for i in np.where(good)[0]:
227  sample = set.newSample()
228  sample.setCatindex(catindex)
229  sample.setExtindex(ext)
230 
231  sample.setVig(vignet[i])
232 
233  sample.setNorm(float(flux[i]))
234  sample.setBacknoise2(backnoise2)
235  sample.setGain(gain)
236  sample.setX(float(xm[i]))
237  sample.setY(float(ym[i]))
238  sample.setFluxrad(float(fluxrad[i]))
239 
240  for j in range(set.getNcontext()):
241  sample.setContext(j, float(contextvalp[j][i]))
242 
243  set.finiSample(sample, prefs.getProfAccuracy())
244 
245  #---- Update min and max
246  for j in range(set.getNcontext()):
247  cmin[j] = contextvalp[j][good].min()
248  cmax[j] = contextvalp[j][good].max()
249 
250  # Update the scaling
251  if set.getNsample():
252  for i in range(set.getNcontext()):
253  set.setContextScale(i, cmax[i] - cmin[i])
254  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
255 
256  # Don't waste memory!
257  set.trimMemory()
258 
259  return set
260 
261 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
262 
263 
264 def getSexFlags(*args):
265  return {1: "flux blended",
266  2: "blended",
267  4: "saturated",
268  8: "edge",
269  16: "bad aperture",
270  32: "bad isophotal",
271  64: "memory error (deblend)",
272  128: "memory error (extract)",
273  }
274 
275 
276 def select_candidates(set, prefs, frmin, frmax,
277  flags, flux, fluxerr, rmsSize, elong, vignet,
278  plot=dict(), title=""):
279  maxbad = prefs.getBadpixNmax()
280  maxbadflag = prefs.getBadpixFlag()
281  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100.0
282  minsn = prefs.getMinsn()
283 
284  sn = flux/np.where(fluxerr > 0, fluxerr, 1)
285  sn[fluxerr <= 0] = -psfexLib.BIG
286  #---- Apply some selection over flags, fluxes...
287  plotFlags = plot.get("showFlags") if plt else False
288  plotRejection = plot.get("showRejection") if plt else False
289 
290  bad = flags & prefs.getFlagMask() != 0
291  set.setBadFlags(int(sum(bad != 0)))
292 
293  if plotRejection:
294  selectionVectors = []
295  selectionVectors.append((bad, "flags %d" % sum(bad != 0)))
296 
297  dbad = sn < minsn
298  set.setBadSN(int(sum(dbad)))
299  bad = np.logical_or(bad, dbad)
300  if plotRejection:
301  selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
302 
303  dbad = rmsSize < frmin
304  set.setBadFrmin(int(sum(dbad)))
305  bad = np.logical_or(bad, dbad)
306  if plotRejection:
307  selectionVectors.append((dbad, "frmin %d" % sum(dbad)))
308 
309  dbad = rmsSize > frmax
310  set.setBadFrmax(int(sum(dbad)))
311  bad = np.logical_or(bad, dbad)
312  if plotRejection:
313  selectionVectors.append((dbad, "frmax %d" % sum(dbad)))
314 
315  dbad = elong > maxelong
316  set.setBadElong(int(sum(dbad)))
317  bad = np.logical_or(bad, dbad)
318  if plotRejection:
319  selectionVectors.append((dbad, "elong %d" % sum(dbad)))
320 
321  #-- ... and check the integrity of the sample
322  if maxbadflag:
323  nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
324  dbad = nbad > maxbad
325  set.setBadPix(int(sum(dbad)))
326  bad = np.logical_or(bad, dbad)
327  if plotRejection:
328  selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
329 
330  good = np.logical_not(bad)
331  if plotFlags or plotRejection:
332  imag = -2.5*np.log10(flux)
333  plt.clf()
334 
335  alpha = 0.5
336  if plotFlags:
337  labels = getFlags()
338 
339  isSet = np.where(flags == 0x0)[0]
340  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label="good")
341 
342  for i in range(16):
343  mask = 1 << i
344  if mask & prefs.getFlagMask():
345  isSet = np.where(np.bitwise_and(flags, mask))[0]
346  if isSet.any():
347  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label=labels[mask])
348  else:
349  for bad, label in selectionVectors:
350  plt.plot(imag[bad], rmsSize[bad], 'o', alpha=alpha, label=label)
351 
352  plt.plot(imag[good], rmsSize[good], 'o', color="black", label="selected")
353  [plt.axhline(_, color='red') for _ in [frmin, frmax]]
354  plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
355  plt.ylim(-0.1, 2*frmax)
356  plt.legend(loc=2)
357  plt.xlabel("Instrumental Magnitude")
358  plt.ylabel("rmsSize")
359  plt.title("%s %d selected" % (title, sum(good)))
360 
361  input("Continue? ")
362 
363  return good
364 
365 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
366 
367 try:
368  _dataType
369 except NameError:
370  class _SExtractor(object):
371  pass
372 
373  class _LSST(object):
374  pass
375 
376  _dataTypes = dict(LSST=_LSST,
377  SExtractor=_SExtractor
378  )
379  _dataType = _SExtractor
380 
381 
382 def setDataType(t):
383  _dataType = _dataTypes[t]
384 
385 
386 def getFlags():
387  if _dataType == _LSST:
388  return getLsstFlags(None)
389  else:
390  return getSexFlags(None)
391 
392 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
393 
394 
395 def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
396  minsn = prefs.getMinsn()
397  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
398  min = prefs.getFwhmrange()[0]
399  max = prefs.getFwhmrange()[1]
400 
401  filenames = prefs.getCatalogs()
402 
403  ncat = len(filenames)
404  fwhmmin = np.empty(ncat)
405  fwhmmax = np.empty(ncat)
406 
407  if not prefs.getAutoselectFlag():
408  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
409  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
410  fwhmmode = (fwhmmin + fwhmmax)/2.0
411  else:
412  fwhms = {}
413 
414  #-- Try to estimate the most appropriate Half-light Radius range
415  #-- Get the Half-light radii
416  nobj = 0
417  for i, fileName in enumerate(filenames):
418  fwhms[i] = []
419 
420  if prefs.getVerboseType() != prefs.QUIET:
421  print("Examining Catalog #%d" % (i+1))
422 
423  #---- Read input catalog
424  backnoises = []
425  with pyfits.open(fileName) as cat:
426  extCtr = -1
427  for tab in cat:
428  if tab.name == "LDAC_IMHEAD":
429  extCtr += 1
430 
431  if extCtr != ext and ext != prefs.ALL_EXTENSIONS:
432  if extCtr > ext:
433  break
434  continue
435 
436  if tab.name == "PRIMARY":
437  pass
438  elif tab.name == "LDAC_IMHEAD":
439  hdr = tab.data[0][0] # the fits header from the original fits image
440  for line in hdr:
441  try:
442  k, v = splitFitsCard(line)
443  except AttributeError:
444  continue
445 
446  if k == "SEXBKDEV":
447  if v < 1/psfexLib.BIG:
448  v = 1.0
449 
450  backnoises.append(v)
451  break
452  elif tab.name == "LDAC_OBJECTS":
453  #-------- Fill the FWHM array
454  rmsSize = tab.data["FLUX_RADIUS"]
455  fmax = tab.data["FLUX_MAX"]
456  flags = tab.data["FLAGS"]
457  elong = tab.data["ELONGATION"]
458  backnoise = backnoises[-1]
459 
460  good = np.logical_and(fmax/backnoise > minsn,
461  np.logical_not(flags & prefs.getFlagMask()))
462  good = np.logical_and(good, elong < maxelong)
463  fwhm = 2.0*rmsSize
464  good = np.logical_and(good, fwhm >= min)
465  good = np.logical_and(good, fwhm < max)
466  fwhms[i] = fwhm[good]
467 
468  if prefs.getVarType() == prefs.VAR_NONE:
469  if nobj:
470  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
471  i = 0
472  for l in fwhms.values():
473  fwhms_all[i:len(l)] = l
474  i += len(l)
475  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
476  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
477  plot=plot)
478  else:
479  raise RuntimeError("No source with appropriate FWHM found!!")
480  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
481 
482  fwhmmin = np.zeros(ncat) + min
483  fwhmmax = np.zeros(ncat) + max
484  fwhmmode = np.zeros(ncat) + mode
485  else:
486  fwhmmode = np.empty(ncat)
487  fwhmmin = np.empty(ncat)
488  fwhmmax = np.empty(ncat)
489 
490  for i in range(ncat):
491  nobj = len(fwhms[i])
492  if (nobj):
493  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
494  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
495  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
496  else:
497  raise RuntimeError("No source with appropriate FWHM found!!")
498  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
499 
500  # Read the samples
501  mode = psfexLib.BIG # mode of FWHM distribution
502 
503  sets = []
504  for i, fileName in enumerate(filenames):
505  set = None
506  if ext == prefs.ALL_EXTENSIONS:
507  extensions = list(range(len(backnoises)))
508  else:
509  extensions = [ext]
510 
511  for e in extensions:
512  set = read_samples(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
513  e, next, i, context, context.getPc(i) if context.getNpc() else None, plot=plot)
514 
515  if fwhmmode[i] < mode:
516  mode = fwhmmode[i]
517 
518  set.setFwhm(mode)
519 
520  if prefs.getVerboseType() != prefs.QUIET:
521  if set.getNsample():
522  print("%d samples loaded." % set.getNsample())
523  else:
524  raise RuntimeError("No appropriate source found!!")
525 
526  sets.append(set)
527 
528  return sets
529 
530 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
531 
532 
533 def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5,
534  diagnostics=False, outDir="", frame=None, title=None):
535  """Show a PSF on ds9"""
536 
537  if ext is not None:
538  psf = psf[ext]
539 
540  if wcsData:
541  if ext is not None:
542  wcsData = wcsData[ext]
543  wcs, naxis1, naxis2 = wcsData
544  else:
545  wcs, naxis1, naxis2 = None, None, None
546 
547  naxis = [naxis1, naxis2]
548  for i in range(2):
549  if naxis[i] is None:
550  # cmin, cmax are the range of input star positions
551  cmin, cmax = [set.getContextOffset(i) + d*set.getContextScale(i) for d in (-0.5, 0.5)]
552  naxis[i] = cmax + cmin # a decent guess
553 
554  import lsst.afw.display.utils as ds9Utils
555  if naxis[0] > naxis[1]:
556  nx, ny = int(nspot*naxis[0]/float(naxis[1]) + 0.5), nspot
557  else:
558  nx, ny = nspot, int(nspot*naxis[1]/float(naxis[0]) + 0.5)
559 
560  mos = ds9Utils.Mosaic(gutter=2, background=0.02)
561 
562  xpos, ypos = np.linspace(0, naxis[0], nx), np.linspace(0, naxis[1], ny)
563  for y in ypos:
564  for x in xpos:
565  psf.build(x, y)
566 
567  im = afwImage.ImageF(*psf.getLoc().shape)
568  im.getArray()[:] = psf.getLoc()
569  im /= float(im.getArray().max())
570  if trim:
571  if trim > im.getHeight()//2:
572  trim = im.getHeight()//2
573 
574  im = im[trim:-trim, trim:-trim]
575 
576  mos.append(im)
577 
578  mosaic = mos.makeMosaic(mode=nx)
579  if frame is not None:
580  ds9.mtv(mosaic, frame=frame, title=title)
581  #
582  # Figure out the WCS for the mosaic
583  #
584  pos = []
585  if False:
586  for x, y, i in zip((xpos[0], xpos[-1]), (ypos[0], ypos[-1]), (0, mos.nImage - 1)):
587  bbox = mos.getBBox(i)
588  mosx = bbox.getMinX() + 0.5*(bbox.getWidth() - 1)
589  mosy = bbox.getMinY() + 0.5*(bbox.getHeight() - 1)
590  pos.append([afwGeom.PointD(mosx, mosy), wcs.pixelToSky(afwGeom.PointD(x, y))])
591  else:
592  pos.append([afwGeom.PointD(0, 0), wcs.pixelToSky(afwGeom.PointD(0, 0))])
593  pos.append([afwGeom.PointD(*mosaic.getDimensions()), wcs.pixelToSky(afwGeom.PointD(naxis1, naxis2))])
594 
595  CD = []
596  for i in range(2):
597  delta = pos[1][1][i].asDegrees() - pos[0][1][i].asDegrees()
598  CD.append(delta/(pos[1][0][i] - pos[0][0][i]))
599  mosWcs = afwImage.makeWcs(pos[0][1], pos[0][0], CD[0], 0, 0, CD[1])
600 
601  if ext is not None:
602  title = "%s-%d" % (title, ext)
603 
604  if frame is not None:
605  ds9.mtv(mosaic, frame=frame, title=title, wcs=mosWcs)
606 
607  if diagnostics:
608  outFile = "%s-mod.fits" % title
609  if outDir:
610  outFile = os.path.join(outDir, outFile)
611  mosaic.writeFits(outFile, mosWcs.getFitsMetadata())
612 
613  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
614 
615  mos = ds9Utils.Mosaic(gutter=4, background=0.002)
616  for i in range(set.getNsample()):
617  s = set.getSample(i)
618  if ext is not None and s.getExtindex() != ext:
619  continue
620 
621  smos = ds9Utils.Mosaic(gutter=2, background=-0.003)
622  for func in [s.getVig, s.getVigResi]:
623  arr = func()
624  if func == s.getVig:
625  norm = float(arr.max()) if True else s.getNorm()
626 
627  arr /= norm
628  im = afwImage.ImageF(*arr.shape)
629  im.getArray()[:] = arr
630  smos.append(im)
631 
632  mos.append(smos.makeMosaic(mode="x"))
633 
634  mosaic = mos.makeMosaic(title=title)
635 
636  if frame is not None:
637  ds9.mtv(mosaic, title=title, frame=frame+1)
638 
639  if diagnostics:
640  outFile = "%s-psfstars.fits" % title
641  if outDir:
642  outFile = os.path.join(outDir, outFile)
643 
644  mosaic.writeFits(outFile)
645 
646 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
647 
648 
649 def getLsstFlags(tab=None):
650  flagKeys = [
651  "base_PixelFlags_flag_edge",
652  #"base_PixelFlags_flag_interpolated",
653  #"base_PixelFlags_flag_interpolatedCenter",
654  #"base_PixelFlags_flag_saturated",
655  "base_PixelFlags_flag_saturatedCenter",
656  #"base_PixelFlags_flag_cr",
657  "base_PixelFlags_flag_crCenter",
658  "base_PixelFlags_flag_bad",
659  "base_PsfFlux_flag",
660  "parent",
661  ]
662 
663  if tab is None:
664  flags = {}
665  for i, k in enumerate(flagKeys):
666  flags[1 << i] = re.sub(r"\_flag", "",
667  re.sub(r"^base\_", "", re.sub(r"^base\_PixelFlags\_flag\_", "", k)))
668  else:
669  flags = 0
670  for i, k in enumerate(flagKeys):
671  if k == "parent":
672  try:
673  isSet = tab.get("deblend_nChild") > 0
674  except KeyError:
675  isSet = 0
676  else:
677  isSet = tab.get(k)
678  flags = np.bitwise_or(flags, np.where(isSet, 1 << i, 0))
679 
680  return flags
681 
682 
683 def guessCalexp(fileName):
684  for guess in [
685  re.sub("/src", r"", fileName),
686  re.sub("(SRC([^.]+))", r"CORR\2-exp", fileName),
687  ]:
688  if guess != fileName and os.path.exists(guess):
689  return guess
690 
691  raise RuntimeError("Unable to find a calexp to go with %s" % fileName)
692 
693 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
694 
695 
696 def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
697  """This is the python wrapper that reads lsst tables"""
698  # Create an array of PSFs (one PSF for each extension)
699  if prefs.getVerboseType() != prefs.QUIET:
700  print("----- %d input catalogues:" % prefs.getNcat())
701 
702  if saveWcs: # only needed for making plots
703  wcssList = []
704 
705  fields = psfexLib.vectorField()
706  for cat in prefs.getCatalogs():
707  field = psfexLib.Field(cat)
708  wcss = []
709  wcssList.append(wcss)
710  with pyfits.open(cat):
711  # Hack: I want the WCS so I'll guess where the calexp is to be found
712  calexpFile = guessCalexp(cat)
713  md = afwImage.readMetadata(calexpFile)
714  wcs = afwImage.makeWcs(md)
715 
716  if not wcs:
717  crval = afwCoord.makeCoord(afwCoord.ICRS, 0.0*afwGeom.degrees, 0.0*afwGeom.degrees)
718  wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0, 1.0)
719 
720  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
721  # Find how many rows there are in the catalogue
722  md = afwImage.readMetadata(cat)
723 
724  field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2"))
725  if saveWcs:
726  wcss.append((wcs, naxis1, naxis2))
727 
728  field.finalize()
729  fields.append(field)
730 
731  fields[0].getNext() # number of extensions
732 
733  prefs.getPsfStep()
734 
735  sets = psfexLib.vectorSet()
736  for set in load_samplesLsst(prefs, context, plot=plot):
737  sets.append(set)
738 
739  psfexLib.makeit(fields, sets)
740 
741  ret = [[f.getPsfs() for f in fields], sets]
742  if saveWcs:
743  ret.append(wcssList)
744 
745  return ret
746 
747 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
748 
749 
750 def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
751  plot=dict(showFlags=False, showRejection=False)):
752  # allocate a new set iff set is None
753  if not set:
754  set = psfexLib.Set(context)
755 
756  cmin, cmax = None, None
757  if set.getNcontext():
758  cmin = np.empty(set.getNcontext())
759  cmax = np.empty(set.getNcontext())
760  for i in range(set.getNcontext()):
761  if set.getNsample():
762  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
763  cmax[i] = cmin[i] + set.getContextScale(i)
764  else:
765  cmin[i] = psfexLib.BIG
766  cmax[i] = -psfexLib.BIG
767  #
768  # Read data
769  #
770  tab = afwTable.SourceCatalog.readFits(filename)
771 
772  centroid = tab.getCentroidDefinition()
773  xm = tab.get("%s.x" % centroid)
774  ym = tab.get("%s.y" % centroid)
775 
776  shape = tab.getShapeDefinition()
777  ixx = tab.get("%s.xx" % shape)
778  iyy = tab.get("%s.yy" % shape)
779 
780  rmsSize = np.sqrt(0.5*(ixx + iyy))
781  elong = 0.5*(ixx - iyy)/(ixx + iyy)
782 
783  flux = tab.get(prefs.getPhotfluxRkey())
784  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
785  flags = getLsstFlags(tab)
786 
787  #
788  # Now the VIGNET data
789  #
790  vigw, vigh = 35, 35 # [prefs.getPsfsize()[i] for i in range(2)]
791  if set.empty():
792  set.setVigSize(vigw, vigh)
793 
794  vignet = np.empty(nobj*vigw*vigh, "float32").reshape(nobj, vigw, vigh)
795 
796  # Hack: I want the WCS so I'll guess where the calexp is to be found
797  calexpFile = guessCalexp(filename)
798  mi = afwImage.MaskedImageF(calexpFile)
799  backnoise2 = np.median(mi.getVariance().getArray())
800  gain = 1.0
801 
802  edgeBit = [k for k, v in getFlags().items() if v == "edge"][0]
803 
804  for i, xc, yc in zip(range(nobj), xm, ym):
805  try:
806  x, y = int(xc), int(yc)
807  except ValueError:
808  flags[i] |= edgeBit # mark star as bad
809 
810  try:
811  pstamp = mi[x - vigw//2:x + vigw//2 + 1, y - vigh//2:y + vigh//2 + 1]
812  vignet[i] = pstamp.getImage().getArray().transpose()
813  except Exception:
814  flags[i] |= edgeBit # mark star as bad
815 
816  # Try to load the set of context keys
817  pc = 0
818  contextvalp = []
819  for i, key in enumerate(context.getName()):
820  if context.getPcflag(i):
821  contextvalp.append(pcval[pc])
822  pc += 1
823  elif key[0] == ':':
824  try:
825  contextvalp.append(tab.header[key[1:]])
826  except KeyError:
827  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
828  (key[1:], filename))
829  else:
830  try:
831  contextvalp.append(tab.get(key))
832  except KeyError:
833  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
834  (key, filename))
835  set.setContextname(i, key)
836 
837  # Now examine each vector of the shipment
838  good = select_candidates(set, prefs, frmin, frmax,
839  flags, flux, fluxErr, rmsSize, elong, vignet,
840  plot=plot, title="%s[%d]" % (filename, ext + 1) if next > 1 else filename)
841  #
842  # Insert the good candidates into the set
843  #
844  if not vignet.dtype.isnative:
845  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
846  vignet = vignet.byteswap()
847 
848  for i in np.where(good)[0]:
849  sample = set.newSample()
850  sample.setCatindex(catindex)
851  sample.setExtindex(ext)
852 
853  sample.setVig(vignet[i])
854  if False:
855  pstamp = afwImage.ImageF(*vignet[i].shape)
856  pstamp.getArray()[:] = sample.getVig()
857 
858  ds9.mtv(pstamp)
859 
860  sample.setNorm(float(flux[i]))
861  sample.setBacknoise2(backnoise2)
862  sample.setGain(gain)
863  sample.setX(float(xm[i]))
864  sample.setY(float(ym[i]))
865  sample.setFluxrad(float(rmsSize[i]))
866 
867  for j in range(set.getNcontext()):
868  sample.setContext(j, float(contextvalp[j][i]))
869 
870  set.finiSample(sample, prefs.getProfAccuracy())
871 
872  #---- Update min and max
873  for j in range(set.getNcontext()):
874  cmin[j] = contextvalp[j][good].min()
875  cmax[j] = contextvalp[j][good].max()
876 
877  # Update the scaling
878  if set.getNsample():
879  for i in range(set.getNcontext()):
880  set.setContextScale(i, cmax[i] - cmin[i])
881  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
882 
883  # Don't waste memory!
884  set.trimMemory()
885 
886  return set
887 
888 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
889 
890 
891 def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
892  minsn = prefs.getMinsn()
893  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
894  min = prefs.getFwhmrange()[0]
895  max = prefs.getFwhmrange()[1]
896 
897  filenames = prefs.getCatalogs()
898 
899  ncat = len(filenames)
900  fwhmmin = np.empty(ncat)
901  fwhmmax = np.empty(ncat)
902 
903  if not prefs.getAutoselectFlag():
904  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
905  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
906  fwhmmode = (fwhmmin + fwhmmax)/2.0
907  else:
908  fwhms = {}
909 
910  #-- Try to estimate the most appropriate Half-light Radius range
911  #-- Get the Half-light radii
912  nobj = 0
913  for i, fileName in enumerate(filenames):
914  fwhms[i] = []
915 
916  if prefs.getVerboseType() != prefs.QUIET:
917  print("Examining Catalog #%d" % (i+1))
918 
919  #---- Read input catalog
920  tab = afwTable.SourceCatalog.readFits(fileName)
921 
922  #-------- Fill the FWHM array
923  shape = tab.getShapeDefinition()
924  ixx = tab.get("%s.xx" % shape)
925  iyy = tab.get("%s.yy" % shape)
926 
927  rmsSize = np.sqrt(0.5*(ixx + iyy))
928  elong = 0.5*(ixx - iyy)/(ixx + iyy)
929 
930  flux = tab.get(prefs.getPhotfluxRkey())
931  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
932 
933  flags = getLsstFlags(tab)
934 
935  good = np.logical_and(flux/fluxErr > minsn,
936  np.logical_not(flags & prefs.getFlagMask()))
937  good = np.logical_and(good, elong < maxelong)
938  fwhm = 2.0*rmsSize
939  good = np.logical_and(good, fwhm >= min)
940  good = np.logical_and(good, fwhm < max)
941  fwhms[i] = fwhm[good]
942 
943  if prefs.getVarType() == prefs.VAR_NONE:
944  if nobj:
945  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
946  i = 0
947  for l in fwhms.values():
948  fwhms_all[i:len(l)] = l
949  i += len(l)
950  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
951  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
952  plot=plot)
953  else:
954  raise RuntimeError("No source with appropriate FWHM found!!")
955  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
956 
957  fwhmmin = np.zeros(ncat) + min
958  fwhmmax = np.zeros(ncat) + max
959  fwhmmode = np.zeros(ncat) + mode
960  else:
961  fwhmmode = np.empty(ncat)
962  fwhmmin = np.empty(ncat)
963  fwhmmax = np.empty(ncat)
964 
965  for i in range(ncat):
966  nobj = len(fwhms[i])
967  if (nobj):
968  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
969  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
970  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
971  else:
972  raise RuntimeError("No source with appropriate FWHM found!!")
973  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
974 
975  # Read the samples
976  mode = psfexLib.BIG # mode of FWHM distribution
977 
978  sets = []
979  for i, fileName in enumerate(filenames):
980  set = None
981  for ext in range(next):
982  set = read_samplesLsst(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
983  ext, next, i, context,
984  context.getPc(i) if context.getNpc() else None, plot=plot)
985 
986  if fwhmmode[i] < mode:
987  mode = fwhmmode[i]
988 
989  set.setFwhm(mode)
990 
991  if prefs.getVerboseType() != prefs.QUIET:
992  if set.getNsample():
993  print("%d samples loaded." % set.getNsample())
994  else:
995  raise RuntimeError("No appropriate source found!!")
996 
997  sets.append(set)
998 
999  return sets
1000 
1001 
1002 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
1003 
1004 def makeit(prefs, context, saveWcs=False, plot=dict()):
1005  """This is the python wrapper for the original psfex that reads SExtractor outputs"""
1006  # Create an array of PSFs (one PSF for each extension)
1007  if prefs.getVerboseType() != prefs.QUIET:
1008  print("----- %d input catalogues:" % prefs.getNcat())
1009 
1010  if saveWcs: # only needed for making plots
1011  wcssList = []
1012 
1013  fields = psfexLib.vectorField()
1014  for cat in prefs.getCatalogs():
1015  field = psfexLib.Field(cat)
1016  wcss = []
1017  wcssList.append(wcss)
1018  with pyfits.open(cat) as pf:
1019  for hdu in pf:
1020  if hdu.name == "PRIMARY":
1021  pass
1022  elif hdu.name == "LDAC_IMHEAD":
1023  hdr = hdu.data[0][0] # the fits header from the original fits image
1024  md = PropertySet()
1025  for line in hdr:
1026  try:
1027  md.set(*splitFitsCard(line))
1028  except AttributeError:
1029  continue
1030 
1031  if not md.exists("CRPIX1"): # no WCS; try WCSA
1032  for k in md.names():
1033  if re.search(r"A$", k):
1034  md.set(k[:-1], md.get(k))
1035  wcs = afwImage.makeWcs(md)
1036  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
1037  elif hdu.name == "LDAC_OBJECTS":
1038  nobj = len(hdu.data)
1039 
1040  assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
1041  field.addExt(wcs, naxis1, naxis2, nobj)
1042  if saveWcs:
1043  wcss.append((wcs, naxis1, naxis2))
1044  wcs = None
1045 
1046  field.finalize()
1047  fields.append(field)
1048 
1049  sets = psfexLib.vectorSet()
1050  for set in load_samples(prefs, context, plot=plot):
1051  sets.append(set)
1052 
1053  psfexLib.makeit(fields, sets)
1054 
1055  ret = [[f.getPsfs() for f in fields], sets]
1056  if saveWcs:
1057  ret.append(wcssList)
1058 
1059  return ret
def select_candidates(set, prefs, frmin, frmax, flags, flux, fluxerr, rmsSize, elong, vignet, plot=dict(), title="")
Definition: psfex.py:278
def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:105
def makeit(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:1004
def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5, diagnostics=False, outDir="", frame=None, title=None)
Definition: psfex.py:534
def makeitLsst(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:696
def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False))
Definition: psfex.py:50
def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:751
def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:395
def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:891