lsst.meas.extensions.psfex  16.0-10-gc1446dd+48
psfex.py
Go to the documentation of this file.
1 import os
2 import re
3 import sys
4 
5 import numpy as np
6 from astropy.io import fits
7 try:
8  import matplotlib.pyplot as plt
9 except ImportError:
10  plt = None
11 
12 import lsst.afw.geom as afwGeom
13 from lsst.afw.fits import readMetadata
14 import lsst.afw.image as afwImage
15 import lsst.afw.table as afwTable
16 import lsst.afw.display.ds9 as ds9
17 from lsst.daf.base import PropertySet
18 from . import psfexLib
19 
20 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
21 
22 
23 def splitFitsCard(line):
24  """Split a fits header, returning (key, value)"""
25  try:
26  k, v = re.search(r"(\S+)\s*=\s*'?((?:\S+|'))", line).groups()
27  except AttributeError:
28  raise
29 
30  try:
31  v = int(v)
32  except ValueError:
33  try:
34  v = float(v)
35  except ValueError:
36  pass
37 
38  return k, v
39 
40 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
41 
42 
43 def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False)):
44  """
45  PURPOSE Compute the FWHM range associated to a series of FWHM measurements.
46  INPUT Pointer to an array of FWHMs,
47  maximum allowed FWHM variation,
48  minimum allowed FWHM,
49  maximum allowed FWHM,
50 
51  OUTPUT FWHM mode, lower FWHM range, upper FWHM range
52  NOTES -.
53  AUTHOR E. Bertin (IAP, Leiden observatory & ESO)
54  VERSION 20/03/2008
55  """
56  nfwhm = len(fwhm)
57  fwhm.sort()
58 
59  # Find the mode
60  nw = nfwhm//4
61  if nw < 4:
62  nw = 1
63  dfmin = psfexLib.BIG
64  fmin = 0.0
65  for i in range(nfwhm - nw):
66  df = fwhm[i + nw] - fwhm[i]
67  if df < dfmin:
68  dfmin = df
69  fmin = (fwhm[i + nw] + fwhm[i])/2.0
70 
71  if nfwhm < 2:
72  fmin = fwhm[0]
73 
74  dfmin = (maxvar + 1.0)**0.3333333
75  minout = fmin/dfmin if dfmin > 0.0 else 0.0
76  if minout < minin:
77  minout = minin
78 
79  maxout = fmin*dfmin**2
80  if maxout > maxin:
81  maxout = maxin
82 
83  if plt and plot.get("fwhmHistogram"):
84  plt.clf()
85  plt.hist(fwhm, nfwhm//10 + 1, normed=1, facecolor='g', alpha=0.75)
86  plt.xlabel("FWHM")
87  plt.axvline(fmin, color='red')
88  [plt.axvline(_, color='blue') for _ in (minout, maxout)]
89 
90  input("Continue? ")
91 
92  return fmin, minout, maxout
93 
94 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
95 
96 
97 def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
98  plot=dict(showFlags=False, showRejection=False)):
99  # allocate a new set iff set is None
100  if not set:
101  set = psfexLib.Set(context)
102 
103  cmin, cmax = None, None
104  if set.getNcontext():
105  cmin = np.empty(set.getNcontext())
106  cmax = np.empty(set.getNcontext())
107  for i in range(set.getNcontext()):
108  if set.getNsample():
109  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
110  cmax[i] = cmin[i] + set.getContextScale(i)
111  else:
112  cmin[i] = psfexLib.BIG
113  cmax[i] = -psfexLib.BIG
114  #
115  # Read data
116  #
117  with fits.open(filename) as cat:
118  extCtr = -1
119  for tab in cat:
120  if tab.name == "LDAC_IMHEAD":
121  extCtr += 1
122 
123  if extCtr < ext:
124  continue
125  elif extCtr > ext:
126  break
127 
128  if tab.name == "PRIMARY":
129  pass
130  elif tab.name == "LDAC_IMHEAD":
131  hdr = tab.data[0][0] # the fits header from the original fits image
132  foundCards = 0 # how many of our desired cards have we found?
133 
134  for line in hdr:
135  try:
136  k, v = splitFitsCard(line)
137  except AttributeError:
138  continue
139 
140  if k == "SEXBKDEV":
141  backnoise2 = v**2
142  foundCards += 1
143  elif k == "SEXGAIN":
144  gain = v
145  foundCards += 1
146 
147  if foundCards == 2:
148  break
149  elif tab.name == "LDAC_OBJECTS":
150  xm = tab.data[prefs.getCenterKey(0)]
151  ym = tab.data[prefs.getCenterKey(1)]
152  fluxrad = tab.data["FLUX_RADIUS"]
153  flux = tab.data[prefs.getPhotfluxRkey()]
154  fluxerr = tab.data[prefs.getPhotfluxerrRkey()]
155  elong = tab.data["ELONGATION"]
156  flags = tab.data["FLAGS"]
157 
158  n = prefs.getPhotfluxNum() - 1
159  if n:
160  assert False, "Code to handle e.g. FLUX_APER(3) isn't yet converted"
161  if key.naxis == 1 and n < key.naxisn[0]:
162  flux += n
163  else:
164  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
165  (prefs.getPhotfluxRkey(), filename), file=sys.stderr)
166 
167  n = prefs.getPhotfluxerrNum() - 1
168  if n:
169  if key.naxis == 1 and n < key.naxisn[0]:
170  fluxerr += n
171  else:
172  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
173  (prefs.getPhotfluxerrRkey(), filename), file=sys.stderr)
174  #
175  # Now the VIGNET data
176  #
177  vignet = tab.data["VIGNET"]
178 
179  try:
180  vigw, vigh = vignet[0].shape
181  except ValueError:
182  raise RuntimeError("*Error*: VIGNET should be a 2D vector; saw %s" % str(vignet[0].shape))
183 
184  if set.empty():
185  set.setVigSize(vigw, vigh)
186 
187  # Try to load the set of context keys
188  pc = 0
189  contextvalp = []
190  for i, key in enumerate(context.getName()):
191  if context.getPcflag(i):
192  contextvalp.append(pcval[pc])
193  pc += 1
194  elif key[0] == ':':
195  try:
196  contextvalp.append(tab.header[key[1:]])
197  except KeyError:
198  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
199  (key[1:], filename))
200  else:
201  try:
202  contextvalp.append(tab.data[key])
203  except KeyError:
204  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
205  (key, filename))
206  set.setContextname(i, key)
207 
208  # Now examine each vector of the shipment
209  good = select_candidates(set, prefs, frmin, frmax,
210  flags, flux, fluxerr, fluxrad, elong, vignet,
211  plot=plot, title="%s[%d]" % (filename, ext + 1))
212  #
213  # Insert the good candidates into the set
214  #
215  if not vignet.dtype.isnative:
216  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
217  vignet = vignet.byteswap()
218 
219  for i in np.where(good)[0]:
220  sample = set.newSample()
221  sample.setCatindex(catindex)
222  sample.setExtindex(ext)
223 
224  sample.setVig(vignet[i])
225 
226  sample.setNorm(float(flux[i]))
227  sample.setBacknoise2(backnoise2)
228  sample.setGain(gain)
229  sample.setX(float(xm[i]))
230  sample.setY(float(ym[i]))
231  sample.setFluxrad(float(fluxrad[i]))
232 
233  for j in range(set.getNcontext()):
234  sample.setContext(j, float(contextvalp[j][i]))
235 
236  set.finiSample(sample, prefs.getProfAccuracy())
237 
238  #---- Update min and max
239  for j in range(set.getNcontext()):
240  cmin[j] = contextvalp[j][good].min()
241  cmax[j] = contextvalp[j][good].max()
242 
243  # Update the scaling
244  if set.getNsample():
245  for i in range(set.getNcontext()):
246  set.setContextScale(i, cmax[i] - cmin[i])
247  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
248 
249  # Don't waste memory!
250  set.trimMemory()
251 
252  return set
253 
254 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
255 
256 
257 def getSexFlags(*args):
258  return {1: "flux blended",
259  2: "blended",
260  4: "saturated",
261  8: "edge",
262  16: "bad aperture",
263  32: "bad isophotal",
264  64: "memory error (deblend)",
265  128: "memory error (extract)",
266  }
267 
268 
269 def select_candidates(set, prefs, frmin, frmax,
270  flags, flux, fluxerr, rmsSize, elong, vignet,
271  plot=dict(), title=""):
272  maxbad = prefs.getBadpixNmax()
273  maxbadflag = prefs.getBadpixFlag()
274  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100.0
275  minsn = prefs.getMinsn()
276 
277  sn = flux/np.where(fluxerr > 0, fluxerr, 1)
278  sn[fluxerr <= 0] = -psfexLib.BIG
279  #---- Apply some selection over flags, fluxes...
280  plotFlags = plot.get("showFlags") if plt else False
281  plotRejection = plot.get("showRejection") if plt else False
282 
283  bad = flags & prefs.getFlagMask() != 0
284  set.setBadFlags(int(sum(bad != 0)))
285 
286  if plotRejection:
287  selectionVectors = []
288  selectionVectors.append((bad, "flags %d" % sum(bad != 0)))
289 
290  dbad = sn < minsn
291  set.setBadSN(int(sum(dbad)))
292  bad = np.logical_or(bad, dbad)
293  if plotRejection:
294  selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
295 
296  dbad = rmsSize < frmin
297  set.setBadFrmin(int(sum(dbad)))
298  bad = np.logical_or(bad, dbad)
299  if plotRejection:
300  selectionVectors.append((dbad, "frmin %d" % sum(dbad)))
301 
302  dbad = rmsSize > frmax
303  set.setBadFrmax(int(sum(dbad)))
304  bad = np.logical_or(bad, dbad)
305  if plotRejection:
306  selectionVectors.append((dbad, "frmax %d" % sum(dbad)))
307 
308  dbad = elong > maxelong
309  set.setBadElong(int(sum(dbad)))
310  bad = np.logical_or(bad, dbad)
311  if plotRejection:
312  selectionVectors.append((dbad, "elong %d" % sum(dbad)))
313 
314  #-- ... and check the integrity of the sample
315  if maxbadflag:
316  nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
317  dbad = nbad > maxbad
318  set.setBadPix(int(sum(dbad)))
319  bad = np.logical_or(bad, dbad)
320  if plotRejection:
321  selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
322 
323  good = np.logical_not(bad)
324  if plotFlags or plotRejection:
325  imag = -2.5*np.log10(flux)
326  plt.clf()
327 
328  alpha = 0.5
329  if plotFlags:
330  labels = getFlags()
331 
332  isSet = np.where(flags == 0x0)[0]
333  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label="good")
334 
335  for i in range(16):
336  mask = 1 << i
337  if mask & prefs.getFlagMask():
338  isSet = np.where(np.bitwise_and(flags, mask))[0]
339  if isSet.any():
340  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label=labels[mask])
341  else:
342  for bad, label in selectionVectors:
343  plt.plot(imag[bad], rmsSize[bad], 'o', alpha=alpha, label=label)
344 
345  plt.plot(imag[good], rmsSize[good], 'o', color="black", label="selected")
346  [plt.axhline(_, color='red') for _ in [frmin, frmax]]
347  plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
348  plt.ylim(-0.1, 2*frmax)
349  plt.legend(loc=2)
350  plt.xlabel("Instrumental Magnitude")
351  plt.ylabel("rmsSize")
352  plt.title("%s %d selected" % (title, sum(good)))
353 
354  input("Continue? ")
355 
356  return good
357 
358 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
359 
360 try:
361  _dataType
362 except NameError:
363  class _SExtractor():
364  pass
365 
366  class _LSST():
367  pass
368 
369  _dataTypes = dict(LSST=_LSST,
370  SExtractor=_SExtractor
371  )
372  _dataType = _SExtractor
373 
374 
375 def setDataType(t):
376  _dataType = _dataTypes[t]
377 
378 
379 def getFlags():
380  if _dataType == _LSST:
381  return getLsstFlags(None)
382  else:
383  return getSexFlags(None)
384 
385 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
386 
387 
388 def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
389  minsn = prefs.getMinsn()
390  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
391  min = prefs.getFwhmrange()[0]
392  max = prefs.getFwhmrange()[1]
393 
394  filenames = prefs.getCatalogs()
395 
396  ncat = len(filenames)
397  fwhmmin = np.empty(ncat)
398  fwhmmax = np.empty(ncat)
399 
400  if not prefs.getAutoselectFlag():
401  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
402  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
403  fwhmmode = (fwhmmin + fwhmmax)/2.0
404  else:
405  fwhms = {}
406 
407  #-- Try to estimate the most appropriate Half-light Radius range
408  #-- Get the Half-light radii
409  nobj = 0
410  for i, fileName in enumerate(filenames):
411  fwhms[i] = []
412 
413  if prefs.getVerboseType() != prefs.QUIET:
414  print("Examining Catalog #%d" % (i+1))
415 
416  #---- Read input catalog
417  backnoises = []
418  with fits.open(fileName) as cat:
419  extCtr = -1
420  for tab in cat:
421  if tab.name == "LDAC_IMHEAD":
422  extCtr += 1
423 
424  if extCtr != ext and ext != prefs.ALL_EXTENSIONS:
425  if extCtr > ext:
426  break
427  continue
428 
429  if tab.name == "PRIMARY":
430  pass
431  elif tab.name == "LDAC_IMHEAD":
432  hdr = tab.data[0][0] # the fits header from the original fits image
433  for line in hdr:
434  try:
435  k, v = splitFitsCard(line)
436  except AttributeError:
437  continue
438 
439  if k == "SEXBKDEV":
440  if v < 1/psfexLib.BIG:
441  v = 1.0
442 
443  backnoises.append(v)
444  break
445  elif tab.name == "LDAC_OBJECTS":
446  #-------- Fill the FWHM array
447  rmsSize = tab.data["FLUX_RADIUS"]
448  fmax = tab.data["FLUX_MAX"]
449  flags = tab.data["FLAGS"]
450  elong = tab.data["ELONGATION"]
451  backnoise = backnoises[-1]
452 
453  good = np.logical_and(fmax/backnoise > minsn,
454  np.logical_not(flags & prefs.getFlagMask()))
455  good = np.logical_and(good, elong < maxelong)
456  fwhm = 2.0*rmsSize
457  good = np.logical_and(good, fwhm >= min)
458  good = np.logical_and(good, fwhm < max)
459  fwhms[i] = fwhm[good]
460 
461  if prefs.getVarType() == prefs.VAR_NONE:
462  if nobj:
463  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
464  i = 0
465  for l in fwhms.values():
466  fwhms_all[i:len(l)] = l
467  i += len(l)
468  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
469  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
470  plot=plot)
471  else:
472  raise RuntimeError("No source with appropriate FWHM found!!")
473  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
474 
475  fwhmmin = np.zeros(ncat) + min
476  fwhmmax = np.zeros(ncat) + max
477  fwhmmode = np.zeros(ncat) + mode
478  else:
479  fwhmmode = np.empty(ncat)
480  fwhmmin = np.empty(ncat)
481  fwhmmax = np.empty(ncat)
482 
483  for i in range(ncat):
484  nobj = len(fwhms[i])
485  if (nobj):
486  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
487  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
488  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
489  else:
490  raise RuntimeError("No source with appropriate FWHM found!!")
491  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
492 
493  # Read the samples
494  mode = psfexLib.BIG # mode of FWHM distribution
495 
496  sets = []
497  for i, fileName in enumerate(filenames):
498  set = None
499  if ext == prefs.ALL_EXTENSIONS:
500  extensions = list(range(len(backnoises)))
501  else:
502  extensions = [ext]
503 
504  for e in extensions:
505  set = read_samples(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
506  e, next, i, context, context.getPc(i) if context.getNpc() else None, plot=plot)
507 
508  if fwhmmode[i] < mode:
509  mode = fwhmmode[i]
510 
511  set.setFwhm(mode)
512 
513  if prefs.getVerboseType() != prefs.QUIET:
514  if set.getNsample():
515  print("%d samples loaded." % set.getNsample())
516  else:
517  raise RuntimeError("No appropriate source found!!")
518 
519  sets.append(set)
520 
521  return sets
522 
523 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
524 
525 
526 def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5,
527  diagnostics=False, outDir="", frame=None, title=None):
528  """Show a PSF on ds9"""
529 
530  if ext is not None:
531  psf = psf[ext]
532 
533  if wcsData:
534  if ext is not None:
535  wcsData = wcsData[ext]
536  wcs, naxis1, naxis2 = wcsData
537  else:
538  wcs, naxis1, naxis2 = None, None, None
539 
540  naxis = [naxis1, naxis2]
541  for i in range(2):
542  if naxis[i] is None:
543  # cmin, cmax are the range of input star positions
544  cmin, cmax = [set.getContextOffset(i) + d*set.getContextScale(i) for d in (-0.5, 0.5)]
545  naxis[i] = cmax + cmin # a decent guess
546 
547  import lsst.afw.display.utils as ds9Utils
548  if naxis[0] > naxis[1]:
549  nx, ny = int(nspot*naxis[0]/float(naxis[1]) + 0.5), nspot
550  else:
551  nx, ny = nspot, int(nspot*naxis[1]/float(naxis[0]) + 0.5)
552 
553  mos = ds9Utils.Mosaic(gutter=2, background=0.02)
554 
555  xpos, ypos = np.linspace(0, naxis[0], nx), np.linspace(0, naxis[1], ny)
556  for y in ypos:
557  for x in xpos:
558  psf.build(x, y)
559 
560  im = afwImage.ImageF(*psf.getLoc().shape)
561  im.getArray()[:] = psf.getLoc()
562  im /= float(im.getArray().max())
563  if trim:
564  if trim > im.getHeight()//2:
565  trim = im.getHeight()//2
566 
567  im = im[trim:-trim, trim:-trim]
568 
569  mos.append(im)
570 
571  mosaic = mos.makeMosaic(mode=nx)
572  if frame is not None:
573  ds9.mtv(mosaic, frame=frame, title=title)
574  #
575  # Figure out the WCS for the mosaic
576  #
577  pos = []
578  if False:
579  for x, y, i in zip((xpos[0], xpos[-1]), (ypos[0], ypos[-1]), (0, mos.nImage - 1)):
580  bbox = mos.getBBox(i)
581  mosx = bbox.getMinX() + 0.5*(bbox.getWidth() - 1)
582  mosy = bbox.getMinY() + 0.5*(bbox.getHeight() - 1)
583  pos.append([afwGeom.PointD(mosx, mosy), wcs.pixelToSky(afwGeom.PointD(x, y))])
584  else:
585  pos.append([afwGeom.PointD(0, 0), wcs.pixelToSky(afwGeom.PointD(0, 0))])
586  pos.append([afwGeom.PointD(*mosaic.getDimensions()), wcs.pixelToSky(afwGeom.PointD(naxis1, naxis2))])
587 
588  CD = []
589  for i in range(2):
590  delta = pos[1][1][i].asDegrees() - pos[0][1][i].asDegrees()
591  CD.append(delta/(pos[1][0][i] - pos[0][0][i]))
592  CD = np.array(CD)
593  CD.shape = (2, 2)
594  mosWcs = afwGeom.makeSkyWcs(crval=pos[0][0], crpix=pos[0][1], cdMatrix=CD)
595 
596  if ext is not None:
597  title = "%s-%d" % (title, ext)
598 
599  if frame is not None:
600  ds9.mtv(mosaic, frame=frame, title=title, wcs=mosWcs)
601 
602  if diagnostics:
603  outFile = "%s-mod.fits" % title
604  if outDir:
605  outFile = os.path.join(outDir, outFile)
606  mosaic.writeFits(outFile, mosWcs.getFitsMetadata())
607 
608  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
609 
610  mos = ds9Utils.Mosaic(gutter=4, background=0.002)
611  for i in range(set.getNsample()):
612  s = set.getSample(i)
613  if ext is not None and s.getExtindex() != ext:
614  continue
615 
616  smos = ds9Utils.Mosaic(gutter=2, background=-0.003)
617  for func in [s.getVig, s.getVigResi]:
618  arr = func()
619  if func == s.getVig:
620  norm = float(arr.max()) if True else s.getNorm()
621 
622  arr /= norm
623  im = afwImage.ImageF(*arr.shape)
624  im.getArray()[:] = arr
625  smos.append(im)
626 
627  mos.append(smos.makeMosaic(mode="x"))
628 
629  mosaic = mos.makeMosaic(title=title)
630 
631  if frame is not None:
632  ds9.mtv(mosaic, title=title, frame=frame+1)
633 
634  if diagnostics:
635  outFile = "%s-psfstars.fits" % title
636  if outDir:
637  outFile = os.path.join(outDir, outFile)
638 
639  mosaic.writeFits(outFile)
640 
641 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
642 
643 
644 def getLsstFlags(tab=None):
645  flagKeys = [
646  "base_PixelFlags_flag_edge",
647  #"base_PixelFlags_flag_interpolated",
648  #"base_PixelFlags_flag_interpolatedCenter",
649  #"base_PixelFlags_flag_saturated",
650  "base_PixelFlags_flag_saturatedCenter",
651  #"base_PixelFlags_flag_cr",
652  "base_PixelFlags_flag_crCenter",
653  "base_PixelFlags_flag_bad",
654  "base_PsfFlux_flag",
655  "parent",
656  ]
657 
658  if tab is None:
659  flags = {}
660  for i, k in enumerate(flagKeys):
661  flags[1 << i] = re.sub(r"\_flag", "",
662  re.sub(r"^base\_", "", re.sub(r"^base\_PixelFlags\_flag\_", "", k)))
663  else:
664  flags = 0
665  for i, k in enumerate(flagKeys):
666  if k == "parent":
667  try:
668  isSet = tab.get("deblend_nChild") > 0
669  except KeyError:
670  isSet = 0
671  else:
672  isSet = tab.get(k)
673  flags = np.bitwise_or(flags, np.where(isSet, 1 << i, 0))
674 
675  return flags
676 
677 
678 def guessCalexp(fileName):
679  for guess in [
680  re.sub("/src", r"", fileName),
681  re.sub("(SRC([^.]+))", r"CORR\2-exp", fileName),
682  ]:
683  if guess != fileName and os.path.exists(guess):
684  return guess
685 
686  raise RuntimeError("Unable to find a calexp to go with %s" % fileName)
687 
688 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
689 
690 
691 def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
692  """This is the python wrapper that reads lsst tables"""
693  # Create an array of PSFs (one PSF for each extension)
694  if prefs.getVerboseType() != prefs.QUIET:
695  print("----- %d input catalogues:" % prefs.getNcat())
696 
697  if saveWcs: # only needed for making plots
698  wcssList = []
699 
700  fields = psfexLib.vectorField()
701  for cat in prefs.getCatalogs():
702  field = psfexLib.Field(cat)
703  wcss = []
704  wcssList.append(wcss)
705  with fits.open(cat):
706  # Hack: I want the WCS so I'll guess where the calexp is to be found
707  calexpFile = guessCalexp(cat)
708  md = readMetadata(calexpFile)
709  wcs = afwGeom.makeSkyWcs(md)
710 
711  if not wcs:
712  cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
713  cdMatrix.shape = (2, 2)
714  wcs = afwGeom.makeSkyWcs(crpix=afwGeom.PointD(0, 0),
715  crval=afwGeom.SpherePoint(0.0, 0.0, afwGeom.degrees),
716  cdMatrix=cdMatrix)
717 
718  naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2")
719  # Find how many rows there are in the catalogue
720  md = readMetadata(cat)
721 
722  field.addExt(wcs, naxis1, naxis2, md.getScalar("NAXIS2"))
723  if saveWcs:
724  wcss.append((wcs, naxis1, naxis2))
725 
726  field.finalize()
727  fields.append(field)
728 
729  fields[0].getNext() # number of extensions
730 
731  prefs.getPsfStep()
732 
733  sets = psfexLib.vectorSet()
734  for set in load_samplesLsst(prefs, context, plot=plot):
735  sets.append(set)
736 
737  psfexLib.makeit(fields, sets)
738 
739  ret = [[f.getPsfs() for f in fields], sets]
740  if saveWcs:
741  ret.append(wcssList)
742 
743  return ret
744 
745 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
746 
747 
748 def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
749  plot=dict(showFlags=False, showRejection=False)):
750  # allocate a new set iff set is None
751  if not set:
752  set = psfexLib.Set(context)
753 
754  cmin, cmax = None, None
755  if set.getNcontext():
756  cmin = np.empty(set.getNcontext())
757  cmax = np.empty(set.getNcontext())
758  for i in range(set.getNcontext()):
759  if set.getNsample():
760  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
761  cmax[i] = cmin[i] + set.getContextScale(i)
762  else:
763  cmin[i] = psfexLib.BIG
764  cmax[i] = -psfexLib.BIG
765  #
766  # Read data
767  #
768  tab = afwTable.SourceCatalog.readFits(filename)
769 
770  centroid = tab.getCentroidDefinition()
771  xm = tab.get("%s.x" % centroid)
772  ym = tab.get("%s.y" % centroid)
773 
774  shape = tab.getShapeDefinition()
775  ixx = tab.get("%s.xx" % shape)
776  iyy = tab.get("%s.yy" % shape)
777 
778  rmsSize = np.sqrt(0.5*(ixx + iyy))
779  elong = 0.5*(ixx - iyy)/(ixx + iyy)
780 
781  flux = tab.get(prefs.getPhotfluxRkey())
782  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
783  flags = getLsstFlags(tab)
784 
785  #
786  # Now the VIGNET data
787  #
788  vigw, vigh = 35, 35 # [prefs.getPsfsize()[i] for i in range(2)]
789  if set.empty():
790  set.setVigSize(vigw, vigh)
791 
792  vignet = np.empty(nobj*vigw*vigh, "float32").reshape(nobj, vigw, vigh)
793 
794  # Hack: I want the WCS so I'll guess where the calexp is to be found
795  calexpFile = guessCalexp(filename)
796  mi = afwImage.MaskedImageF(calexpFile)
797  backnoise2 = np.median(mi.getVariance().getArray())
798  gain = 1.0
799 
800  edgeBit = [k for k, v in getFlags().items() if v == "edge"][0]
801 
802  for i, xc, yc in zip(range(nobj), xm, ym):
803  try:
804  x, y = int(xc), int(yc)
805  except ValueError:
806  flags[i] |= edgeBit # mark star as bad
807 
808  try:
809  pstamp = mi[x - vigw//2:x + vigw//2 + 1, y - vigh//2:y + vigh//2 + 1]
810  vignet[i] = pstamp.getImage().getArray().transpose()
811  except Exception:
812  flags[i] |= edgeBit # mark star as bad
813 
814  # Try to load the set of context keys
815  pc = 0
816  contextvalp = []
817  for i, key in enumerate(context.getName()):
818  if context.getPcflag(i):
819  contextvalp.append(pcval[pc])
820  pc += 1
821  elif key[0] == ':':
822  try:
823  contextvalp.append(tab.header[key[1:]])
824  except KeyError:
825  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
826  (key[1:], filename))
827  else:
828  try:
829  contextvalp.append(tab.get(key))
830  except KeyError:
831  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
832  (key, filename))
833  set.setContextname(i, key)
834 
835  # Now examine each vector of the shipment
836  good = select_candidates(set, prefs, frmin, frmax,
837  flags, flux, fluxErr, rmsSize, elong, vignet,
838  plot=plot, title="%s[%d]" % (filename, ext + 1) if next > 1 else filename)
839  #
840  # Insert the good candidates into the set
841  #
842  if not vignet.dtype.isnative:
843  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
844  vignet = vignet.byteswap()
845 
846  for i in np.where(good)[0]:
847  sample = set.newSample()
848  sample.setCatindex(catindex)
849  sample.setExtindex(ext)
850 
851  sample.setVig(vignet[i])
852  if False:
853  pstamp = afwImage.ImageF(*vignet[i].shape)
854  pstamp.getArray()[:] = sample.getVig()
855 
856  ds9.mtv(pstamp)
857 
858  sample.setNorm(float(flux[i]))
859  sample.setBacknoise2(backnoise2)
860  sample.setGain(gain)
861  sample.setX(float(xm[i]))
862  sample.setY(float(ym[i]))
863  sample.setFluxrad(float(rmsSize[i]))
864 
865  for j in range(set.getNcontext()):
866  sample.setContext(j, float(contextvalp[j][i]))
867 
868  set.finiSample(sample, prefs.getProfAccuracy())
869 
870  #---- Update min and max
871  for j in range(set.getNcontext()):
872  cmin[j] = contextvalp[j][good].min()
873  cmax[j] = contextvalp[j][good].max()
874 
875  # Update the scaling
876  if set.getNsample():
877  for i in range(set.getNcontext()):
878  set.setContextScale(i, cmax[i] - cmin[i])
879  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
880 
881  # Don't waste memory!
882  set.trimMemory()
883 
884  return set
885 
886 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
887 
888 
889 def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
890  minsn = prefs.getMinsn()
891  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
892  min = prefs.getFwhmrange()[0]
893  max = prefs.getFwhmrange()[1]
894 
895  filenames = prefs.getCatalogs()
896 
897  ncat = len(filenames)
898  fwhmmin = np.empty(ncat)
899  fwhmmax = np.empty(ncat)
900 
901  if not prefs.getAutoselectFlag():
902  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
903  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
904  fwhmmode = (fwhmmin + fwhmmax)/2.0
905  else:
906  fwhms = {}
907 
908  #-- Try to estimate the most appropriate Half-light Radius range
909  #-- Get the Half-light radii
910  nobj = 0
911  for i, fileName in enumerate(filenames):
912  fwhms[i] = []
913 
914  if prefs.getVerboseType() != prefs.QUIET:
915  print("Examining Catalog #%d" % (i+1))
916 
917  #---- Read input catalog
918  tab = afwTable.SourceCatalog.readFits(fileName)
919 
920  #-------- Fill the FWHM array
921  shape = tab.getShapeDefinition()
922  ixx = tab.get("%s.xx" % shape)
923  iyy = tab.get("%s.yy" % shape)
924 
925  rmsSize = np.sqrt(0.5*(ixx + iyy))
926  elong = 0.5*(ixx - iyy)/(ixx + iyy)
927 
928  flux = tab.get(prefs.getPhotfluxRkey())
929  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
930 
931  flags = getLsstFlags(tab)
932 
933  good = np.logical_and(flux/fluxErr > minsn,
934  np.logical_not(flags & prefs.getFlagMask()))
935  good = np.logical_and(good, elong < maxelong)
936  fwhm = 2.0*rmsSize
937  good = np.logical_and(good, fwhm >= min)
938  good = np.logical_and(good, fwhm < max)
939  fwhms[i] = fwhm[good]
940 
941  if prefs.getVarType() == prefs.VAR_NONE:
942  if nobj:
943  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
944  i = 0
945  for l in fwhms.values():
946  fwhms_all[i:len(l)] = l
947  i += len(l)
948  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
949  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
950  plot=plot)
951  else:
952  raise RuntimeError("No source with appropriate FWHM found!!")
953  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
954 
955  fwhmmin = np.zeros(ncat) + min
956  fwhmmax = np.zeros(ncat) + max
957  fwhmmode = np.zeros(ncat) + mode
958  else:
959  fwhmmode = np.empty(ncat)
960  fwhmmin = np.empty(ncat)
961  fwhmmax = np.empty(ncat)
962 
963  for i in range(ncat):
964  nobj = len(fwhms[i])
965  if (nobj):
966  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
967  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
968  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
969  else:
970  raise RuntimeError("No source with appropriate FWHM found!!")
971  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
972 
973  # Read the samples
974  mode = psfexLib.BIG # mode of FWHM distribution
975 
976  sets = []
977  for i, fileName in enumerate(filenames):
978  set = None
979  for ext in range(next):
980  set = read_samplesLsst(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
981  ext, next, i, context,
982  context.getPc(i) if context.getNpc() else None, plot=plot)
983 
984  if fwhmmode[i] < mode:
985  mode = fwhmmode[i]
986 
987  set.setFwhm(mode)
988 
989  if prefs.getVerboseType() != prefs.QUIET:
990  if set.getNsample():
991  print("%d samples loaded." % set.getNsample())
992  else:
993  raise RuntimeError("No appropriate source found!!")
994 
995  sets.append(set)
996 
997  return sets
998 
999 
1000 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
1001 
1002 def makeit(prefs, context, saveWcs=False, plot=dict()):
1003  """This is the python wrapper for the original psfex that reads SExtractor outputs"""
1004  # Create an array of PSFs (one PSF for each extension)
1005  if prefs.getVerboseType() != prefs.QUIET:
1006  print("----- %d input catalogues:" % prefs.getNcat())
1007 
1008  if saveWcs: # only needed for making plots
1009  wcssList = []
1010 
1011  fields = psfexLib.vectorField()
1012  for cat in prefs.getCatalogs():
1013  field = psfexLib.Field(cat)
1014  wcss = []
1015  wcssList.append(wcss)
1016  with fits.open(cat) as pf:
1017  for hdu in pf:
1018  if hdu.name == "PRIMARY":
1019  pass
1020  elif hdu.name == "LDAC_IMHEAD":
1021  hdr = hdu.data[0][0] # the fits header from the original fits image
1022  md = PropertySet()
1023  for line in hdr:
1024  try:
1025  md.set(*splitFitsCard(line))
1026  except AttributeError:
1027  continue
1028 
1029  if not md.exists("CRPIX1"): # no WCS; try WCSA
1030  for k in md.names():
1031  if re.search(r"A$", k):
1032  md.set(k[:-1], md.getScalar(k))
1033  wcs = afwGeom.makeSkyWcs(md)
1034  naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2")
1035  elif hdu.name == "LDAC_OBJECTS":
1036  nobj = len(hdu.data)
1037 
1038  assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
1039  field.addExt(wcs, naxis1, naxis2, nobj)
1040  if saveWcs:
1041  wcss.append((wcs, naxis1, naxis2))
1042  wcs = None
1043 
1044  field.finalize()
1045  fields.append(field)
1046 
1047  sets = psfexLib.vectorSet()
1048  for set in load_samples(prefs, context, plot=plot):
1049  sets.append(set)
1050 
1051  psfexLib.makeit(fields, sets)
1052 
1053  ret = [[f.getPsfs() for f in fields], sets]
1054  if saveWcs:
1055  ret.append(wcssList)
1056 
1057  return ret
def select_candidates(set, prefs, frmin, frmax, flags, flux, fluxerr, rmsSize, elong, vignet, plot=dict(), title="")
Definition: psfex.py:271
def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:98
def makeit(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:1002
def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5, diagnostics=False, outDir="", frame=None, title=None)
Definition: psfex.py:527
def makeitLsst(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:691
def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False))
Definition: psfex.py:43
def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:749
def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:388
def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:889