lsst.meas.extensions.psfex  14.0-2-ge3897b5+7
psfex.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 from __future__ import absolute_import, division, print_function
3 from builtins import zip
4 from builtins import input
5 from builtins import str
6 from builtins import range
7 from builtins import object
8 import os
9 import re
10 import sys
11 
12 import numpy as np
13 import pyfits
14 try:
15  import matplotlib.pyplot as plt
16 except ImportError:
17  plt = None
18 
19 from lsst.afw.coord import IcrsCoord
20 import lsst.afw.geom as afwGeom
21 from lsst.afw.fits import readMetadata
22 import lsst.afw.image as afwImage
23 import lsst.afw.table as afwTable
24 import lsst.afw.display.ds9 as ds9
25 from lsst.daf.base import PropertySet
26 from . import psfexLib
27 
28 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
29 
30 
31 def splitFitsCard(line):
32  """Split a fits header, returning (key, value)"""
33  try:
34  k, v = re.search(r"(\S+)\s*=\s*'?((?:\S+|'))", line).groups()
35  except AttributeError:
36  raise
37 
38  try:
39  v = int(v)
40  except ValueError:
41  try:
42  v = float(v)
43  except ValueError:
44  pass
45 
46  return k, v
47 
48 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
49 
50 
51 def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False)):
52  """
53  PURPOSE Compute the FWHM range associated to a series of FWHM measurements.
54  INPUT Pointer to an array of FWHMs,
55  maximum allowed FWHM variation,
56  minimum allowed FWHM,
57  maximum allowed FWHM,
58 
59  OUTPUT FWHM mode, lower FWHM range, upper FWHM range
60  NOTES -.
61  AUTHOR E. Bertin (IAP, Leiden observatory & ESO)
62  VERSION 20/03/2008
63  """
64  nfwhm = len(fwhm)
65  fwhm.sort()
66 
67  # Find the mode
68  nw = nfwhm//4
69  if nw < 4:
70  nw = 1
71  dfmin = psfexLib.BIG
72  fmin = 0.0
73  for i in range(nfwhm - nw):
74  df = fwhm[i + nw] - fwhm[i]
75  if df < dfmin:
76  dfmin = df
77  fmin = (fwhm[i + nw] + fwhm[i])/2.0
78 
79  if nfwhm < 2:
80  fmin = fwhm[0]
81 
82  dfmin = (maxvar + 1.0)**0.3333333
83  minout = fmin/dfmin if dfmin > 0.0 else 0.0
84  if minout < minin:
85  minout = minin
86 
87  maxout = fmin*dfmin**2
88  if maxout > maxin:
89  maxout = maxin
90 
91  if plt and plot.get("fwhmHistogram"):
92  plt.clf()
93  plt.hist(fwhm, nfwhm//10 + 1, normed=1, facecolor='g', alpha=0.75)
94  plt.xlabel("FWHM")
95  plt.axvline(fmin, color='red')
96  [plt.axvline(_, color='blue') for _ in (minout, maxout)]
97 
98  input("Continue? ")
99 
100  return fmin, minout, maxout
101 
102 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
103 
104 
105 def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
106  plot=dict(showFlags=False, showRejection=False)):
107  # allocate a new set iff set is None
108  if not set:
109  set = psfexLib.Set(context)
110 
111  cmin, cmax = None, None
112  if set.getNcontext():
113  cmin = np.empty(set.getNcontext())
114  cmax = np.empty(set.getNcontext())
115  for i in range(set.getNcontext()):
116  if set.getNsample():
117  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
118  cmax[i] = cmin[i] + set.getContextScale(i)
119  else:
120  cmin[i] = psfexLib.BIG
121  cmax[i] = -psfexLib.BIG
122  #
123  # Read data
124  #
125  with pyfits.open(filename) as cat:
126  extCtr = -1
127  for tab in cat:
128  if tab.name == "LDAC_IMHEAD":
129  extCtr += 1
130 
131  if extCtr < ext:
132  continue
133  elif extCtr > ext:
134  break
135 
136  if tab.name == "PRIMARY":
137  pass
138  elif tab.name == "LDAC_IMHEAD":
139  hdr = tab.data[0][0] # the fits header from the original fits image
140  foundCards = 0 # how many of our desired cards have we found?
141 
142  for line in hdr:
143  try:
144  k, v = splitFitsCard(line)
145  except AttributeError:
146  continue
147 
148  if k == "SEXBKDEV":
149  backnoise2 = v**2
150  foundCards += 1
151  elif k == "SEXGAIN":
152  gain = v
153  foundCards += 1
154 
155  if foundCards == 2:
156  break
157  elif tab.name == "LDAC_OBJECTS":
158  xm = tab.data[prefs.getCenterKey(0)]
159  ym = tab.data[prefs.getCenterKey(1)]
160  fluxrad = tab.data["FLUX_RADIUS"]
161  flux = tab.data[prefs.getPhotfluxRkey()]
162  fluxerr = tab.data[prefs.getPhotfluxerrRkey()]
163  elong = tab.data["ELONGATION"]
164  flags = tab.data["FLAGS"]
165 
166  n = prefs.getPhotfluxNum() - 1
167  if n:
168  assert False, "Code to handle e.g. FLUX_APER(3) isn't yet converted"
169  if key.naxis == 1 and n < key.naxisn[0]:
170  flux += n
171  else:
172  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
173  (prefs.getPhotfluxRkey(), filename), file=sys.stderr)
174 
175  n = prefs.getPhotfluxerrNum() - 1
176  if n:
177  if key.naxis == 1 and n < key.naxisn[0]:
178  fluxerr += n
179  else:
180  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
181  (prefs.getPhotfluxerrRkey(), filename), file=sys.stderr)
182  #
183  # Now the VIGNET data
184  #
185  vignet = tab.data["VIGNET"]
186 
187  try:
188  vigw, vigh = vignet[0].shape
189  except ValueError:
190  raise RuntimeError("*Error*: VIGNET should be a 2D vector; saw %s" % str(vignet[0].shape))
191 
192  if set.empty():
193  set.setVigSize(vigw, vigh)
194 
195  # Try to load the set of context keys
196  pc = 0
197  contextvalp = []
198  for i, key in enumerate(context.getName()):
199  if context.getPcflag(i):
200  contextvalp.append(pcval[pc])
201  pc += 1
202  elif key[0] == ':':
203  try:
204  contextvalp.append(tab.header[key[1:]])
205  except KeyError:
206  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
207  (key[1:], filename))
208  else:
209  try:
210  contextvalp.append(tab.data[key])
211  except KeyError:
212  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
213  (key, filename))
214  set.setContextname(i, key)
215 
216  # Now examine each vector of the shipment
217  good = select_candidates(set, prefs, frmin, frmax,
218  flags, flux, fluxerr, fluxrad, elong, vignet,
219  plot=plot, title="%s[%d]" % (filename, ext + 1))
220  #
221  # Insert the good candidates into the set
222  #
223  if not vignet.dtype.isnative:
224  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
225  vignet = vignet.byteswap()
226 
227  for i in np.where(good)[0]:
228  sample = set.newSample()
229  sample.setCatindex(catindex)
230  sample.setExtindex(ext)
231 
232  sample.setVig(vignet[i])
233 
234  sample.setNorm(float(flux[i]))
235  sample.setBacknoise2(backnoise2)
236  sample.setGain(gain)
237  sample.setX(float(xm[i]))
238  sample.setY(float(ym[i]))
239  sample.setFluxrad(float(fluxrad[i]))
240 
241  for j in range(set.getNcontext()):
242  sample.setContext(j, float(contextvalp[j][i]))
243 
244  set.finiSample(sample, prefs.getProfAccuracy())
245 
246  #---- Update min and max
247  for j in range(set.getNcontext()):
248  cmin[j] = contextvalp[j][good].min()
249  cmax[j] = contextvalp[j][good].max()
250 
251  # Update the scaling
252  if set.getNsample():
253  for i in range(set.getNcontext()):
254  set.setContextScale(i, cmax[i] - cmin[i])
255  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
256 
257  # Don't waste memory!
258  set.trimMemory()
259 
260  return set
261 
262 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
263 
264 
265 def getSexFlags(*args):
266  return {1: "flux blended",
267  2: "blended",
268  4: "saturated",
269  8: "edge",
270  16: "bad aperture",
271  32: "bad isophotal",
272  64: "memory error (deblend)",
273  128: "memory error (extract)",
274  }
275 
276 
277 def select_candidates(set, prefs, frmin, frmax,
278  flags, flux, fluxerr, rmsSize, elong, vignet,
279  plot=dict(), title=""):
280  maxbad = prefs.getBadpixNmax()
281  maxbadflag = prefs.getBadpixFlag()
282  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100.0
283  minsn = prefs.getMinsn()
284 
285  sn = flux/np.where(fluxerr > 0, fluxerr, 1)
286  sn[fluxerr <= 0] = -psfexLib.BIG
287  #---- Apply some selection over flags, fluxes...
288  plotFlags = plot.get("showFlags") if plt else False
289  plotRejection = plot.get("showRejection") if plt else False
290 
291  bad = flags & prefs.getFlagMask() != 0
292  set.setBadFlags(int(sum(bad != 0)))
293 
294  if plotRejection:
295  selectionVectors = []
296  selectionVectors.append((bad, "flags %d" % sum(bad != 0)))
297 
298  dbad = sn < minsn
299  set.setBadSN(int(sum(dbad)))
300  bad = np.logical_or(bad, dbad)
301  if plotRejection:
302  selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
303 
304  dbad = rmsSize < frmin
305  set.setBadFrmin(int(sum(dbad)))
306  bad = np.logical_or(bad, dbad)
307  if plotRejection:
308  selectionVectors.append((dbad, "frmin %d" % sum(dbad)))
309 
310  dbad = rmsSize > frmax
311  set.setBadFrmax(int(sum(dbad)))
312  bad = np.logical_or(bad, dbad)
313  if plotRejection:
314  selectionVectors.append((dbad, "frmax %d" % sum(dbad)))
315 
316  dbad = elong > maxelong
317  set.setBadElong(int(sum(dbad)))
318  bad = np.logical_or(bad, dbad)
319  if plotRejection:
320  selectionVectors.append((dbad, "elong %d" % sum(dbad)))
321 
322  #-- ... and check the integrity of the sample
323  if maxbadflag:
324  nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
325  dbad = nbad > maxbad
326  set.setBadPix(int(sum(dbad)))
327  bad = np.logical_or(bad, dbad)
328  if plotRejection:
329  selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
330 
331  good = np.logical_not(bad)
332  if plotFlags or plotRejection:
333  imag = -2.5*np.log10(flux)
334  plt.clf()
335 
336  alpha = 0.5
337  if plotFlags:
338  labels = getFlags()
339 
340  isSet = np.where(flags == 0x0)[0]
341  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label="good")
342 
343  for i in range(16):
344  mask = 1 << i
345  if mask & prefs.getFlagMask():
346  isSet = np.where(np.bitwise_and(flags, mask))[0]
347  if isSet.any():
348  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label=labels[mask])
349  else:
350  for bad, label in selectionVectors:
351  plt.plot(imag[bad], rmsSize[bad], 'o', alpha=alpha, label=label)
352 
353  plt.plot(imag[good], rmsSize[good], 'o', color="black", label="selected")
354  [plt.axhline(_, color='red') for _ in [frmin, frmax]]
355  plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
356  plt.ylim(-0.1, 2*frmax)
357  plt.legend(loc=2)
358  plt.xlabel("Instrumental Magnitude")
359  plt.ylabel("rmsSize")
360  plt.title("%s %d selected" % (title, sum(good)))
361 
362  input("Continue? ")
363 
364  return good
365 
366 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
367 
368 try:
369  _dataType
370 except NameError:
371  class _SExtractor(object):
372  pass
373 
374  class _LSST(object):
375  pass
376 
377  _dataTypes = dict(LSST=_LSST,
378  SExtractor=_SExtractor
379  )
380  _dataType = _SExtractor
381 
382 
383 def setDataType(t):
384  _dataType = _dataTypes[t]
385 
386 
387 def getFlags():
388  if _dataType == _LSST:
389  return getLsstFlags(None)
390  else:
391  return getSexFlags(None)
392 
393 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
394 
395 
396 def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
397  minsn = prefs.getMinsn()
398  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
399  min = prefs.getFwhmrange()[0]
400  max = prefs.getFwhmrange()[1]
401 
402  filenames = prefs.getCatalogs()
403 
404  ncat = len(filenames)
405  fwhmmin = np.empty(ncat)
406  fwhmmax = np.empty(ncat)
407 
408  if not prefs.getAutoselectFlag():
409  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
410  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
411  fwhmmode = (fwhmmin + fwhmmax)/2.0
412  else:
413  fwhms = {}
414 
415  #-- Try to estimate the most appropriate Half-light Radius range
416  #-- Get the Half-light radii
417  nobj = 0
418  for i, fileName in enumerate(filenames):
419  fwhms[i] = []
420 
421  if prefs.getVerboseType() != prefs.QUIET:
422  print("Examining Catalog #%d" % (i+1))
423 
424  #---- Read input catalog
425  backnoises = []
426  with pyfits.open(fileName) as cat:
427  extCtr = -1
428  for tab in cat:
429  if tab.name == "LDAC_IMHEAD":
430  extCtr += 1
431 
432  if extCtr != ext and ext != prefs.ALL_EXTENSIONS:
433  if extCtr > ext:
434  break
435  continue
436 
437  if tab.name == "PRIMARY":
438  pass
439  elif tab.name == "LDAC_IMHEAD":
440  hdr = tab.data[0][0] # the fits header from the original fits image
441  for line in hdr:
442  try:
443  k, v = splitFitsCard(line)
444  except AttributeError:
445  continue
446 
447  if k == "SEXBKDEV":
448  if v < 1/psfexLib.BIG:
449  v = 1.0
450 
451  backnoises.append(v)
452  break
453  elif tab.name == "LDAC_OBJECTS":
454  #-------- Fill the FWHM array
455  rmsSize = tab.data["FLUX_RADIUS"]
456  fmax = tab.data["FLUX_MAX"]
457  flags = tab.data["FLAGS"]
458  elong = tab.data["ELONGATION"]
459  backnoise = backnoises[-1]
460 
461  good = np.logical_and(fmax/backnoise > minsn,
462  np.logical_not(flags & prefs.getFlagMask()))
463  good = np.logical_and(good, elong < maxelong)
464  fwhm = 2.0*rmsSize
465  good = np.logical_and(good, fwhm >= min)
466  good = np.logical_and(good, fwhm < max)
467  fwhms[i] = fwhm[good]
468 
469  if prefs.getVarType() == prefs.VAR_NONE:
470  if nobj:
471  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
472  i = 0
473  for l in fwhms.values():
474  fwhms_all[i:len(l)] = l
475  i += len(l)
476  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
477  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
478  plot=plot)
479  else:
480  raise RuntimeError("No source with appropriate FWHM found!!")
481  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
482 
483  fwhmmin = np.zeros(ncat) + min
484  fwhmmax = np.zeros(ncat) + max
485  fwhmmode = np.zeros(ncat) + mode
486  else:
487  fwhmmode = np.empty(ncat)
488  fwhmmin = np.empty(ncat)
489  fwhmmax = np.empty(ncat)
490 
491  for i in range(ncat):
492  nobj = len(fwhms[i])
493  if (nobj):
494  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
495  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
496  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
497  else:
498  raise RuntimeError("No source with appropriate FWHM found!!")
499  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
500 
501  # Read the samples
502  mode = psfexLib.BIG # mode of FWHM distribution
503 
504  sets = []
505  for i, fileName in enumerate(filenames):
506  set = None
507  if ext == prefs.ALL_EXTENSIONS:
508  extensions = list(range(len(backnoises)))
509  else:
510  extensions = [ext]
511 
512  for e in extensions:
513  set = read_samples(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
514  e, next, i, context, context.getPc(i) if context.getNpc() else None, plot=plot)
515 
516  if fwhmmode[i] < mode:
517  mode = fwhmmode[i]
518 
519  set.setFwhm(mode)
520 
521  if prefs.getVerboseType() != prefs.QUIET:
522  if set.getNsample():
523  print("%d samples loaded." % set.getNsample())
524  else:
525  raise RuntimeError("No appropriate source found!!")
526 
527  sets.append(set)
528 
529  return sets
530 
531 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
532 
533 
534 def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5,
535  diagnostics=False, outDir="", frame=None, title=None):
536  """Show a PSF on ds9"""
537 
538  if ext is not None:
539  psf = psf[ext]
540 
541  if wcsData:
542  if ext is not None:
543  wcsData = wcsData[ext]
544  wcs, naxis1, naxis2 = wcsData
545  else:
546  wcs, naxis1, naxis2 = None, None, None
547 
548  naxis = [naxis1, naxis2]
549  for i in range(2):
550  if naxis[i] is None:
551  # cmin, cmax are the range of input star positions
552  cmin, cmax = [set.getContextOffset(i) + d*set.getContextScale(i) for d in (-0.5, 0.5)]
553  naxis[i] = cmax + cmin # a decent guess
554 
555  import lsst.afw.display.utils as ds9Utils
556  if naxis[0] > naxis[1]:
557  nx, ny = int(nspot*naxis[0]/float(naxis[1]) + 0.5), nspot
558  else:
559  nx, ny = nspot, int(nspot*naxis[1]/float(naxis[0]) + 0.5)
560 
561  mos = ds9Utils.Mosaic(gutter=2, background=0.02)
562 
563  xpos, ypos = np.linspace(0, naxis[0], nx), np.linspace(0, naxis[1], ny)
564  for y in ypos:
565  for x in xpos:
566  psf.build(x, y)
567 
568  im = afwImage.ImageF(*psf.getLoc().shape)
569  im.getArray()[:] = psf.getLoc()
570  im /= float(im.getArray().max())
571  if trim:
572  if trim > im.getHeight()//2:
573  trim = im.getHeight()//2
574 
575  im = im[trim:-trim, trim:-trim]
576 
577  mos.append(im)
578 
579  mosaic = mos.makeMosaic(mode=nx)
580  if frame is not None:
581  ds9.mtv(mosaic, frame=frame, title=title)
582  #
583  # Figure out the WCS for the mosaic
584  #
585  pos = []
586  if False:
587  for x, y, i in zip((xpos[0], xpos[-1]), (ypos[0], ypos[-1]), (0, mos.nImage - 1)):
588  bbox = mos.getBBox(i)
589  mosx = bbox.getMinX() + 0.5*(bbox.getWidth() - 1)
590  mosy = bbox.getMinY() + 0.5*(bbox.getHeight() - 1)
591  pos.append([afwGeom.PointD(mosx, mosy), wcs.pixelToSky(afwGeom.PointD(x, y))])
592  else:
593  pos.append([afwGeom.PointD(0, 0), wcs.pixelToSky(afwGeom.PointD(0, 0))])
594  pos.append([afwGeom.PointD(*mosaic.getDimensions()), wcs.pixelToSky(afwGeom.PointD(naxis1, naxis2))])
595 
596  CD = []
597  for i in range(2):
598  delta = pos[1][1][i].asDegrees() - pos[0][1][i].asDegrees()
599  CD.append(delta/(pos[1][0][i] - pos[0][0][i]))
600  CD = np.array(CD)
601  CD.shape = (2, 2)
602  mosWcs = afwGeom.makeSkyWcs(crval=pos[0][0], crpix=pos[0][1], cdMatrix=CD)
603 
604  if ext is not None:
605  title = "%s-%d" % (title, ext)
606 
607  if frame is not None:
608  ds9.mtv(mosaic, frame=frame, title=title, wcs=mosWcs)
609 
610  if diagnostics:
611  outFile = "%s-mod.fits" % title
612  if outDir:
613  outFile = os.path.join(outDir, outFile)
614  mosaic.writeFits(outFile, mosWcs.getFitsMetadata())
615 
616  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
617 
618  mos = ds9Utils.Mosaic(gutter=4, background=0.002)
619  for i in range(set.getNsample()):
620  s = set.getSample(i)
621  if ext is not None and s.getExtindex() != ext:
622  continue
623 
624  smos = ds9Utils.Mosaic(gutter=2, background=-0.003)
625  for func in [s.getVig, s.getVigResi]:
626  arr = func()
627  if func == s.getVig:
628  norm = float(arr.max()) if True else s.getNorm()
629 
630  arr /= norm
631  im = afwImage.ImageF(*arr.shape)
632  im.getArray()[:] = arr
633  smos.append(im)
634 
635  mos.append(smos.makeMosaic(mode="x"))
636 
637  mosaic = mos.makeMosaic(title=title)
638 
639  if frame is not None:
640  ds9.mtv(mosaic, title=title, frame=frame+1)
641 
642  if diagnostics:
643  outFile = "%s-psfstars.fits" % title
644  if outDir:
645  outFile = os.path.join(outDir, outFile)
646 
647  mosaic.writeFits(outFile)
648 
649 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
650 
651 
652 def getLsstFlags(tab=None):
653  flagKeys = [
654  "base_PixelFlags_flag_edge",
655  #"base_PixelFlags_flag_interpolated",
656  #"base_PixelFlags_flag_interpolatedCenter",
657  #"base_PixelFlags_flag_saturated",
658  "base_PixelFlags_flag_saturatedCenter",
659  #"base_PixelFlags_flag_cr",
660  "base_PixelFlags_flag_crCenter",
661  "base_PixelFlags_flag_bad",
662  "base_PsfFlux_flag",
663  "parent",
664  ]
665 
666  if tab is None:
667  flags = {}
668  for i, k in enumerate(flagKeys):
669  flags[1 << i] = re.sub(r"\_flag", "",
670  re.sub(r"^base\_", "", re.sub(r"^base\_PixelFlags\_flag\_", "", k)))
671  else:
672  flags = 0
673  for i, k in enumerate(flagKeys):
674  if k == "parent":
675  try:
676  isSet = tab.get("deblend_nChild") > 0
677  except KeyError:
678  isSet = 0
679  else:
680  isSet = tab.get(k)
681  flags = np.bitwise_or(flags, np.where(isSet, 1 << i, 0))
682 
683  return flags
684 
685 
686 def guessCalexp(fileName):
687  for guess in [
688  re.sub("/src", r"", fileName),
689  re.sub("(SRC([^.]+))", r"CORR\2-exp", fileName),
690  ]:
691  if guess != fileName and os.path.exists(guess):
692  return guess
693 
694  raise RuntimeError("Unable to find a calexp to go with %s" % fileName)
695 
696 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
697 
698 
699 def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
700  """This is the python wrapper that reads lsst tables"""
701  # Create an array of PSFs (one PSF for each extension)
702  if prefs.getVerboseType() != prefs.QUIET:
703  print("----- %d input catalogues:" % prefs.getNcat())
704 
705  if saveWcs: # only needed for making plots
706  wcssList = []
707 
708  fields = psfexLib.vectorField()
709  for cat in prefs.getCatalogs():
710  field = psfexLib.Field(cat)
711  wcss = []
712  wcssList.append(wcss)
713  with pyfits.open(cat):
714  # Hack: I want the WCS so I'll guess where the calexp is to be found
715  calexpFile = guessCalexp(cat)
716  md = readMetadata(calexpFile)
717  wcs = afwGeom.makeSkyWcs(md)
718 
719  if not wcs:
720  cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
721  cdMatrix.shape = (2, 2)
722  wcs = afwGeom.makeSkyWcs(crpix=afwGeom.PointD(0, 0),
723  crval=IcrsCoord(0.0*afwGeom.degrees, 0.0*afwGeom.degrees),
724  cdMatrix=cdMatrix)
725 
726  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
727  # Find how many rows there are in the catalogue
728  md = readMetadata(cat)
729 
730  field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2"))
731  if saveWcs:
732  wcss.append((wcs, naxis1, naxis2))
733 
734  field.finalize()
735  fields.append(field)
736 
737  fields[0].getNext() # number of extensions
738 
739  prefs.getPsfStep()
740 
741  sets = psfexLib.vectorSet()
742  for set in load_samplesLsst(prefs, context, plot=plot):
743  sets.append(set)
744 
745  psfexLib.makeit(fields, sets)
746 
747  ret = [[f.getPsfs() for f in fields], sets]
748  if saveWcs:
749  ret.append(wcssList)
750 
751  return ret
752 
753 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
754 
755 
756 def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
757  plot=dict(showFlags=False, showRejection=False)):
758  # allocate a new set iff set is None
759  if not set:
760  set = psfexLib.Set(context)
761 
762  cmin, cmax = None, None
763  if set.getNcontext():
764  cmin = np.empty(set.getNcontext())
765  cmax = np.empty(set.getNcontext())
766  for i in range(set.getNcontext()):
767  if set.getNsample():
768  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
769  cmax[i] = cmin[i] + set.getContextScale(i)
770  else:
771  cmin[i] = psfexLib.BIG
772  cmax[i] = -psfexLib.BIG
773  #
774  # Read data
775  #
776  tab = afwTable.SourceCatalog.readFits(filename)
777 
778  centroid = tab.getCentroidDefinition()
779  xm = tab.get("%s.x" % centroid)
780  ym = tab.get("%s.y" % centroid)
781 
782  shape = tab.getShapeDefinition()
783  ixx = tab.get("%s.xx" % shape)
784  iyy = tab.get("%s.yy" % shape)
785 
786  rmsSize = np.sqrt(0.5*(ixx + iyy))
787  elong = 0.5*(ixx - iyy)/(ixx + iyy)
788 
789  flux = tab.get(prefs.getPhotfluxRkey())
790  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
791  flags = getLsstFlags(tab)
792 
793  #
794  # Now the VIGNET data
795  #
796  vigw, vigh = 35, 35 # [prefs.getPsfsize()[i] for i in range(2)]
797  if set.empty():
798  set.setVigSize(vigw, vigh)
799 
800  vignet = np.empty(nobj*vigw*vigh, "float32").reshape(nobj, vigw, vigh)
801 
802  # Hack: I want the WCS so I'll guess where the calexp is to be found
803  calexpFile = guessCalexp(filename)
804  mi = afwImage.MaskedImageF(calexpFile)
805  backnoise2 = np.median(mi.getVariance().getArray())
806  gain = 1.0
807 
808  edgeBit = [k for k, v in getFlags().items() if v == "edge"][0]
809 
810  for i, xc, yc in zip(range(nobj), xm, ym):
811  try:
812  x, y = int(xc), int(yc)
813  except ValueError:
814  flags[i] |= edgeBit # mark star as bad
815 
816  try:
817  pstamp = mi[x - vigw//2:x + vigw//2 + 1, y - vigh//2:y + vigh//2 + 1]
818  vignet[i] = pstamp.getImage().getArray().transpose()
819  except Exception:
820  flags[i] |= edgeBit # mark star as bad
821 
822  # Try to load the set of context keys
823  pc = 0
824  contextvalp = []
825  for i, key in enumerate(context.getName()):
826  if context.getPcflag(i):
827  contextvalp.append(pcval[pc])
828  pc += 1
829  elif key[0] == ':':
830  try:
831  contextvalp.append(tab.header[key[1:]])
832  except KeyError:
833  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
834  (key[1:], filename))
835  else:
836  try:
837  contextvalp.append(tab.get(key))
838  except KeyError:
839  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
840  (key, filename))
841  set.setContextname(i, key)
842 
843  # Now examine each vector of the shipment
844  good = select_candidates(set, prefs, frmin, frmax,
845  flags, flux, fluxErr, rmsSize, elong, vignet,
846  plot=plot, title="%s[%d]" % (filename, ext + 1) if next > 1 else filename)
847  #
848  # Insert the good candidates into the set
849  #
850  if not vignet.dtype.isnative:
851  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
852  vignet = vignet.byteswap()
853 
854  for i in np.where(good)[0]:
855  sample = set.newSample()
856  sample.setCatindex(catindex)
857  sample.setExtindex(ext)
858 
859  sample.setVig(vignet[i])
860  if False:
861  pstamp = afwImage.ImageF(*vignet[i].shape)
862  pstamp.getArray()[:] = sample.getVig()
863 
864  ds9.mtv(pstamp)
865 
866  sample.setNorm(float(flux[i]))
867  sample.setBacknoise2(backnoise2)
868  sample.setGain(gain)
869  sample.setX(float(xm[i]))
870  sample.setY(float(ym[i]))
871  sample.setFluxrad(float(rmsSize[i]))
872 
873  for j in range(set.getNcontext()):
874  sample.setContext(j, float(contextvalp[j][i]))
875 
876  set.finiSample(sample, prefs.getProfAccuracy())
877 
878  #---- Update min and max
879  for j in range(set.getNcontext()):
880  cmin[j] = contextvalp[j][good].min()
881  cmax[j] = contextvalp[j][good].max()
882 
883  # Update the scaling
884  if set.getNsample():
885  for i in range(set.getNcontext()):
886  set.setContextScale(i, cmax[i] - cmin[i])
887  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
888 
889  # Don't waste memory!
890  set.trimMemory()
891 
892  return set
893 
894 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
895 
896 
897 def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
898  minsn = prefs.getMinsn()
899  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
900  min = prefs.getFwhmrange()[0]
901  max = prefs.getFwhmrange()[1]
902 
903  filenames = prefs.getCatalogs()
904 
905  ncat = len(filenames)
906  fwhmmin = np.empty(ncat)
907  fwhmmax = np.empty(ncat)
908 
909  if not prefs.getAutoselectFlag():
910  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
911  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
912  fwhmmode = (fwhmmin + fwhmmax)/2.0
913  else:
914  fwhms = {}
915 
916  #-- Try to estimate the most appropriate Half-light Radius range
917  #-- Get the Half-light radii
918  nobj = 0
919  for i, fileName in enumerate(filenames):
920  fwhms[i] = []
921 
922  if prefs.getVerboseType() != prefs.QUIET:
923  print("Examining Catalog #%d" % (i+1))
924 
925  #---- Read input catalog
926  tab = afwTable.SourceCatalog.readFits(fileName)
927 
928  #-------- Fill the FWHM array
929  shape = tab.getShapeDefinition()
930  ixx = tab.get("%s.xx" % shape)
931  iyy = tab.get("%s.yy" % shape)
932 
933  rmsSize = np.sqrt(0.5*(ixx + iyy))
934  elong = 0.5*(ixx - iyy)/(ixx + iyy)
935 
936  flux = tab.get(prefs.getPhotfluxRkey())
937  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
938 
939  flags = getLsstFlags(tab)
940 
941  good = np.logical_and(flux/fluxErr > minsn,
942  np.logical_not(flags & prefs.getFlagMask()))
943  good = np.logical_and(good, elong < maxelong)
944  fwhm = 2.0*rmsSize
945  good = np.logical_and(good, fwhm >= min)
946  good = np.logical_and(good, fwhm < max)
947  fwhms[i] = fwhm[good]
948 
949  if prefs.getVarType() == prefs.VAR_NONE:
950  if nobj:
951  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
952  i = 0
953  for l in fwhms.values():
954  fwhms_all[i:len(l)] = l
955  i += len(l)
956  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
957  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
958  plot=plot)
959  else:
960  raise RuntimeError("No source with appropriate FWHM found!!")
961  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
962 
963  fwhmmin = np.zeros(ncat) + min
964  fwhmmax = np.zeros(ncat) + max
965  fwhmmode = np.zeros(ncat) + mode
966  else:
967  fwhmmode = np.empty(ncat)
968  fwhmmin = np.empty(ncat)
969  fwhmmax = np.empty(ncat)
970 
971  for i in range(ncat):
972  nobj = len(fwhms[i])
973  if (nobj):
974  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
975  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
976  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
977  else:
978  raise RuntimeError("No source with appropriate FWHM found!!")
979  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
980 
981  # Read the samples
982  mode = psfexLib.BIG # mode of FWHM distribution
983 
984  sets = []
985  for i, fileName in enumerate(filenames):
986  set = None
987  for ext in range(next):
988  set = read_samplesLsst(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
989  ext, next, i, context,
990  context.getPc(i) if context.getNpc() else None, plot=plot)
991 
992  if fwhmmode[i] < mode:
993  mode = fwhmmode[i]
994 
995  set.setFwhm(mode)
996 
997  if prefs.getVerboseType() != prefs.QUIET:
998  if set.getNsample():
999  print("%d samples loaded." % set.getNsample())
1000  else:
1001  raise RuntimeError("No appropriate source found!!")
1002 
1003  sets.append(set)
1004 
1005  return sets
1006 
1007 
1008 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
1009 
1010 def makeit(prefs, context, saveWcs=False, plot=dict()):
1011  """This is the python wrapper for the original psfex that reads SExtractor outputs"""
1012  # Create an array of PSFs (one PSF for each extension)
1013  if prefs.getVerboseType() != prefs.QUIET:
1014  print("----- %d input catalogues:" % prefs.getNcat())
1015 
1016  if saveWcs: # only needed for making plots
1017  wcssList = []
1018 
1019  fields = psfexLib.vectorField()
1020  for cat in prefs.getCatalogs():
1021  field = psfexLib.Field(cat)
1022  wcss = []
1023  wcssList.append(wcss)
1024  with pyfits.open(cat) as pf:
1025  for hdu in pf:
1026  if hdu.name == "PRIMARY":
1027  pass
1028  elif hdu.name == "LDAC_IMHEAD":
1029  hdr = hdu.data[0][0] # the fits header from the original fits image
1030  md = PropertySet()
1031  for line in hdr:
1032  try:
1033  md.set(*splitFitsCard(line))
1034  except AttributeError:
1035  continue
1036 
1037  if not md.exists("CRPIX1"): # no WCS; try WCSA
1038  for k in md.names():
1039  if re.search(r"A$", k):
1040  md.set(k[:-1], md.get(k))
1041  wcs = afwGeom.makeSkyWcs(md)
1042  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
1043  elif hdu.name == "LDAC_OBJECTS":
1044  nobj = len(hdu.data)
1045 
1046  assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
1047  field.addExt(wcs, naxis1, naxis2, nobj)
1048  if saveWcs:
1049  wcss.append((wcs, naxis1, naxis2))
1050  wcs = None
1051 
1052  field.finalize()
1053  fields.append(field)
1054 
1055  sets = psfexLib.vectorSet()
1056  for set in load_samples(prefs, context, plot=plot):
1057  sets.append(set)
1058 
1059  psfexLib.makeit(fields, sets)
1060 
1061  ret = [[f.getPsfs() for f in fields], sets]
1062  if saveWcs:
1063  ret.append(wcssList)
1064 
1065  return ret
def select_candidates(set, prefs, frmin, frmax, flags, flux, fluxerr, rmsSize, elong, vignet, plot=dict(), title="")
Definition: psfex.py:279
def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:106
def makeit(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:1010
def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5, diagnostics=False, outDir="", frame=None, title=None)
Definition: psfex.py:535
def makeitLsst(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:699
def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False))
Definition: psfex.py:51
def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:757
def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:396
def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:897