lsst.meas.extensions.psfex  14.0-3-g82223af+5
psfex.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 from __future__ import absolute_import, division, print_function
3 from builtins import zip
4 from builtins import input
5 from builtins import str
6 from builtins import range
7 from builtins import object
8 import os
9 import re
10 import sys
11 
12 import numpy as np
13 import pyfits
14 try:
15  import matplotlib.pyplot as plt
16 except ImportError:
17  plt = None
18 
19 import lsst.afw.geom as afwGeom
20 from lsst.afw.fits import readMetadata
21 import lsst.afw.image as afwImage
22 import lsst.afw.table as afwTable
23 import lsst.afw.display.ds9 as ds9
24 from lsst.daf.base import PropertySet
25 from . import psfexLib
26 
27 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
28 
29 
30 def splitFitsCard(line):
31  """Split a fits header, returning (key, value)"""
32  try:
33  k, v = re.search(r"(\S+)\s*=\s*'?((?:\S+|'))", line).groups()
34  except AttributeError:
35  raise
36 
37  try:
38  v = int(v)
39  except ValueError:
40  try:
41  v = float(v)
42  except ValueError:
43  pass
44 
45  return k, v
46 
47 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
48 
49 
50 def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False)):
51  """
52  PURPOSE Compute the FWHM range associated to a series of FWHM measurements.
53  INPUT Pointer to an array of FWHMs,
54  maximum allowed FWHM variation,
55  minimum allowed FWHM,
56  maximum allowed FWHM,
57 
58  OUTPUT FWHM mode, lower FWHM range, upper FWHM range
59  NOTES -.
60  AUTHOR E. Bertin (IAP, Leiden observatory & ESO)
61  VERSION 20/03/2008
62  """
63  nfwhm = len(fwhm)
64  fwhm.sort()
65 
66  # Find the mode
67  nw = nfwhm//4
68  if nw < 4:
69  nw = 1
70  dfmin = psfexLib.BIG
71  fmin = 0.0
72  for i in range(nfwhm - nw):
73  df = fwhm[i + nw] - fwhm[i]
74  if df < dfmin:
75  dfmin = df
76  fmin = (fwhm[i + nw] + fwhm[i])/2.0
77 
78  if nfwhm < 2:
79  fmin = fwhm[0]
80 
81  dfmin = (maxvar + 1.0)**0.3333333
82  minout = fmin/dfmin if dfmin > 0.0 else 0.0
83  if minout < minin:
84  minout = minin
85 
86  maxout = fmin*dfmin**2
87  if maxout > maxin:
88  maxout = maxin
89 
90  if plt and plot.get("fwhmHistogram"):
91  plt.clf()
92  plt.hist(fwhm, nfwhm//10 + 1, normed=1, facecolor='g', alpha=0.75)
93  plt.xlabel("FWHM")
94  plt.axvline(fmin, color='red')
95  [plt.axvline(_, color='blue') for _ in (minout, maxout)]
96 
97  input("Continue? ")
98 
99  return fmin, minout, maxout
100 
101 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
102 
103 
104 def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
105  plot=dict(showFlags=False, showRejection=False)):
106  # allocate a new set iff set is None
107  if not set:
108  set = psfexLib.Set(context)
109 
110  cmin, cmax = None, None
111  if set.getNcontext():
112  cmin = np.empty(set.getNcontext())
113  cmax = np.empty(set.getNcontext())
114  for i in range(set.getNcontext()):
115  if set.getNsample():
116  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
117  cmax[i] = cmin[i] + set.getContextScale(i)
118  else:
119  cmin[i] = psfexLib.BIG
120  cmax[i] = -psfexLib.BIG
121  #
122  # Read data
123  #
124  with pyfits.open(filename) as cat:
125  extCtr = -1
126  for tab in cat:
127  if tab.name == "LDAC_IMHEAD":
128  extCtr += 1
129 
130  if extCtr < ext:
131  continue
132  elif extCtr > ext:
133  break
134 
135  if tab.name == "PRIMARY":
136  pass
137  elif tab.name == "LDAC_IMHEAD":
138  hdr = tab.data[0][0] # the fits header from the original fits image
139  foundCards = 0 # how many of our desired cards have we found?
140 
141  for line in hdr:
142  try:
143  k, v = splitFitsCard(line)
144  except AttributeError:
145  continue
146 
147  if k == "SEXBKDEV":
148  backnoise2 = v**2
149  foundCards += 1
150  elif k == "SEXGAIN":
151  gain = v
152  foundCards += 1
153 
154  if foundCards == 2:
155  break
156  elif tab.name == "LDAC_OBJECTS":
157  xm = tab.data[prefs.getCenterKey(0)]
158  ym = tab.data[prefs.getCenterKey(1)]
159  fluxrad = tab.data["FLUX_RADIUS"]
160  flux = tab.data[prefs.getPhotfluxRkey()]
161  fluxerr = tab.data[prefs.getPhotfluxerrRkey()]
162  elong = tab.data["ELONGATION"]
163  flags = tab.data["FLAGS"]
164 
165  n = prefs.getPhotfluxNum() - 1
166  if n:
167  assert False, "Code to handle e.g. FLUX_APER(3) isn't yet converted"
168  if key.naxis == 1 and n < key.naxisn[0]:
169  flux += n
170  else:
171  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
172  (prefs.getPhotfluxRkey(), filename), file=sys.stderr)
173 
174  n = prefs.getPhotfluxerrNum() - 1
175  if n:
176  if key.naxis == 1 and n < key.naxisn[0]:
177  fluxerr += n
178  else:
179  print("Not enough apertures for %s in catalogue %s: using first aperture" % \
180  (prefs.getPhotfluxerrRkey(), filename), file=sys.stderr)
181  #
182  # Now the VIGNET data
183  #
184  vignet = tab.data["VIGNET"]
185 
186  try:
187  vigw, vigh = vignet[0].shape
188  except ValueError:
189  raise RuntimeError("*Error*: VIGNET should be a 2D vector; saw %s" % str(vignet[0].shape))
190 
191  if set.empty():
192  set.setVigSize(vigw, vigh)
193 
194  # Try to load the set of context keys
195  pc = 0
196  contextvalp = []
197  for i, key in enumerate(context.getName()):
198  if context.getPcflag(i):
199  contextvalp.append(pcval[pc])
200  pc += 1
201  elif key[0] == ':':
202  try:
203  contextvalp.append(tab.header[key[1:]])
204  except KeyError:
205  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
206  (key[1:], filename))
207  else:
208  try:
209  contextvalp.append(tab.data[key])
210  except KeyError:
211  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
212  (key, filename))
213  set.setContextname(i, key)
214 
215  # Now examine each vector of the shipment
216  good = select_candidates(set, prefs, frmin, frmax,
217  flags, flux, fluxerr, fluxrad, elong, vignet,
218  plot=plot, title="%s[%d]" % (filename, ext + 1))
219  #
220  # Insert the good candidates into the set
221  #
222  if not vignet.dtype.isnative:
223  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
224  vignet = vignet.byteswap()
225 
226  for i in np.where(good)[0]:
227  sample = set.newSample()
228  sample.setCatindex(catindex)
229  sample.setExtindex(ext)
230 
231  sample.setVig(vignet[i])
232 
233  sample.setNorm(float(flux[i]))
234  sample.setBacknoise2(backnoise2)
235  sample.setGain(gain)
236  sample.setX(float(xm[i]))
237  sample.setY(float(ym[i]))
238  sample.setFluxrad(float(fluxrad[i]))
239 
240  for j in range(set.getNcontext()):
241  sample.setContext(j, float(contextvalp[j][i]))
242 
243  set.finiSample(sample, prefs.getProfAccuracy())
244 
245  #---- Update min and max
246  for j in range(set.getNcontext()):
247  cmin[j] = contextvalp[j][good].min()
248  cmax[j] = contextvalp[j][good].max()
249 
250  # Update the scaling
251  if set.getNsample():
252  for i in range(set.getNcontext()):
253  set.setContextScale(i, cmax[i] - cmin[i])
254  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
255 
256  # Don't waste memory!
257  set.trimMemory()
258 
259  return set
260 
261 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
262 
263 
264 def getSexFlags(*args):
265  return {1: "flux blended",
266  2: "blended",
267  4: "saturated",
268  8: "edge",
269  16: "bad aperture",
270  32: "bad isophotal",
271  64: "memory error (deblend)",
272  128: "memory error (extract)",
273  }
274 
275 
276 def select_candidates(set, prefs, frmin, frmax,
277  flags, flux, fluxerr, rmsSize, elong, vignet,
278  plot=dict(), title=""):
279  maxbad = prefs.getBadpixNmax()
280  maxbadflag = prefs.getBadpixFlag()
281  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100.0
282  minsn = prefs.getMinsn()
283 
284  sn = flux/np.where(fluxerr > 0, fluxerr, 1)
285  sn[fluxerr <= 0] = -psfexLib.BIG
286  #---- Apply some selection over flags, fluxes...
287  plotFlags = plot.get("showFlags") if plt else False
288  plotRejection = plot.get("showRejection") if plt else False
289 
290  bad = flags & prefs.getFlagMask() != 0
291  set.setBadFlags(int(sum(bad != 0)))
292 
293  if plotRejection:
294  selectionVectors = []
295  selectionVectors.append((bad, "flags %d" % sum(bad != 0)))
296 
297  dbad = sn < minsn
298  set.setBadSN(int(sum(dbad)))
299  bad = np.logical_or(bad, dbad)
300  if plotRejection:
301  selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
302 
303  dbad = rmsSize < frmin
304  set.setBadFrmin(int(sum(dbad)))
305  bad = np.logical_or(bad, dbad)
306  if plotRejection:
307  selectionVectors.append((dbad, "frmin %d" % sum(dbad)))
308 
309  dbad = rmsSize > frmax
310  set.setBadFrmax(int(sum(dbad)))
311  bad = np.logical_or(bad, dbad)
312  if plotRejection:
313  selectionVectors.append((dbad, "frmax %d" % sum(dbad)))
314 
315  dbad = elong > maxelong
316  set.setBadElong(int(sum(dbad)))
317  bad = np.logical_or(bad, dbad)
318  if plotRejection:
319  selectionVectors.append((dbad, "elong %d" % sum(dbad)))
320 
321  #-- ... and check the integrity of the sample
322  if maxbadflag:
323  nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
324  dbad = nbad > maxbad
325  set.setBadPix(int(sum(dbad)))
326  bad = np.logical_or(bad, dbad)
327  if plotRejection:
328  selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
329 
330  good = np.logical_not(bad)
331  if plotFlags or plotRejection:
332  imag = -2.5*np.log10(flux)
333  plt.clf()
334 
335  alpha = 0.5
336  if plotFlags:
337  labels = getFlags()
338 
339  isSet = np.where(flags == 0x0)[0]
340  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label="good")
341 
342  for i in range(16):
343  mask = 1 << i
344  if mask & prefs.getFlagMask():
345  isSet = np.where(np.bitwise_and(flags, mask))[0]
346  if isSet.any():
347  plt.plot(imag[isSet], rmsSize[isSet], 'o', alpha=alpha, label=labels[mask])
348  else:
349  for bad, label in selectionVectors:
350  plt.plot(imag[bad], rmsSize[bad], 'o', alpha=alpha, label=label)
351 
352  plt.plot(imag[good], rmsSize[good], 'o', color="black", label="selected")
353  [plt.axhline(_, color='red') for _ in [frmin, frmax]]
354  plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
355  plt.ylim(-0.1, 2*frmax)
356  plt.legend(loc=2)
357  plt.xlabel("Instrumental Magnitude")
358  plt.ylabel("rmsSize")
359  plt.title("%s %d selected" % (title, sum(good)))
360 
361  input("Continue? ")
362 
363  return good
364 
365 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
366 
367 try:
368  _dataType
369 except NameError:
370  class _SExtractor(object):
371  pass
372 
373  class _LSST(object):
374  pass
375 
376  _dataTypes = dict(LSST=_LSST,
377  SExtractor=_SExtractor
378  )
379  _dataType = _SExtractor
380 
381 
382 def setDataType(t):
383  _dataType = _dataTypes[t]
384 
385 
386 def getFlags():
387  if _dataType == _LSST:
388  return getLsstFlags(None)
389  else:
390  return getSexFlags(None)
391 
392 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
393 
394 
395 def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
396  minsn = prefs.getMinsn()
397  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
398  min = prefs.getFwhmrange()[0]
399  max = prefs.getFwhmrange()[1]
400 
401  filenames = prefs.getCatalogs()
402 
403  ncat = len(filenames)
404  fwhmmin = np.empty(ncat)
405  fwhmmax = np.empty(ncat)
406 
407  if not prefs.getAutoselectFlag():
408  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
409  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
410  fwhmmode = (fwhmmin + fwhmmax)/2.0
411  else:
412  fwhms = {}
413 
414  #-- Try to estimate the most appropriate Half-light Radius range
415  #-- Get the Half-light radii
416  nobj = 0
417  for i, fileName in enumerate(filenames):
418  fwhms[i] = []
419 
420  if prefs.getVerboseType() != prefs.QUIET:
421  print("Examining Catalog #%d" % (i+1))
422 
423  #---- Read input catalog
424  backnoises = []
425  with pyfits.open(fileName) as cat:
426  extCtr = -1
427  for tab in cat:
428  if tab.name == "LDAC_IMHEAD":
429  extCtr += 1
430 
431  if extCtr != ext and ext != prefs.ALL_EXTENSIONS:
432  if extCtr > ext:
433  break
434  continue
435 
436  if tab.name == "PRIMARY":
437  pass
438  elif tab.name == "LDAC_IMHEAD":
439  hdr = tab.data[0][0] # the fits header from the original fits image
440  for line in hdr:
441  try:
442  k, v = splitFitsCard(line)
443  except AttributeError:
444  continue
445 
446  if k == "SEXBKDEV":
447  if v < 1/psfexLib.BIG:
448  v = 1.0
449 
450  backnoises.append(v)
451  break
452  elif tab.name == "LDAC_OBJECTS":
453  #-------- Fill the FWHM array
454  rmsSize = tab.data["FLUX_RADIUS"]
455  fmax = tab.data["FLUX_MAX"]
456  flags = tab.data["FLAGS"]
457  elong = tab.data["ELONGATION"]
458  backnoise = backnoises[-1]
459 
460  good = np.logical_and(fmax/backnoise > minsn,
461  np.logical_not(flags & prefs.getFlagMask()))
462  good = np.logical_and(good, elong < maxelong)
463  fwhm = 2.0*rmsSize
464  good = np.logical_and(good, fwhm >= min)
465  good = np.logical_and(good, fwhm < max)
466  fwhms[i] = fwhm[good]
467 
468  if prefs.getVarType() == prefs.VAR_NONE:
469  if nobj:
470  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
471  i = 0
472  for l in fwhms.values():
473  fwhms_all[i:len(l)] = l
474  i += len(l)
475  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
476  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
477  plot=plot)
478  else:
479  raise RuntimeError("No source with appropriate FWHM found!!")
480  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
481 
482  fwhmmin = np.zeros(ncat) + min
483  fwhmmax = np.zeros(ncat) + max
484  fwhmmode = np.zeros(ncat) + mode
485  else:
486  fwhmmode = np.empty(ncat)
487  fwhmmin = np.empty(ncat)
488  fwhmmax = np.empty(ncat)
489 
490  for i in range(ncat):
491  nobj = len(fwhms[i])
492  if (nobj):
493  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
494  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
495  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
496  else:
497  raise RuntimeError("No source with appropriate FWHM found!!")
498  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
499 
500  # Read the samples
501  mode = psfexLib.BIG # mode of FWHM distribution
502 
503  sets = []
504  for i, fileName in enumerate(filenames):
505  set = None
506  if ext == prefs.ALL_EXTENSIONS:
507  extensions = list(range(len(backnoises)))
508  else:
509  extensions = [ext]
510 
511  for e in extensions:
512  set = read_samples(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
513  e, next, i, context, context.getPc(i) if context.getNpc() else None, plot=plot)
514 
515  if fwhmmode[i] < mode:
516  mode = fwhmmode[i]
517 
518  set.setFwhm(mode)
519 
520  if prefs.getVerboseType() != prefs.QUIET:
521  if set.getNsample():
522  print("%d samples loaded." % set.getNsample())
523  else:
524  raise RuntimeError("No appropriate source found!!")
525 
526  sets.append(set)
527 
528  return sets
529 
530 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
531 
532 
533 def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5,
534  diagnostics=False, outDir="", frame=None, title=None):
535  """Show a PSF on ds9"""
536 
537  if ext is not None:
538  psf = psf[ext]
539 
540  if wcsData:
541  if ext is not None:
542  wcsData = wcsData[ext]
543  wcs, naxis1, naxis2 = wcsData
544  else:
545  wcs, naxis1, naxis2 = None, None, None
546 
547  naxis = [naxis1, naxis2]
548  for i in range(2):
549  if naxis[i] is None:
550  # cmin, cmax are the range of input star positions
551  cmin, cmax = [set.getContextOffset(i) + d*set.getContextScale(i) for d in (-0.5, 0.5)]
552  naxis[i] = cmax + cmin # a decent guess
553 
554  import lsst.afw.display.utils as ds9Utils
555  if naxis[0] > naxis[1]:
556  nx, ny = int(nspot*naxis[0]/float(naxis[1]) + 0.5), nspot
557  else:
558  nx, ny = nspot, int(nspot*naxis[1]/float(naxis[0]) + 0.5)
559 
560  mos = ds9Utils.Mosaic(gutter=2, background=0.02)
561 
562  xpos, ypos = np.linspace(0, naxis[0], nx), np.linspace(0, naxis[1], ny)
563  for y in ypos:
564  for x in xpos:
565  psf.build(x, y)
566 
567  im = afwImage.ImageF(*psf.getLoc().shape)
568  im.getArray()[:] = psf.getLoc()
569  im /= float(im.getArray().max())
570  if trim:
571  if trim > im.getHeight()//2:
572  trim = im.getHeight()//2
573 
574  im = im[trim:-trim, trim:-trim]
575 
576  mos.append(im)
577 
578  mosaic = mos.makeMosaic(mode=nx)
579  if frame is not None:
580  ds9.mtv(mosaic, frame=frame, title=title)
581  #
582  # Figure out the WCS for the mosaic
583  #
584  pos = []
585  if False:
586  for x, y, i in zip((xpos[0], xpos[-1]), (ypos[0], ypos[-1]), (0, mos.nImage - 1)):
587  bbox = mos.getBBox(i)
588  mosx = bbox.getMinX() + 0.5*(bbox.getWidth() - 1)
589  mosy = bbox.getMinY() + 0.5*(bbox.getHeight() - 1)
590  pos.append([afwGeom.PointD(mosx, mosy), wcs.pixelToSky(afwGeom.PointD(x, y))])
591  else:
592  pos.append([afwGeom.PointD(0, 0), wcs.pixelToSky(afwGeom.PointD(0, 0))])
593  pos.append([afwGeom.PointD(*mosaic.getDimensions()), wcs.pixelToSky(afwGeom.PointD(naxis1, naxis2))])
594 
595  CD = []
596  for i in range(2):
597  delta = pos[1][1][i].asDegrees() - pos[0][1][i].asDegrees()
598  CD.append(delta/(pos[1][0][i] - pos[0][0][i]))
599  CD = np.array(CD)
600  CD.shape = (2, 2)
601  mosWcs = afwGeom.makeSkyWcs(crval=pos[0][0], crpix=pos[0][1], cdMatrix=CD)
602 
603  if ext is not None:
604  title = "%s-%d" % (title, ext)
605 
606  if frame is not None:
607  ds9.mtv(mosaic, frame=frame, title=title, wcs=mosWcs)
608 
609  if diagnostics:
610  outFile = "%s-mod.fits" % title
611  if outDir:
612  outFile = os.path.join(outDir, outFile)
613  mosaic.writeFits(outFile, mosWcs.getFitsMetadata())
614 
615  #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
616 
617  mos = ds9Utils.Mosaic(gutter=4, background=0.002)
618  for i in range(set.getNsample()):
619  s = set.getSample(i)
620  if ext is not None and s.getExtindex() != ext:
621  continue
622 
623  smos = ds9Utils.Mosaic(gutter=2, background=-0.003)
624  for func in [s.getVig, s.getVigResi]:
625  arr = func()
626  if func == s.getVig:
627  norm = float(arr.max()) if True else s.getNorm()
628 
629  arr /= norm
630  im = afwImage.ImageF(*arr.shape)
631  im.getArray()[:] = arr
632  smos.append(im)
633 
634  mos.append(smos.makeMosaic(mode="x"))
635 
636  mosaic = mos.makeMosaic(title=title)
637 
638  if frame is not None:
639  ds9.mtv(mosaic, title=title, frame=frame+1)
640 
641  if diagnostics:
642  outFile = "%s-psfstars.fits" % title
643  if outDir:
644  outFile = os.path.join(outDir, outFile)
645 
646  mosaic.writeFits(outFile)
647 
648 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
649 
650 
651 def getLsstFlags(tab=None):
652  flagKeys = [
653  "base_PixelFlags_flag_edge",
654  #"base_PixelFlags_flag_interpolated",
655  #"base_PixelFlags_flag_interpolatedCenter",
656  #"base_PixelFlags_flag_saturated",
657  "base_PixelFlags_flag_saturatedCenter",
658  #"base_PixelFlags_flag_cr",
659  "base_PixelFlags_flag_crCenter",
660  "base_PixelFlags_flag_bad",
661  "base_PsfFlux_flag",
662  "parent",
663  ]
664 
665  if tab is None:
666  flags = {}
667  for i, k in enumerate(flagKeys):
668  flags[1 << i] = re.sub(r"\_flag", "",
669  re.sub(r"^base\_", "", re.sub(r"^base\_PixelFlags\_flag\_", "", k)))
670  else:
671  flags = 0
672  for i, k in enumerate(flagKeys):
673  if k == "parent":
674  try:
675  isSet = tab.get("deblend_nChild") > 0
676  except KeyError:
677  isSet = 0
678  else:
679  isSet = tab.get(k)
680  flags = np.bitwise_or(flags, np.where(isSet, 1 << i, 0))
681 
682  return flags
683 
684 
685 def guessCalexp(fileName):
686  for guess in [
687  re.sub("/src", r"", fileName),
688  re.sub("(SRC([^.]+))", r"CORR\2-exp", fileName),
689  ]:
690  if guess != fileName and os.path.exists(guess):
691  return guess
692 
693  raise RuntimeError("Unable to find a calexp to go with %s" % fileName)
694 
695 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
696 
697 
698 def makeitLsst(prefs, context, saveWcs=False, plot=dict()):
699  """This is the python wrapper that reads lsst tables"""
700  # Create an array of PSFs (one PSF for each extension)
701  if prefs.getVerboseType() != prefs.QUIET:
702  print("----- %d input catalogues:" % prefs.getNcat())
703 
704  if saveWcs: # only needed for making plots
705  wcssList = []
706 
707  fields = psfexLib.vectorField()
708  for cat in prefs.getCatalogs():
709  field = psfexLib.Field(cat)
710  wcss = []
711  wcssList.append(wcss)
712  with pyfits.open(cat):
713  # Hack: I want the WCS so I'll guess where the calexp is to be found
714  calexpFile = guessCalexp(cat)
715  md = readMetadata(calexpFile)
716  wcs = afwGeom.makeSkyWcs(md)
717 
718  if not wcs:
719  cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
720  cdMatrix.shape = (2, 2)
721  wcs = afwGeom.makeSkyWcs(crpix=afwGeom.PointD(0, 0),
722  crval=afwGeom.SpherePoint(0.0, 0.0, afwGeom.degrees),
723  cdMatrix=cdMatrix)
724 
725  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
726  # Find how many rows there are in the catalogue
727  md = readMetadata(cat)
728 
729  field.addExt(wcs, naxis1, naxis2, md.get("NAXIS2"))
730  if saveWcs:
731  wcss.append((wcs, naxis1, naxis2))
732 
733  field.finalize()
734  fields.append(field)
735 
736  fields[0].getNext() # number of extensions
737 
738  prefs.getPsfStep()
739 
740  sets = psfexLib.vectorSet()
741  for set in load_samplesLsst(prefs, context, plot=plot):
742  sets.append(set)
743 
744  psfexLib.makeit(fields, sets)
745 
746  ret = [[f.getPsfs() for f in fields], sets]
747  if saveWcs:
748  ret.append(wcssList)
749 
750  return ret
751 
752 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
753 
754 
755 def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval,
756  plot=dict(showFlags=False, showRejection=False)):
757  # allocate a new set iff set is None
758  if not set:
759  set = psfexLib.Set(context)
760 
761  cmin, cmax = None, None
762  if set.getNcontext():
763  cmin = np.empty(set.getNcontext())
764  cmax = np.empty(set.getNcontext())
765  for i in range(set.getNcontext()):
766  if set.getNsample():
767  cmin[i] = set.getContextOffset(i) - set.getContextScale(i)/2.0
768  cmax[i] = cmin[i] + set.getContextScale(i)
769  else:
770  cmin[i] = psfexLib.BIG
771  cmax[i] = -psfexLib.BIG
772  #
773  # Read data
774  #
775  tab = afwTable.SourceCatalog.readFits(filename)
776 
777  centroid = tab.getCentroidDefinition()
778  xm = tab.get("%s.x" % centroid)
779  ym = tab.get("%s.y" % centroid)
780 
781  shape = tab.getShapeDefinition()
782  ixx = tab.get("%s.xx" % shape)
783  iyy = tab.get("%s.yy" % shape)
784 
785  rmsSize = np.sqrt(0.5*(ixx + iyy))
786  elong = 0.5*(ixx - iyy)/(ixx + iyy)
787 
788  flux = tab.get(prefs.getPhotfluxRkey())
789  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
790  flags = getLsstFlags(tab)
791 
792  #
793  # Now the VIGNET data
794  #
795  vigw, vigh = 35, 35 # [prefs.getPsfsize()[i] for i in range(2)]
796  if set.empty():
797  set.setVigSize(vigw, vigh)
798 
799  vignet = np.empty(nobj*vigw*vigh, "float32").reshape(nobj, vigw, vigh)
800 
801  # Hack: I want the WCS so I'll guess where the calexp is to be found
802  calexpFile = guessCalexp(filename)
803  mi = afwImage.MaskedImageF(calexpFile)
804  backnoise2 = np.median(mi.getVariance().getArray())
805  gain = 1.0
806 
807  edgeBit = [k for k, v in getFlags().items() if v == "edge"][0]
808 
809  for i, xc, yc in zip(range(nobj), xm, ym):
810  try:
811  x, y = int(xc), int(yc)
812  except ValueError:
813  flags[i] |= edgeBit # mark star as bad
814 
815  try:
816  pstamp = mi[x - vigw//2:x + vigw//2 + 1, y - vigh//2:y + vigh//2 + 1]
817  vignet[i] = pstamp.getImage().getArray().transpose()
818  except Exception:
819  flags[i] |= edgeBit # mark star as bad
820 
821  # Try to load the set of context keys
822  pc = 0
823  contextvalp = []
824  for i, key in enumerate(context.getName()):
825  if context.getPcflag(i):
826  contextvalp.append(pcval[pc])
827  pc += 1
828  elif key[0] == ':':
829  try:
830  contextvalp.append(tab.header[key[1:]])
831  except KeyError:
832  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
833  (key[1:], filename))
834  else:
835  try:
836  contextvalp.append(tab.get(key))
837  except KeyError:
838  raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
839  (key, filename))
840  set.setContextname(i, key)
841 
842  # Now examine each vector of the shipment
843  good = select_candidates(set, prefs, frmin, frmax,
844  flags, flux, fluxErr, rmsSize, elong, vignet,
845  plot=plot, title="%s[%d]" % (filename, ext + 1) if next > 1 else filename)
846  #
847  # Insert the good candidates into the set
848  #
849  if not vignet.dtype.isnative:
850  # without the swap setVig fails with "ValueError: 'unaligned arrays cannot be converted to C++'"
851  vignet = vignet.byteswap()
852 
853  for i in np.where(good)[0]:
854  sample = set.newSample()
855  sample.setCatindex(catindex)
856  sample.setExtindex(ext)
857 
858  sample.setVig(vignet[i])
859  if False:
860  pstamp = afwImage.ImageF(*vignet[i].shape)
861  pstamp.getArray()[:] = sample.getVig()
862 
863  ds9.mtv(pstamp)
864 
865  sample.setNorm(float(flux[i]))
866  sample.setBacknoise2(backnoise2)
867  sample.setGain(gain)
868  sample.setX(float(xm[i]))
869  sample.setY(float(ym[i]))
870  sample.setFluxrad(float(rmsSize[i]))
871 
872  for j in range(set.getNcontext()):
873  sample.setContext(j, float(contextvalp[j][i]))
874 
875  set.finiSample(sample, prefs.getProfAccuracy())
876 
877  #---- Update min and max
878  for j in range(set.getNcontext()):
879  cmin[j] = contextvalp[j][good].min()
880  cmax[j] = contextvalp[j][good].max()
881 
882  # Update the scaling
883  if set.getNsample():
884  for i in range(set.getNcontext()):
885  set.setContextScale(i, cmax[i] - cmin[i])
886  set.setContextOffset(i, (cmin[i] + cmax[i])/2.0)
887 
888  # Don't waste memory!
889  set.trimMemory()
890 
891  return set
892 
893 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
894 
895 
896 def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict()):
897  minsn = prefs.getMinsn()
898  maxelong = (prefs.getMaxellip() + 1.0)/(1.0 - prefs.getMaxellip()) if prefs.getMaxellip() < 1.0 else 100
899  min = prefs.getFwhmrange()[0]
900  max = prefs.getFwhmrange()[1]
901 
902  filenames = prefs.getCatalogs()
903 
904  ncat = len(filenames)
905  fwhmmin = np.empty(ncat)
906  fwhmmax = np.empty(ncat)
907 
908  if not prefs.getAutoselectFlag():
909  fwhmmin = np.zeros(ncat) + prefs.getFwhmrange()[0]
910  fwhmmax = np.zeros(ncat) + prefs.getFwhmrange()[1]
911  fwhmmode = (fwhmmin + fwhmmax)/2.0
912  else:
913  fwhms = {}
914 
915  #-- Try to estimate the most appropriate Half-light Radius range
916  #-- Get the Half-light radii
917  nobj = 0
918  for i, fileName in enumerate(filenames):
919  fwhms[i] = []
920 
921  if prefs.getVerboseType() != prefs.QUIET:
922  print("Examining Catalog #%d" % (i+1))
923 
924  #---- Read input catalog
925  tab = afwTable.SourceCatalog.readFits(fileName)
926 
927  #-------- Fill the FWHM array
928  shape = tab.getShapeDefinition()
929  ixx = tab.get("%s.xx" % shape)
930  iyy = tab.get("%s.yy" % shape)
931 
932  rmsSize = np.sqrt(0.5*(ixx + iyy))
933  elong = 0.5*(ixx - iyy)/(ixx + iyy)
934 
935  flux = tab.get(prefs.getPhotfluxRkey())
936  fluxErr = tab.get(prefs.getPhotfluxerrRkey())
937 
938  flags = getLsstFlags(tab)
939 
940  good = np.logical_and(flux/fluxErr > minsn,
941  np.logical_not(flags & prefs.getFlagMask()))
942  good = np.logical_and(good, elong < maxelong)
943  fwhm = 2.0*rmsSize
944  good = np.logical_and(good, fwhm >= min)
945  good = np.logical_and(good, fwhm < max)
946  fwhms[i] = fwhm[good]
947 
948  if prefs.getVarType() == prefs.VAR_NONE:
949  if nobj:
950  fwhms_all = np.empty(sum([len(l) for l in fwhms.values()]))
951  i = 0
952  for l in fwhms.values():
953  fwhms_all[i:len(l)] = l
954  i += len(l)
955  mode, min, max = compute_fwhmrange(fwhms_all, prefs.getMaxvar(),
956  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1],
957  plot=plot)
958  else:
959  raise RuntimeError("No source with appropriate FWHM found!!")
960  mode = min = max = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
961 
962  fwhmmin = np.zeros(ncat) + min
963  fwhmmax = np.zeros(ncat) + max
964  fwhmmode = np.zeros(ncat) + mode
965  else:
966  fwhmmode = np.empty(ncat)
967  fwhmmin = np.empty(ncat)
968  fwhmmax = np.empty(ncat)
969 
970  for i in range(ncat):
971  nobj = len(fwhms[i])
972  if (nobj):
973  fwhmmode[i], fwhmmin[i], fwhmmax[i] = \
974  compute_fwhmrange(fwhms[i], prefs.getMaxvar(),
975  prefs.getFwhmrange()[0], prefs.getFwhmrange()[1], plot=plot)
976  else:
977  raise RuntimeError("No source with appropriate FWHM found!!")
978  fwhmmode[i] = fwhmmin[i] = fwhmmax[i] = 2.35/(1.0 - 1.0/psfexLib.cvar.INTERPFAC)
979 
980  # Read the samples
981  mode = psfexLib.BIG # mode of FWHM distribution
982 
983  sets = []
984  for i, fileName in enumerate(filenames):
985  set = None
986  for ext in range(next):
987  set = read_samplesLsst(prefs, set, fileName, fwhmmin[i]/2.0, fwhmmax[i]/2.0,
988  ext, next, i, context,
989  context.getPc(i) if context.getNpc() else None, plot=plot)
990 
991  if fwhmmode[i] < mode:
992  mode = fwhmmode[i]
993 
994  set.setFwhm(mode)
995 
996  if prefs.getVerboseType() != prefs.QUIET:
997  if set.getNsample():
998  print("%d samples loaded." % set.getNsample())
999  else:
1000  raise RuntimeError("No appropriate source found!!")
1001 
1002  sets.append(set)
1003 
1004  return sets
1005 
1006 
1007 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
1008 
1009 def makeit(prefs, context, saveWcs=False, plot=dict()):
1010  """This is the python wrapper for the original psfex that reads SExtractor outputs"""
1011  # Create an array of PSFs (one PSF for each extension)
1012  if prefs.getVerboseType() != prefs.QUIET:
1013  print("----- %d input catalogues:" % prefs.getNcat())
1014 
1015  if saveWcs: # only needed for making plots
1016  wcssList = []
1017 
1018  fields = psfexLib.vectorField()
1019  for cat in prefs.getCatalogs():
1020  field = psfexLib.Field(cat)
1021  wcss = []
1022  wcssList.append(wcss)
1023  with pyfits.open(cat) as pf:
1024  for hdu in pf:
1025  if hdu.name == "PRIMARY":
1026  pass
1027  elif hdu.name == "LDAC_IMHEAD":
1028  hdr = hdu.data[0][0] # the fits header from the original fits image
1029  md = PropertySet()
1030  for line in hdr:
1031  try:
1032  md.set(*splitFitsCard(line))
1033  except AttributeError:
1034  continue
1035 
1036  if not md.exists("CRPIX1"): # no WCS; try WCSA
1037  for k in md.names():
1038  if re.search(r"A$", k):
1039  md.set(k[:-1], md.get(k))
1040  wcs = afwGeom.makeSkyWcs(md)
1041  naxis1, naxis2 = md.get("NAXIS1"), md.get("NAXIS2")
1042  elif hdu.name == "LDAC_OBJECTS":
1043  nobj = len(hdu.data)
1044 
1045  assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
1046  field.addExt(wcs, naxis1, naxis2, nobj)
1047  if saveWcs:
1048  wcss.append((wcs, naxis1, naxis2))
1049  wcs = None
1050 
1051  field.finalize()
1052  fields.append(field)
1053 
1054  sets = psfexLib.vectorSet()
1055  for set in load_samples(prefs, context, plot=plot):
1056  sets.append(set)
1057 
1058  psfexLib.makeit(fields, sets)
1059 
1060  ret = [[f.getPsfs() for f in fields], sets]
1061  if saveWcs:
1062  ret.append(wcssList)
1063 
1064  return ret
def select_candidates(set, prefs, frmin, frmax, flags, flux, fluxerr, rmsSize, elong, vignet, plot=dict(), title="")
Definition: psfex.py:278
def read_samples(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:105
def makeit(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:1009
def showPsf(psf, set, ext=None, wcsData=None, trim=0, nspot=5, diagnostics=False, outDir="", frame=None, title=None)
Definition: psfex.py:534
def makeitLsst(prefs, context, saveWcs=False, plot=dict())
Definition: psfex.py:698
def compute_fwhmrange(fwhm, maxvar, minin, maxin, plot=dict(fwhmHistogram=False))
Definition: psfex.py:50
def read_samplesLsst(prefs, set, filename, frmin, frmax, ext, next, catindex, context, pcval, plot=dict(showFlags=False, showRejection=False))
Definition: psfex.py:756
def load_samples(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:395
def load_samplesLsst(prefs, context, ext=psfexLib.Prefs.ALL_EXTENSIONS, next=1, plot=dict())
Definition: psfex.py:896