lsst.pipe.tasks  13.0-54-gc325bc5f
photoCal.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 # \package lsst.pipe.tasks.
23 
24 from __future__ import absolute_import, division, print_function
25 from builtins import zip
26 from builtins import input
27 from builtins import range
28 
29 import math
30 import random
31 import sys
32 
33 import numpy as np
34 
35 import lsst.pex.config as pexConf
36 import lsst.pipe.base as pipeBase
37 import lsst.afw.table as afwTable
38 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, fluxFromABMag, Calib
39 from lsst.meas.astrom import RefMatchTask, RefMatchConfig
40 import lsst.afw.display.ds9 as ds9
41 from lsst.meas.algorithms import getRefFluxField
42 from .colorterms import ColortermLibrary
43 
44 __all__ = ["PhotoCalTask", "PhotoCalConfig"]
45 
46 
47 def checkSourceFlags(source, sourceKeys):
48  """!Return True if the given source has all good flags set and none of the bad flags set.
49 
50  \param[in] source SourceRecord object to process.
51  \param[in] sourceKeys Struct of source catalog keys, as returned by PhotCalTask.getSourceKeys()
52  """
53  for k in sourceKeys.goodFlags:
54  if not source.get(k):
55  return False
56  if source.getPsfFluxFlag():
57  return False
58  for k in sourceKeys.badFlags:
59  if source.get(k):
60  return False
61  return True
62 
63 
64 class PhotoCalConfig(RefMatchConfig):
65  """Config for PhotoCal"""
66  magLimit = pexConf.Field(
67  dtype=float,
68  default=22.0,
69  doc="Don't use objects fainter than this magnitude",
70  )
71  reserveFraction = pexConf.Field(
72  dtype=float,
73  doc="Fraction of candidates to reserve from fitting; none if <= 0",
74  default=-1.0,
75  )
76  reserveSeed = pexConf.Field(
77  dtype = int,
78  doc = "This number will be multiplied by the exposure ID "
79  "to set the random seed for reserving candidates",
80  default = 1,
81  )
82  fluxField = pexConf.Field(
83  dtype=str,
84  default="slot_CalibFlux_flux",
85  doc=("Name of the source flux field to use. The associated flag field\n"
86  "('<name>_flags') will be implicitly included in badFlags."),
87  )
88  applyColorTerms = pexConf.Field(
89  dtype=bool,
90  default=None,
91  doc=("Apply photometric color terms to reference stars? One of:\n"
92  "None: apply if colorterms and photoCatName are not None;\n"
93  " fail if color term data is not available for the specified ref catalog and filter.\n"
94  "True: always apply colorterms; fail if color term data is not available for the\n"
95  " specified reference catalog and filter.\n"
96  "False: do not apply."),
97  optional=True,
98  )
99  goodFlags = pexConf.ListField(
100  dtype=str,
101  default=[],
102  doc="List of source flag fields that must be set for a source to be used.",
103  )
104  badFlags = pexConf.ListField(
105  dtype=str,
106  default=["base_PixelFlags_flag_edge", "base_PixelFlags_flag_interpolated",
107  "base_PixelFlags_flag_saturated"],
108  doc="List of source flag fields that will cause a source to be rejected when they are set.",
109  )
110  sigmaMax = pexConf.Field(
111  dtype=float,
112  default=0.25,
113  doc="maximum sigma to use when clipping",
114  optional=True,
115  )
116  nSigma = pexConf.Field(
117  dtype=float,
118  default=3.0,
119  doc="clip at nSigma",
120  )
121  useMedian = pexConf.Field(
122  dtype=bool,
123  default=True,
124  doc="use median instead of mean to compute zeropoint",
125  )
126  nIter = pexConf.Field(
127  dtype=int,
128  default=20,
129  doc="number of iterations",
130  )
131  colorterms = pexConf.ConfigField(
132  dtype=ColortermLibrary,
133  doc="Library of photometric reference catalog name: color term dict",
134  )
135  photoCatName = pexConf.Field(
136  dtype=str,
137  optional=True,
138  doc=("Name of photometric reference catalog; used to select a color term dict in colorterms."
139  " see also applyColorTerms"),
140  )
141  magErrFloor = pexConf.RangeField(
142  dtype=float,
143  default=0.0,
144  doc="Additional magnitude uncertainty to be added in quadrature with measurement errors.",
145  min=0.0,
146  )
147  doSelectUnresolved = pexConf.Field(
148  dtype=bool,
149  default=True,
150  doc=("Use the extendedness parameter to select objects to use in photometric calibration?\n"
151  "This applies only to the sources detected on the exposure, not the reference catalog"),
152  )
153 
154  def validate(self):
155  pexConf.Config.validate(self)
156  if self.applyColorTerms and self.photoCatName is None:
157  raise RuntimeError("applyColorTerms=True requires photoCatName is non-None")
158  if self.applyColorTerms and len(self.colorterms.data) == 0:
159  raise RuntimeError("applyColorTerms=True requires colorterms be provided")
160 
161 
162 
168 
169 class PhotoCalTask(RefMatchTask):
170  r"""!
171 \anchor PhotoCalTask_
172 
173 \brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
174 
175 \section pipe_tasks_photocal_Contents Contents
176 
177  - \ref pipe_tasks_photocal_Purpose
178  - \ref pipe_tasks_photocal_Initialize
179  - \ref pipe_tasks_photocal_IO
180  - \ref pipe_tasks_photocal_Config
181  - \ref pipe_tasks_photocal_Debug
182  - \ref pipe_tasks_photocal_Example
183 
184 \section pipe_tasks_photocal_Purpose Description
185 
186 \copybrief PhotoCalTask
187 
188 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue.
189 The type of flux to use is specified by PhotoCalConfig.fluxField.
190 
191 The algorithm clips outliers iteratively, with parameters set in the configuration.
192 
193 \note This task can adds fields to the schema, so any code calling this task must ensure that
194 these columns are indeed present in the input match list; see \ref pipe_tasks_photocal_Example
195 
196 \section pipe_tasks_photocal_Initialize Task initialisation
197 
198 \copydoc \_\_init\_\_
199 
200 \section pipe_tasks_photocal_IO Inputs/Outputs to the run method
201 
202 \copydoc run
203 
204 \section pipe_tasks_photocal_Config Configuration parameters
205 
206 See \ref PhotoCalConfig
207 
208 \section pipe_tasks_photocal_Debug Debug variables
209 
210 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
211 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
212 
213 The available variables in PhotoCalTask are:
214 <DL>
215  <DT> \c display
216  <DD> If True enable other debug outputs
217  <DT> \c displaySources
218  <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue:
219  <DL>
220  <DT> red x
221  <DD> Bad objects
222  <DT> blue +
223  <DD> Matched objects deemed unsuitable for photometric calibration.
224  Additional information is:
225  - a cyan o for galaxies
226  - a magenta o for variables
227  <DT> magenta *
228  <DD> Objects that failed the flux cut
229  <DT> green o
230  <DD> Objects used in the photometric calibration
231  </DL>
232  <DT> \c scatterPlot
233  <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude.
234  - good objects in blue
235  - rejected objects in red
236  (if \c scatterPlot is 2 or more, prompt to continue after each iteration)
237 </DL>
238 
239 \section pipe_tasks_photocal_Example A complete example of using PhotoCalTask
240 
241 This code is in \link examples/photoCalTask.py\endlink, and can be run as \em e.g.
242 \code
243 examples/photoCalTask.py
244 \endcode
245 \dontinclude photoCalTask.py
246 
247 Import the tasks (there are some other standard imports; read the file for details)
248 \skipline from lsst.pipe.tasks.astrometry
249 \skipline measPhotocal
250 
251 We need to create both our tasks before processing any data as the task constructors
252 can add extra columns to the schema which we get from the input catalogue, \c scrCat:
253 \skipline getSchema
254 
255 Astrometry first:
256 \skip AstrometryTask.ConfigClass
257 \until aTask
258 (that \c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises,
259 so we tell it to use the \c r band)
260 
261 Then photometry:
262 \skip measPhotocal
263 \until pTask
264 
265 If the schema has indeed changed we need to add the new columns to the source table
266 (yes; this should be easier!)
267 \skip srcCat
268 \until srcCat = cat
269 
270 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same
271 task objects):
272 \skip matches
273 \until result
274 
275 We can then unpack and use the results:
276 \skip calib
277 \until np.log
278 
279 <HR>
280 To investigate the \ref pipe_tasks_photocal_Debug, put something like
281 \code{.py}
282  import lsstDebug
283  def DebugInfo(name):
284  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
285  if name.endswith(".PhotoCal"):
286  di.display = 1
287 
288  return di
289 
290  lsstDebug.Info = DebugInfo
291 \endcode
292 into your debug.py file and run photoCalTask.py with the \c --debug flag.
293  """
294  ConfigClass = PhotoCalConfig
295  _DefaultName = "photoCal"
296 
297  def __init__(self, refObjLoader, schema=None, **kwds):
298  """!Create the photometric calibration task. See PhotoCalTask.init for documentation
299  """
300  RefMatchTask.__init__(self, refObjLoader, schema=None, **kwds)
301  self.scatterPlot = None
302  self.fig = None
303  if schema is not None:
304  self.usedKey = schema.addField("calib_photometryUsed", type="Flag",
305  doc="set if source was used in photometric calibration")
306  self.candidateKey = schema.addField("calib_photometryCandidate", type="Flag",
307  doc="set if source was a candidate for use in calibration")
308  self.reservedKey = schema.addField("calib_photometryReserved", type="Flag",
309  doc="set if source was reserved, so not used in calibration")
310  else:
311  self.usedKey = None
312  self.candidateKey = None
313  self.reservedKey = None
314 
315  def getSourceKeys(self, schema):
316  """!Return a struct containing the source catalog keys for fields used by PhotoCalTask.
317 
318  Returned fields include:
319  - flux
320  - fluxErr
321  - goodFlags: a list of keys for field names in self.config.goodFlags
322  - badFlags: a list of keys for field names in self.config.badFlags
323  - starGal: key for star/galaxy classification
324  """
325  goodFlags = [schema.find(name).key for name in self.config.goodFlags]
326  flux = schema.find(self.config.fluxField).key
327  fluxErr = schema.find(self.config.fluxField + "Sigma").key
328  badFlags = [schema.find(name).key for name in self.config.badFlags]
329  try:
330  starGal = schema.find("base_ClassificationExtendedness_value").key
331  except KeyError:
332  starGal = None
333  return pipeBase.Struct(flux=flux, fluxErr=fluxErr, goodFlags=goodFlags, badFlags=badFlags,
334  starGal=starGal)
335 
336  def isUnresolved(self, source, starGalKey=None):
337  """!Return whether the provided source is unresolved or not
338 
339  This particular implementation is designed to work with the
340  base_ClassificationExtendedness_value=0.0 or 1.0 scheme. Because
341  of the diversity of star/galaxy classification outputs (binary
342  decision vs probabilities; signs), it's difficult to make this
343  configurable without using code. This method should therefore
344  be overridden to use the appropriate classification output.
345 
346  \param[in] source Source to test
347  \param[in] starGalKey Struct of schema keys for source
348  \return boolean value for starGalKey (True indicates Unresolved)
349  """
350  return source.get(starGalKey) < 0.5 if starGalKey is not None else True
351 
352  @pipeBase.timeMethod
353  def selectMatches(self, matches, sourceKeys, filterName, frame=None):
354  """!Select reference/source matches according the criteria specified in the config.
355 
356  \param[in] matches ReferenceMatchVector (not modified)
357  \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
358  \param[in] filterName name of camera filter; used to obtain the reference flux field
359  \param[in] frame ds9 frame number to use for debugging display
360  if frame is non-None, display information about trimmed objects on that ds9 frame:
361  - Bad: red x
362  - Unsuitable objects: blue + (and a cyan o if a galaxy)
363  - Failed flux cut: magenta *
364 
365  \return a \link lsst.afw.table.ReferenceMatchVector\endlink that contains only the selected matches.
366  If a schema was passed during task construction, a flag field will be set on sources
367  in the selected matches.
368 
369  \throws ValueError There are no valid matches.
370  """
371  self.log.debug("Number of input matches: %d", len(matches))
372 
373  if self.config.doSelectUnresolved:
374  # Select only resolved sources if asked to do so.
375  matches = [m for m in matches if self.isUnresolved(m.second, sourceKeys.starGal)]
376  self.log.debug("Number of matches after culling resolved sources: %d", len(matches))
377 
378  if len(matches) == 0:
379  raise ValueError("No input matches")
380 
381  for m in matches:
382  if self.candidateKey is not None:
383  m.second.set(self.candidateKey, True)
384 
385  # Only use stars for which the flags indicate the photometry is good.
386  afterFlagCutInd = [i for i, m in enumerate(matches) if checkSourceFlags(m.second, sourceKeys)]
387  afterFlagCut = [matches[i] for i in afterFlagCutInd]
388  self.log.debug("Number of matches after source flag cuts: %d", len(afterFlagCut))
389 
390  if len(afterFlagCut) != len(matches):
391  if frame is not None:
392  with ds9.Buffering():
393  for i, m in enumerate(matches):
394  if i not in afterFlagCutInd:
395  x, y = m.second.getCentroid()
396  ds9.dot("x", x, y, size=4, frame=frame, ctype=ds9.RED)
397 
398  matches = afterFlagCut
399 
400  if len(matches) == 0:
401  raise ValueError("All matches eliminated by source flags")
402 
403  refSchema = matches[0].first.schema
404  try:
405  photometricKey = refSchema.find("photometric").key
406  try:
407  resolvedKey = refSchema.find("resolved").key
408  except:
409  resolvedKey = None
410 
411  try:
412  variableKey = refSchema.find("variable").key
413  except:
414  variableKey = None
415  except:
416  self.log.warn("No 'photometric' flag key found in reference schema.")
417  photometricKey = None
418 
419  if photometricKey is not None:
420  afterRefCutInd = [i for i, m in enumerate(matches) if m.first.get(photometricKey)]
421  afterRefCut = [matches[i] for i in afterRefCutInd]
422 
423  if len(afterRefCut) != len(matches):
424  if frame is not None:
425  with ds9.Buffering():
426  for i, m in enumerate(matches):
427  if i not in afterRefCutInd:
428  x, y = m.second.getCentroid()
429  ds9.dot("+", x, y, size=4, frame=frame, ctype=ds9.BLUE)
430 
431  if resolvedKey and m.first.get(resolvedKey):
432  ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.CYAN)
433  if variableKey and m.first.get(variableKey):
434  ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.MAGENTA)
435 
436  matches = afterRefCut
437 
438  self.log.debug("Number of matches after reference catalog cuts: %d", len(matches))
439  if len(matches) == 0:
440  raise RuntimeError("No sources remain in match list after reference catalog cuts.")
441  fluxName = getRefFluxField(refSchema, filterName)
442  fluxKey = refSchema.find(fluxName).key
443  if self.config.magLimit is not None:
444  fluxLimit = fluxFromABMag(self.config.magLimit)
445 
446  afterMagCutInd = [i for i, m in enumerate(matches) if (m.first.get(fluxKey) > fluxLimit and
447  m.second.getPsfFlux() > 0.0)]
448  else:
449  afterMagCutInd = [i for i, m in enumerate(matches) if m.second.getPsfFlux() > 0.0]
450 
451  afterMagCut = [matches[i] for i in afterMagCutInd]
452 
453  if len(afterMagCut) != len(matches):
454  if frame is not None:
455  with ds9.Buffering():
456  for i, m in enumerate(matches):
457  if i not in afterMagCutInd:
458  x, y = m.second.getCentroid()
459  ds9.dot("*", x, y, size=4, frame=frame, ctype=ds9.MAGENTA)
460 
461  matches = afterMagCut
462 
463  self.log.debug("Number of matches after magnitude limit cuts: %d", len(matches))
464 
465  if len(matches) == 0:
466  raise RuntimeError("No sources remaining in match list after magnitude limit cuts.")
467 
468  if frame is not None:
469  with ds9.Buffering():
470  for m in matches:
471  x, y = m.second.getCentroid()
472  ds9.dot("o", x, y, size=4, frame=frame, ctype=ds9.GREEN)
473 
474  result = []
475  for m in matches:
476  if self.usedKey is not None:
477  m.second.set(self.usedKey, True)
478  result.append(m)
479  return result
480 
481  @pipeBase.timeMethod
482  def extractMagArrays(self, matches, filterName, sourceKeys):
483  """!Extract magnitude and magnitude error arrays from the given matches.
484 
485  \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink
486  \param[in] filterName Name of filter being calibrated
487  \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
488 
489  \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
490  where magErr is an error in the magnitude; the error in srcMag - refMag
491  If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
492  magErr is what is later used to determine the zero point.
493  Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
494  (1 or 2 strings)
495  \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been
496  discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
497  """
498  srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
499  srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches])
500  if not np.all(np.isfinite(srcFluxErrArr)):
501  # this is an unpleasant hack; see DM-2308 requesting a better solution
502  self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
503  srcFluxErrArr = np.sqrt(srcFluxArr)
504 
505  # convert source flux from DN to an estimate of Jy
506  JanskysPerABFlux = 3631.0
507  srcFluxArr = srcFluxArr * JanskysPerABFlux
508  srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
509 
510  if not matches:
511  raise RuntimeError("No reference stars are available")
512  refSchema = matches[0].first.schema
513 
514  applyColorTerms = self.config.applyColorTerms
515  applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
516  if self.config.applyColorTerms is None:
517  # apply color terms if color term data is available and photoCatName specified
518  ctDataAvail = len(self.config.colorterms.data) > 0
519  photoCatSpecified = self.config.photoCatName is not None
520  applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
521  applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is")
522  applyColorTerms = ctDataAvail and photoCatSpecified
523 
524  if applyColorTerms:
525  self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s",
526  filterName, self.config.photoCatName, applyCTReason)
527  ct = self.config.colorterms.getColorterm(
528  filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
529  else:
530  self.log.info("Not applying color terms because %s", applyCTReason)
531  ct = None
532 
533  if ct: # we have a color term to worry about
534  fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)]
535  missingFluxFieldList = []
536  for fluxField in fluxFieldList:
537  try:
538  refSchema.find(fluxField).key
539  except KeyError:
540  missingFluxFieldList.append(fluxField)
541 
542  if missingFluxFieldList:
543  self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms",
544  " ".join(missingFluxFieldList))
545  ct = None
546 
547  if not ct:
548  fluxFieldList = [getRefFluxField(refSchema, filterName)]
549 
550  refFluxArrList = [] # list of ref arrays, one per flux field
551  refFluxErrArrList = [] # list of ref flux arrays, one per flux field
552  for fluxField in fluxFieldList:
553  fluxKey = refSchema.find(fluxField).key
554  refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
555  try:
556  fluxErrKey = refSchema.find(fluxField + "Sigma").key
557  refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
558  except KeyError:
559  # Reference catalogue may not have flux uncertainties; HACK
560  self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
561  fluxField)
562  refFluxErrArr = np.sqrt(refFluxArr)
563 
564  refFluxArrList.append(refFluxArr)
565  refFluxErrArrList.append(refFluxErrArr)
566 
567  if ct: # we have a color term to worry about
568  refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
569  refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
570 
571  refMagArr = ct.transformMags(refMagArr1, refMagArr2)
572  refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
573  else:
574  refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]])
575 
576  srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
577 
578  # Fitting with error bars in both axes is hard
579  # for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
580  magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in zip(srcFluxErrArr, srcFluxArr)])
581  if self.config.magErrFloor != 0.0:
582  magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
583 
584  srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in zip(srcFluxErrArr, srcFluxArr)])
585  refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in zip(refFluxErrArr, refFluxArr)])
586 
587  return pipeBase.Struct(
588  srcMag=srcMagArr,
589  refMag=refMagArr,
590  magErr=magErrArr,
591  srcMagErr=srcMagErrArr,
592  refMagErr=refMagErrArr,
593  refFluxFieldList=fluxFieldList,
594  )
595 
596  @pipeBase.timeMethod
597  def run(self, exposure, sourceCat, expId=0):
598  """!Do photometric calibration - select matches to use and (possibly iteratively) compute
599  the zero point.
600 
601  \param[in] exposure Exposure upon which the sources in the matches were detected.
602  \param[in] sourceCat A catalog of sources to use in the calibration
603  (\em i.e. a list of lsst.afw.table.Match with
604  \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord ---
605  the reference object and matched object respectively).
606  (will not be modified except to set the outputField if requested.).
607 
608  \return Struct of:
609  - calib ------- \link lsst::afw::image::Calib\endlink object containing the zero point
610  - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays
611  - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches.
612  - zp ---------- Photometric zero point (mag)
613  - sigma ------- Standard deviation of fit of photometric zero point (mag)
614  - ngood ------- Number of sources used to fit photometric zero point
615 
616  The exposure is only used to provide the name of the filter being calibrated (it may also be
617  used to generate debugging plots).
618 
619  The reference objects:
620  - Must include a field \c photometric; True for objects which should be considered as
621  photometric standards
622  - Must include a field \c flux; the flux used to impose a magnitude limit and also to calibrate
623  the data to (unless a color term is specified, in which case ColorTerm.primary is used;
624  See https://jira.lsstcorp.org/browse/DM-933)
625  - May include a field \c stargal; if present, True means that the object is a star
626  - May include a field \c var; if present, True means that the object is variable
627 
628  The measured sources:
629  - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration
630 
631  \throws RuntimeError with the following strings:
632 
633  <DL>
634  <DT> `sources' schema does not contain the calibration object flag "XXX"`
635  <DD> The constructor added fields to the schema that aren't in the Sources
636  <DT> No input matches
637  <DD> The input match vector is empty
638  <DT> All matches eliminated by source flags
639  <DD> The flags specified by \c badFlags in the config eliminated all candidate objects
640  <DT> No sources remain in match list after reference catalog cuts
641  <DD> The reference catalogue has a column "photometric", but no matched objects have it set
642  <DT> No sources remaining in match list after magnitude limit cuts
643  <DD> All surviving matches are either too faint in the catalogue or have negative or \c NaN flux
644  <DT> No reference stars are available
645  <DD> No matches survive all the checks
646  </DL>
647  """
648  import lsstDebug
649 
650  display = lsstDebug.Info(__name__).display
651  displaySources = display and lsstDebug.Info(__name__).displaySources
652  self.scatterPlot = display and lsstDebug.Info(__name__).scatterPlot
653 
654  if self.scatterPlot:
655  from matplotlib import pyplot
656  try:
657  self.fig.clf()
658  except:
659  self.fig = pyplot.figure()
660 
661  if displaySources:
662  frame = 1
663  ds9.mtv(exposure, frame=frame, title="photocal")
664  else:
665  frame = None
666 
667  res = self.loadAndMatch(exposure, sourceCat)
668 
669  #from res.matches, reserve a fraction of the population and mark the sources as reserved
670 
671  if self.config.reserveFraction > 0:
672  random.seed(self.config.reserveSeed*expId)
673  reserveList = random.sample(res.matches,
674  int((self.config.reserveFraction)*len(res.matches)))
675 
676  for candidate in reserveList:
677  res.matches.remove(candidate)
678 
679  if reserveList and self.reservedKey is not None:
680  for candidate in reserveList:
681  candidate.second.set(self.reservedKey, True)
682 
683  matches = res.matches
684  for m in matches:
685  if self.candidateKey is not None:
686  m.second.set(self.candidateKey, True)
687 
688  filterName = exposure.getFilter().getName()
689  sourceKeys = self.getSourceKeys(matches[0].second.schema)
690 
691  matches = self.selectMatches(matches=matches, sourceKeys=sourceKeys, filterName=filterName,
692  frame=frame)
693  arrays = self.extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
694 
695  if matches and self.usedKey:
696  try:
697  # matches[].second is a measured source, wherein we wish to set outputField.
698  # Check that the field is present in the Sources schema.
699  matches[0].second.getSchema().find(self.usedKey)
700  except:
701  raise RuntimeError("sources' schema does not contain the calib_photometryUsed flag \"%s\"" %
702  self.usedKey)
703 
704  # Fit for zeropoint. We can run the code more than once, so as to
705  # give good stars that got clipped by a bad first guess a second
706  # chance.
707 
708  calib = Calib()
709  zp = None # initial guess
710  r = self.getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr, zp0=zp)
711  zp = r.zp
712  self.log.info("Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
713 
714  flux0 = 10**(0.4*r.zp) # Flux of mag=0 star
715  flux0err = 0.4*math.log(10)*flux0*r.sigma # Error in flux0
716 
717  calib.setFluxMag0(flux0, flux0err)
718 
719  return pipeBase.Struct(
720  calib=calib,
721  arrays=arrays,
722  matches=matches,
723  zp=r.zp,
724  sigma=r.sigma,
725  ngood=r.ngood,
726  )
727 
728  def getZeroPoint(self, src, ref, srcErr=None, zp0=None):
729  """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars)
730 
731  We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists:
732  1. We use the median/interquartile range to estimate the position to clip around, and the
733  "sigma" to use.
734  2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently
735  large estimate will prevent the clipping from ever taking effect.
736  3. Rather than start with the median we start with a crude mode. This means that a set of magnitude
737  residuals with a tight core and asymmetrical outliers will start in the core. We use the width of
738  this core to set our maximum sigma (see 2.)
739 
740  \return Struct of:
741  - zp ---------- Photometric zero point (mag)
742  - sigma ------- Standard deviation of fit of zero point (mag)
743  - ngood ------- Number of sources used to fit zero point
744  """
745  sigmaMax = self.config.sigmaMax
746 
747  dmag = ref - src
748 
749  indArr = np.argsort(dmag)
750  dmag = dmag[indArr]
751 
752  if srcErr is not None:
753  dmagErr = srcErr[indArr]
754  else:
755  dmagErr = np.ones(len(dmag))
756 
757  # need to remove nan elements to avoid errors in stats calculation with numpy
758  ind_noNan = np.array([i for i in range(len(dmag))
759  if (not np.isnan(dmag[i]) and not np.isnan(dmagErr[i]))])
760  dmag = dmag[ind_noNan]
761  dmagErr = dmagErr[ind_noNan]
762 
763  IQ_TO_STDEV = 0.741301109252802 # 1 sigma in units of interquartile (assume Gaussian)
764 
765  npt = len(dmag)
766  ngood = npt
767  good = None # set at end of first iteration
768  for i in range(self.config.nIter):
769  if i > 0:
770  npt = sum(good)
771 
772  center = None
773  if i == 0:
774  #
775  # Start by finding the mode
776  #
777  nhist = 20
778  try:
779  hist, edges = np.histogram(dmag, nhist, new=True)
780  except TypeError:
781  hist, edges = np.histogram(dmag, nhist) # they removed new=True around numpy 1.5
782  imode = np.arange(nhist)[np.where(hist == hist.max())]
783 
784  if imode[-1] - imode[0] + 1 == len(imode): # Multiple modes, but all contiguous
785  if zp0:
786  center = zp0
787  else:
788  center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
789 
790  peak = sum(hist[imode])/len(imode) # peak height
791 
792  # Estimate FWHM of mode
793  j = imode[0]
794  while j >= 0 and hist[j] > 0.5*peak:
795  j -= 1
796  j = max(j, 0)
797  q1 = dmag[sum(hist[range(j)])]
798 
799  j = imode[-1]
800  while j < nhist and hist[j] > 0.5*peak:
801  j += 1
802  j = min(j, nhist - 1)
803  j = min(sum(hist[range(j)]), npt - 1)
804  q3 = dmag[j]
805 
806  if q1 == q3:
807  q1 = dmag[int(0.25*npt)]
808  q3 = dmag[int(0.75*npt)]
809 
810  sig = (q3 - q1)/2.3 # estimate of standard deviation (based on FWHM; 2.358 for Gaussian)
811 
812  if sigmaMax is None:
813  sigmaMax = 2*sig # upper bound on st. dev. for clipping. multiplier is a heuristic
814 
815  self.log.debug("Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
816 
817  else:
818  if sigmaMax is None:
819  sigmaMax = dmag[-1] - dmag[0]
820 
821  center = np.median(dmag)
822  q1 = dmag[int(0.25*npt)]
823  q3 = dmag[int(0.75*npt)]
824  sig = (q3 - q1)/2.3 # estimate of standard deviation (based on FWHM; 2.358 for Gaussian)
825 
826  if center is None: # usually equivalent to (i > 0)
827  gdmag = dmag[good]
828  if self.config.useMedian:
829  center = np.median(gdmag)
830  else:
831  gdmagErr = dmagErr[good]
832  center = np.average(gdmag, weights=gdmagErr)
833 
834  q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
835  q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
836 
837  sig = IQ_TO_STDEV*(q3 - q1) # estimate of standard deviation
838 
839  good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax) # don't clip too softly
840 
841  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
842  if self.scatterPlot:
843  try:
844  self.fig.clf()
845 
846  axes = self.fig.add_axes((0.1, 0.1, 0.85, 0.80))
847 
848  axes.plot(ref[good], dmag[good] - center, "b+")
849  axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
850  linestyle='', color='b')
851 
852  bad = np.logical_not(good)
853  if len(ref[bad]) > 0:
854  axes.plot(ref[bad], dmag[bad] - center, "r+")
855  axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
856  linestyle='', color='r')
857 
858  axes.plot((-100, 100), (0, 0), "g-")
859  for x in (-1, 1):
860  axes.plot((-100, 100), x*0.05*np.ones(2), "g--")
861 
862  axes.set_ylim(-1.1, 1.1)
863  axes.set_xlim(24, 13)
864  axes.set_xlabel("Reference")
865  axes.set_ylabel("Reference - Instrumental")
866 
867  self.fig.show()
868 
869  if self.scatterPlot > 1:
870  reply = None
871  while i == 0 or reply != "c":
872  try:
873  reply = input("Next iteration? [ynhpc] ")
874  except EOFError:
875  reply = "n"
876 
877  if reply == "h":
878  print("Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
879  continue
880 
881  if reply in ("", "c", "n", "p", "y"):
882  break
883  else:
884  print("Unrecognised response: %s" % reply, file=sys.stderr)
885 
886  if reply == "n":
887  break
888  elif reply == "p":
889  import pdb
890  pdb.set_trace()
891  except Exception as e:
892  print("Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
893 
894  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
895 
896  old_ngood = ngood
897  ngood = sum(good)
898  if ngood == 0:
899  msg = "PhotoCal.getZeroPoint: no good stars remain"
900 
901  if i == 0: # failed the first time round -- probably all fell in one bin
902  center = np.average(dmag, weights=dmagErr)
903  msg += " on first iteration; using average of all calibration stars"
904 
905  self.log.warn(msg)
906 
907  return pipeBase.Struct(
908  zp=center,
909  sigma=sig,
910  ngood=len(dmag))
911  elif ngood == old_ngood:
912  break
913 
914  if False:
915  ref = ref[good]
916  dmag = dmag[good]
917  dmagErr = dmagErr[good]
918 
919  dmag = dmag[good]
920  dmagErr = dmagErr[good]
921  zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=True)
922  sigma = np.sqrt(1.0/weightSum)
923  return pipeBase.Struct(
924  zp=zp,
925  sigma=sigma,
926  ngood=len(dmag),
927  )
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
Definition: photoCal.py:297
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
Definition: photoCal.py:597
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
Definition: photoCal.py:315
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Definition: photoCal.py:482
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
Definition: photoCal.py:169
def checkSourceFlags(source, sourceKeys)
Return True if the given source has all good flags set and none of the bad flags set.
Definition: photoCal.py:47
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
Definition: photoCal.py:728
def isUnresolved(self, source, starGalKey=None)
Return whether the provided source is unresolved or not.
Definition: photoCal.py:336
def selectMatches(self, matches, sourceKeys, filterName, frame=None)
Select reference/source matches according the criteria specified in the config.
Definition: photoCal.py:353