lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
photoCal.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 # \package lsst.pipe.tasks.
23 
24 from __future__ import absolute_import, division, print_function
25 from builtins import zip
26 from builtins import input
27 from builtins import range
28 
29 import math
30 import sys
31 
32 import numpy as np
33 
34 import lsst.pex.config as pexConf
35 import lsst.pipe.base as pipeBase
36 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, fluxFromABMag, Calib
37 import lsst.afw.math as afwMath
38 from lsst.meas.astrom import RefMatchTask, RefMatchConfig
39 import lsst.afw.display.ds9 as ds9
40 from lsst.meas.algorithms import getRefFluxField
41 from .colorterms import ColortermLibrary
42 
43 __all__ = ["PhotoCalTask", "PhotoCalConfig"]
44 
45 
46 def checkSourceFlags(source, sourceKeys):
47  """!Return True if the given source has all good flags set and none of the bad flags set.
48 
49  \param[in] source SourceRecord object to process.
50  \param[in] sourceKeys Struct of source catalog keys, as returned by PhotCalTask.getSourceKeys()
51  """
52  for k in sourceKeys.goodFlags:
53  if not source.get(k):
54  return False
55  if source.getPsfFluxFlag():
56  return False
57  for k in sourceKeys.badFlags:
58  if source.get(k):
59  return False
60  return True
61 
62 
63 class PhotoCalConfig(RefMatchConfig):
64  """Config for PhotoCal"""
65  magLimit = pexConf.Field(
66  dtype=float,
67  default=22.0,
68  doc="Don't use objects fainter than this magnitude",
69  )
70  reserveFraction = pexConf.Field(
71  dtype=float,
72  doc="Fraction of candidates to reserve from fitting; none if <= 0",
73  default=-1.0,
74  )
75  reserveSeed = pexConf.Field(
76  dtype=int,
77  doc="This number will be multiplied by the exposure ID "
78  "to set the random seed for reserving candidates",
79  default=1,
80  )
81  fluxField = pexConf.Field(
82  dtype=str,
83  default="slot_CalibFlux_flux",
84  doc=("Name of the source flux field to use. The associated flag field\n"
85  "('<name>_flags') will be implicitly included in badFlags."),
86  )
87  applyColorTerms = pexConf.Field(
88  dtype=bool,
89  default=None,
90  doc=("Apply photometric color terms to reference stars? One of:\n"
91  "None: apply if colorterms and photoCatName are not None;\n"
92  " fail if color term data is not available for the specified ref catalog and filter.\n"
93  "True: always apply colorterms; fail if color term data is not available for the\n"
94  " specified reference catalog and filter.\n"
95  "False: do not apply."),
96  optional=True,
97  )
98  goodFlags = pexConf.ListField(
99  dtype=str,
100  default=[],
101  doc="List of source flag fields that must be set for a source to be used.",
102  )
103  badFlags = pexConf.ListField(
104  dtype=str,
105  default=["base_PixelFlags_flag_edge", "base_PixelFlags_flag_interpolated",
106  "base_PixelFlags_flag_saturated"],
107  doc="List of source flag fields that will cause a source to be rejected when they are set.",
108  )
109  sigmaMax = pexConf.Field(
110  dtype=float,
111  default=0.25,
112  doc="maximum sigma to use when clipping",
113  optional=True,
114  )
115  nSigma = pexConf.Field(
116  dtype=float,
117  default=3.0,
118  doc="clip at nSigma",
119  )
120  useMedian = pexConf.Field(
121  dtype=bool,
122  default=True,
123  doc="use median instead of mean to compute zeropoint",
124  )
125  nIter = pexConf.Field(
126  dtype=int,
127  default=20,
128  doc="number of iterations",
129  )
130  colorterms = pexConf.ConfigField(
131  dtype=ColortermLibrary,
132  doc="Library of photometric reference catalog name: color term dict",
133  )
134  photoCatName = pexConf.Field(
135  dtype=str,
136  optional=True,
137  doc=("Name of photometric reference catalog; used to select a color term dict in colorterms."
138  " see also applyColorTerms"),
139  )
140  magErrFloor = pexConf.RangeField(
141  dtype=float,
142  default=0.0,
143  doc="Additional magnitude uncertainty to be added in quadrature with measurement errors.",
144  min=0.0,
145  )
146  doSelectUnresolved = pexConf.Field(
147  dtype=bool,
148  default=True,
149  doc=("Use the extendedness parameter to select objects to use in photometric calibration?\n"
150  "This applies only to the sources detected on the exposure, not the reference catalog"),
151  )
152 
153  def validate(self):
154  pexConf.Config.validate(self)
155  if self.applyColorTerms and self.photoCatName is None:
156  raise RuntimeError("applyColorTerms=True requires photoCatName is non-None")
157  if self.applyColorTerms and len(self.colorterms.data) == 0:
158  raise RuntimeError("applyColorTerms=True requires colorterms be provided")
159 
160 
161 
167 
168 class PhotoCalTask(RefMatchTask):
169  r"""!
170 \anchor PhotoCalTask_
171 
172 \brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
173 
174 \section pipe_tasks_photocal_Contents Contents
175 
176  - \ref pipe_tasks_photocal_Purpose
177  - \ref pipe_tasks_photocal_Initialize
178  - \ref pipe_tasks_photocal_IO
179  - \ref pipe_tasks_photocal_Config
180  - \ref pipe_tasks_photocal_Debug
181  - \ref pipe_tasks_photocal_Example
182 
183 \section pipe_tasks_photocal_Purpose Description
184 
185 \copybrief PhotoCalTask
186 
187 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue.
188 The type of flux to use is specified by PhotoCalConfig.fluxField.
189 
190 The algorithm clips outliers iteratively, with parameters set in the configuration.
191 
192 \note This task can adds fields to the schema, so any code calling this task must ensure that
193 these columns are indeed present in the input match list; see \ref pipe_tasks_photocal_Example
194 
195 \section pipe_tasks_photocal_Initialize Task initialisation
196 
197 \copydoc \_\_init\_\_
198 
199 \section pipe_tasks_photocal_IO Inputs/Outputs to the run method
200 
201 \copydoc run
202 
203 \section pipe_tasks_photocal_Config Configuration parameters
204 
205 See \ref PhotoCalConfig
206 
207 \section pipe_tasks_photocal_Debug Debug variables
208 
209 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
210 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
211 
212 The available variables in PhotoCalTask are:
213 <DL>
214  <DT> \c display
215  <DD> If True enable other debug outputs
216  <DT> \c displaySources
217  <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue:
218  <DL>
219  <DT> red x
220  <DD> Bad objects
221  <DT> blue +
222  <DD> Matched objects deemed unsuitable for photometric calibration.
223  Additional information is:
224  - a cyan o for galaxies
225  - a magenta o for variables
226  <DT> magenta *
227  <DD> Objects that failed the flux cut
228  <DT> green o
229  <DD> Objects used in the photometric calibration
230  </DL>
231  <DT> \c scatterPlot
232  <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude.
233  - good objects in blue
234  - rejected objects in red
235  (if \c scatterPlot is 2 or more, prompt to continue after each iteration)
236 </DL>
237 
238 \section pipe_tasks_photocal_Example A complete example of using PhotoCalTask
239 
240 This code is in \link examples/photoCalTask.py\endlink, and can be run as \em e.g.
241 \code
242 examples/photoCalTask.py
243 \endcode
244 \dontinclude photoCalTask.py
245 
246 Import the tasks (there are some other standard imports; read the file for details)
247 \skipline from lsst.pipe.tasks.astrometry
248 \skipline measPhotocal
249 
250 We need to create both our tasks before processing any data as the task constructors
251 can add extra columns to the schema which we get from the input catalogue, \c scrCat:
252 \skipline getSchema
253 
254 Astrometry first:
255 \skip AstrometryTask.ConfigClass
256 \until aTask
257 (that \c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises,
258 so we tell it to use the \c r band)
259 
260 Then photometry:
261 \skip measPhotocal
262 \until pTask
263 
264 If the schema has indeed changed we need to add the new columns to the source table
265 (yes; this should be easier!)
266 \skip srcCat
267 \until srcCat = cat
268 
269 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same
270 task objects):
271 \skip matches
272 \until result
273 
274 We can then unpack and use the results:
275 \skip calib
276 \until np.log
277 
278 <HR>
279 To investigate the \ref pipe_tasks_photocal_Debug, put something like
280 \code{.py}
281  import lsstDebug
282  def DebugInfo(name):
283  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
284  if name.endswith(".PhotoCal"):
285  di.display = 1
286 
287  return di
288 
289  lsstDebug.Info = DebugInfo
290 \endcode
291 into your debug.py file and run photoCalTask.py with the \c --debug flag.
292  """
293  ConfigClass = PhotoCalConfig
294  _DefaultName = "photoCal"
295 
296  def __init__(self, refObjLoader, schema=None, **kwds):
297  """!Create the photometric calibration task. See PhotoCalTask.init for documentation
298  """
299  RefMatchTask.__init__(self, refObjLoader, schema=None, **kwds)
300  self.scatterPlot = None
301  self.fig = None
302  if schema is not None:
303  self.usedKey = schema.addField("calib_photometryUsed", type="Flag",
304  doc="set if source was used in photometric calibration")
305  self.candidateKey = schema.addField("calib_photometryCandidate", type="Flag",
306  doc="set if source was a candidate for use in calibration")
307  self.reservedKey = schema.addField("calib_photometryReserved", type="Flag",
308  doc="set if source was reserved, so not used in calibration")
309  else:
310  self.usedKey = None
311  self.candidateKey = None
312  self.reservedKey = None
313 
314  def getSourceKeys(self, schema):
315  """!Return a struct containing the source catalog keys for fields used by PhotoCalTask.
316 
317  Returned fields include:
318  - flux
319  - fluxErr
320  - goodFlags: a list of keys for field names in self.config.goodFlags
321  - badFlags: a list of keys for field names in self.config.badFlags
322  - starGal: key for star/galaxy classification
323  """
324  goodFlags = [schema.find(name).key for name in self.config.goodFlags]
325  flux = schema.find(self.config.fluxField).key
326  fluxErr = schema.find(self.config.fluxField + "Sigma").key
327  badFlags = [schema.find(name).key for name in self.config.badFlags]
328  try:
329  starGal = schema.find("base_ClassificationExtendedness_value").key
330  except KeyError:
331  starGal = None
332  return pipeBase.Struct(flux=flux, fluxErr=fluxErr, goodFlags=goodFlags, badFlags=badFlags,
333  starGal=starGal)
334 
335  def isUnresolved(self, source, starGalKey=None):
336  """!Return whether the provided source is unresolved or not
337 
338  This particular implementation is designed to work with the
339  base_ClassificationExtendedness_value=0.0 or 1.0 scheme. Because
340  of the diversity of star/galaxy classification outputs (binary
341  decision vs probabilities; signs), it's difficult to make this
342  configurable without using code. This method should therefore
343  be overridden to use the appropriate classification output.
344 
345  \param[in] source Source to test
346  \param[in] starGalKey Struct of schema keys for source
347  \return boolean value for starGalKey (True indicates Unresolved)
348  """
349  return source.get(starGalKey) < 0.5 if starGalKey is not None else True
350 
351  @pipeBase.timeMethod
352  def selectMatches(self, matches, sourceKeys, filterName, frame=None):
353  """!Select reference/source matches according the criteria specified in the config.
354 
355  \param[in] matches ReferenceMatchVector (not modified)
356  \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
357  \param[in] filterName name of camera filter; used to obtain the reference flux field
358  \param[in] frame ds9 frame number to use for debugging display
359  if frame is non-None, display information about trimmed objects on that ds9 frame:
360  - Bad: red x
361  - Unsuitable objects: blue + (and a cyan o if a galaxy)
362  - Failed flux cut: magenta *
363 
364  \return a \link lsst.afw.table.ReferenceMatchVector\endlink that contains only the selected matches.
365  If a schema was passed during task construction, a flag field will be set on sources
366  in the selected matches.
367 
368  \throws ValueError There are no valid matches.
369  """
370  self.log.debug("Number of input matches: %d", len(matches))
371 
372  if self.config.doSelectUnresolved:
373  # Select only resolved sources if asked to do so.
374  matches = [m for m in matches if self.isUnresolved(m.second, sourceKeys.starGal)]
375  self.log.debug("Number of matches after culling resolved sources: %d", len(matches))
376 
377  if len(matches) == 0:
378  raise ValueError("No input matches")
379 
380  for m in matches:
381  if self.candidateKey is not None:
382  m.second.set(self.candidateKey, True)
383 
384  # Only use stars for which the flags indicate the photometry is good.
385  afterFlagCutInd = [i for i, m in enumerate(matches) if checkSourceFlags(m.second, sourceKeys)]
386  afterFlagCut = [matches[i] for i in afterFlagCutInd]
387  self.log.debug("Number of matches after source flag cuts: %d", len(afterFlagCut))
388 
389  if len(afterFlagCut) != len(matches):
390  if frame is not None:
391  with ds9.Buffering():
392  for i, m in enumerate(matches):
393  if i not in afterFlagCutInd:
394  x, y = m.second.getCentroid()
395  ds9.dot("x", x, y, size=4, frame=frame, ctype=ds9.RED)
396 
397  matches = afterFlagCut
398 
399  if len(matches) == 0:
400  raise ValueError("All matches eliminated by source flags")
401 
402  refSchema = matches[0].first.schema
403  try:
404  photometricKey = refSchema.find("photometric").key
405  try:
406  resolvedKey = refSchema.find("resolved").key
407  except:
408  resolvedKey = None
409 
410  try:
411  variableKey = refSchema.find("variable").key
412  except:
413  variableKey = None
414  except:
415  self.log.warn("No 'photometric' flag key found in reference schema.")
416  photometricKey = None
417 
418  if photometricKey is not None:
419  afterRefCutInd = [i for i, m in enumerate(matches) if m.first.get(photometricKey)]
420  afterRefCut = [matches[i] for i in afterRefCutInd]
421 
422  if len(afterRefCut) != len(matches):
423  if frame is not None:
424  with ds9.Buffering():
425  for i, m in enumerate(matches):
426  if i not in afterRefCutInd:
427  x, y = m.second.getCentroid()
428  ds9.dot("+", x, y, size=4, frame=frame, ctype=ds9.BLUE)
429 
430  if resolvedKey and m.first.get(resolvedKey):
431  ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.CYAN)
432  if variableKey and m.first.get(variableKey):
433  ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.MAGENTA)
434 
435  matches = afterRefCut
436 
437  self.log.debug("Number of matches after reference catalog cuts: %d", len(matches))
438  if len(matches) == 0:
439  raise RuntimeError("No sources remain in match list after reference catalog cuts.")
440  fluxName = getRefFluxField(refSchema, filterName)
441  fluxKey = refSchema.find(fluxName).key
442  if self.config.magLimit is not None:
443  fluxLimit = fluxFromABMag(self.config.magLimit)
444 
445  afterMagCutInd = [i for i, m in enumerate(matches) if (m.first.get(fluxKey) > fluxLimit and
446  m.second.getPsfFlux() > 0.0)]
447  else:
448  afterMagCutInd = [i for i, m in enumerate(matches) if m.second.getPsfFlux() > 0.0]
449 
450  afterMagCut = [matches[i] for i in afterMagCutInd]
451 
452  if len(afterMagCut) != len(matches):
453  if frame is not None:
454  with ds9.Buffering():
455  for i, m in enumerate(matches):
456  if i not in afterMagCutInd:
457  x, y = m.second.getCentroid()
458  ds9.dot("*", x, y, size=4, frame=frame, ctype=ds9.MAGENTA)
459 
460  matches = afterMagCut
461 
462  self.log.debug("Number of matches after magnitude limit cuts: %d", len(matches))
463 
464  if len(matches) == 0:
465  raise RuntimeError("No sources remaining in match list after magnitude limit cuts.")
466 
467  if frame is not None:
468  with ds9.Buffering():
469  for m in matches:
470  x, y = m.second.getCentroid()
471  ds9.dot("o", x, y, size=4, frame=frame, ctype=ds9.GREEN)
472 
473  result = []
474  for m in matches:
475  if self.usedKey is not None:
476  m.second.set(self.usedKey, True)
477  result.append(m)
478  return result
479 
480  @pipeBase.timeMethod
481  def extractMagArrays(self, matches, filterName, sourceKeys):
482  """!Extract magnitude and magnitude error arrays from the given matches.
483 
484  \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink
485  \param[in] filterName Name of filter being calibrated
486  \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
487 
488  \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
489  where magErr is an error in the magnitude; the error in srcMag - refMag
490  If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
491  magErr is what is later used to determine the zero point.
492  Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
493  (1 or 2 strings)
494  \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been
495  discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
496  """
497  srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
498  srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches])
499  if not np.all(np.isfinite(srcFluxErrArr)):
500  # this is an unpleasant hack; see DM-2308 requesting a better solution
501  self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
502  srcFluxErrArr = np.sqrt(srcFluxArr)
503 
504  # convert source flux from DN to an estimate of Jy
505  JanskysPerABFlux = 3631.0
506  srcFluxArr = srcFluxArr * JanskysPerABFlux
507  srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
508 
509  if not matches:
510  raise RuntimeError("No reference stars are available")
511  refSchema = matches[0].first.schema
512 
513  applyColorTerms = self.config.applyColorTerms
514  applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
515  if self.config.applyColorTerms is None:
516  # apply color terms if color term data is available and photoCatName specified
517  ctDataAvail = len(self.config.colorterms.data) > 0
518  photoCatSpecified = self.config.photoCatName is not None
519  applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
520  applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is")
521  applyColorTerms = ctDataAvail and photoCatSpecified
522 
523  if applyColorTerms:
524  self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s",
525  filterName, self.config.photoCatName, applyCTReason)
526  ct = self.config.colorterms.getColorterm(
527  filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
528  else:
529  self.log.info("Not applying color terms because %s", applyCTReason)
530  ct = None
531 
532  if ct: # we have a color term to worry about
533  fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)]
534  missingFluxFieldList = []
535  for fluxField in fluxFieldList:
536  try:
537  refSchema.find(fluxField).key
538  except KeyError:
539  missingFluxFieldList.append(fluxField)
540 
541  if missingFluxFieldList:
542  self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms",
543  " ".join(missingFluxFieldList))
544  ct = None
545 
546  if not ct:
547  fluxFieldList = [getRefFluxField(refSchema, filterName)]
548 
549  refFluxArrList = [] # list of ref arrays, one per flux field
550  refFluxErrArrList = [] # list of ref flux arrays, one per flux field
551  for fluxField in fluxFieldList:
552  fluxKey = refSchema.find(fluxField).key
553  refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
554  try:
555  fluxErrKey = refSchema.find(fluxField + "Sigma").key
556  refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
557  except KeyError:
558  # Reference catalogue may not have flux uncertainties; HACK
559  self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
560  fluxField)
561  refFluxErrArr = np.sqrt(refFluxArr)
562 
563  refFluxArrList.append(refFluxArr)
564  refFluxErrArrList.append(refFluxErrArr)
565 
566  if ct: # we have a color term to worry about
567  refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
568  refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
569 
570  refMagArr = ct.transformMags(refMagArr1, refMagArr2)
571  refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
572  else:
573  refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]])
574 
575  srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
576 
577  # Fitting with error bars in both axes is hard
578  # for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
579  magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in zip(srcFluxErrArr, srcFluxArr)])
580  if self.config.magErrFloor != 0.0:
581  magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
582 
583  srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in zip(srcFluxErrArr, srcFluxArr)])
584  refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in zip(refFluxErrArr, refFluxArr)])
585 
586  return pipeBase.Struct(
587  srcMag=srcMagArr,
588  refMag=refMagArr,
589  magErr=magErrArr,
590  srcMagErr=srcMagErrArr,
591  refMagErr=refMagErrArr,
592  refFluxFieldList=fluxFieldList,
593  )
594 
595  @pipeBase.timeMethod
596  def run(self, exposure, sourceCat, expId=0):
597  """!Do photometric calibration - select matches to use and (possibly iteratively) compute
598  the zero point.
599 
600  \param[in] exposure Exposure upon which the sources in the matches were detected.
601  \param[in] sourceCat A catalog of sources to use in the calibration
602  (\em i.e. a list of lsst.afw.table.Match with
603  \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord ---
604  the reference object and matched object respectively).
605  (will not be modified except to set the outputField if requested.).
606 
607  \return Struct of:
608  - calib ------- \link lsst::afw::image::Calib\endlink object containing the zero point
609  - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays
610  - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches.
611  - zp ---------- Photometric zero point (mag)
612  - sigma ------- Standard deviation of fit of photometric zero point (mag)
613  - ngood ------- Number of sources used to fit photometric zero point
614 
615  The exposure is only used to provide the name of the filter being calibrated (it may also be
616  used to generate debugging plots).
617 
618  The reference objects:
619  - Must include a field \c photometric; True for objects which should be considered as
620  photometric standards
621  - Must include a field \c flux; the flux used to impose a magnitude limit and also to calibrate
622  the data to (unless a color term is specified, in which case ColorTerm.primary is used;
623  See https://jira.lsstcorp.org/browse/DM-933)
624  - May include a field \c stargal; if present, True means that the object is a star
625  - May include a field \c var; if present, True means that the object is variable
626 
627  The measured sources:
628  - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration
629 
630  \throws RuntimeError with the following strings:
631 
632  <DL>
633  <DT> `sources' schema does not contain the calibration object flag "XXX"`
634  <DD> The constructor added fields to the schema that aren't in the Sources
635  <DT> No input matches
636  <DD> The input match vector is empty
637  <DT> All matches eliminated by source flags
638  <DD> The flags specified by \c badFlags in the config eliminated all candidate objects
639  <DT> No sources remain in match list after reference catalog cuts
640  <DD> The reference catalogue has a column "photometric", but no matched objects have it set
641  <DT> No sources remaining in match list after magnitude limit cuts
642  <DD> All surviving matches are either too faint in the catalogue or have negative or \c NaN flux
643  <DT> No reference stars are available
644  <DD> No matches survive all the checks
645  </DL>
646  """
647  import lsstDebug
648 
649  display = lsstDebug.Info(__name__).display
650  displaySources = display and lsstDebug.Info(__name__).displaySources
651  self.scatterPlot = display and lsstDebug.Info(__name__).scatterPlot
652 
653  if self.scatterPlot:
654  from matplotlib import pyplot
655  try:
656  self.fig.clf()
657  except:
658  self.fig = pyplot.figure()
659 
660  if displaySources:
661  frame = 1
662  ds9.mtv(exposure, frame=frame, title="photocal")
663  else:
664  frame = None
665 
666  res = self.loadAndMatch(exposure, sourceCat)
667 
668  # from res.matches, reserve a fraction of the sources from res.matches and mark the sources reserved
669 
670  if self.config.reserveFraction > 0:
671  # Note that the seed can't be set to 0, so guard against an improper expId.
672  random = afwMath.Random(seed=self.config.reserveSeed*(expId if expId else 1))
673  reserveList = []
674  n = len(res.matches)
675  for i in range(int(n*self.config.reserveFraction)):
676  index = random.uniformInt(n)
677  n -= 1
678  candidate = res.matches[index]
679  res.matches.remove(candidate)
680  reserveList.append(candidate)
681 
682  if reserveList and self.reservedKey is not None:
683  for candidate in reserveList:
684  candidate.second.set(self.reservedKey, True)
685 
686  matches = res.matches
687  for m in matches:
688  if self.candidateKey is not None:
689  m.second.set(self.candidateKey, True)
690 
691  filterName = exposure.getFilter().getName()
692  sourceKeys = self.getSourceKeys(matches[0].second.schema)
693 
694  matches = self.selectMatches(matches=matches, sourceKeys=sourceKeys, filterName=filterName,
695  frame=frame)
696  arrays = self.extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
697 
698  if matches and self.usedKey:
699  try:
700  # matches[].second is a measured source, wherein we wish to set outputField.
701  # Check that the field is present in the Sources schema.
702  matches[0].second.getSchema().find(self.usedKey)
703  except:
704  raise RuntimeError("sources' schema does not contain the calib_photometryUsed flag \"%s\"" %
705  self.usedKey)
706 
707  # Fit for zeropoint. We can run the code more than once, so as to
708  # give good stars that got clipped by a bad first guess a second
709  # chance.
710 
711  calib = Calib()
712  zp = None # initial guess
713  r = self.getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr, zp0=zp)
714  zp = r.zp
715  self.log.info("Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
716 
717  flux0 = 10**(0.4*r.zp) # Flux of mag=0 star
718  flux0err = 0.4*math.log(10)*flux0*r.sigma # Error in flux0
719 
720  calib.setFluxMag0(flux0, flux0err)
721 
722  return pipeBase.Struct(
723  calib=calib,
724  arrays=arrays,
725  matches=matches,
726  zp=r.zp,
727  sigma=r.sigma,
728  ngood=r.ngood,
729  )
730 
731  def getZeroPoint(self, src, ref, srcErr=None, zp0=None):
732  """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars)
733 
734  We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists:
735  1. We use the median/interquartile range to estimate the position to clip around, and the
736  "sigma" to use.
737  2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently
738  large estimate will prevent the clipping from ever taking effect.
739  3. Rather than start with the median we start with a crude mode. This means that a set of magnitude
740  residuals with a tight core and asymmetrical outliers will start in the core. We use the width of
741  this core to set our maximum sigma (see 2.)
742 
743  \return Struct of:
744  - zp ---------- Photometric zero point (mag)
745  - sigma ------- Standard deviation of fit of zero point (mag)
746  - ngood ------- Number of sources used to fit zero point
747  """
748  sigmaMax = self.config.sigmaMax
749 
750  dmag = ref - src
751 
752  indArr = np.argsort(dmag)
753  dmag = dmag[indArr]
754 
755  if srcErr is not None:
756  dmagErr = srcErr[indArr]
757  else:
758  dmagErr = np.ones(len(dmag))
759 
760  # need to remove nan elements to avoid errors in stats calculation with numpy
761  ind_noNan = np.array([i for i in range(len(dmag))
762  if (not np.isnan(dmag[i]) and not np.isnan(dmagErr[i]))])
763  dmag = dmag[ind_noNan]
764  dmagErr = dmagErr[ind_noNan]
765 
766  IQ_TO_STDEV = 0.741301109252802 # 1 sigma in units of interquartile (assume Gaussian)
767 
768  npt = len(dmag)
769  ngood = npt
770  good = None # set at end of first iteration
771  for i in range(self.config.nIter):
772  if i > 0:
773  npt = sum(good)
774 
775  center = None
776  if i == 0:
777  #
778  # Start by finding the mode
779  #
780  nhist = 20
781  try:
782  hist, edges = np.histogram(dmag, nhist, new=True)
783  except TypeError:
784  hist, edges = np.histogram(dmag, nhist) # they removed new=True around numpy 1.5
785  imode = np.arange(nhist)[np.where(hist == hist.max())]
786 
787  if imode[-1] - imode[0] + 1 == len(imode): # Multiple modes, but all contiguous
788  if zp0:
789  center = zp0
790  else:
791  center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
792 
793  peak = sum(hist[imode])/len(imode) # peak height
794 
795  # Estimate FWHM of mode
796  j = imode[0]
797  while j >= 0 and hist[j] > 0.5*peak:
798  j -= 1
799  j = max(j, 0)
800  q1 = dmag[sum(hist[range(j)])]
801 
802  j = imode[-1]
803  while j < nhist and hist[j] > 0.5*peak:
804  j += 1
805  j = min(j, nhist - 1)
806  j = min(sum(hist[range(j)]), npt - 1)
807  q3 = dmag[j]
808 
809  if q1 == q3:
810  q1 = dmag[int(0.25*npt)]
811  q3 = dmag[int(0.75*npt)]
812 
813  sig = (q3 - q1)/2.3 # estimate of standard deviation (based on FWHM; 2.358 for Gaussian)
814 
815  if sigmaMax is None:
816  sigmaMax = 2*sig # upper bound on st. dev. for clipping. multiplier is a heuristic
817 
818  self.log.debug("Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
819 
820  else:
821  if sigmaMax is None:
822  sigmaMax = dmag[-1] - dmag[0]
823 
824  center = np.median(dmag)
825  q1 = dmag[int(0.25*npt)]
826  q3 = dmag[int(0.75*npt)]
827  sig = (q3 - q1)/2.3 # estimate of standard deviation (based on FWHM; 2.358 for Gaussian)
828 
829  if center is None: # usually equivalent to (i > 0)
830  gdmag = dmag[good]
831  if self.config.useMedian:
832  center = np.median(gdmag)
833  else:
834  gdmagErr = dmagErr[good]
835  center = np.average(gdmag, weights=gdmagErr)
836 
837  q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
838  q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
839 
840  sig = IQ_TO_STDEV*(q3 - q1) # estimate of standard deviation
841 
842  good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax) # don't clip too softly
843 
844  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
845  if self.scatterPlot:
846  try:
847  self.fig.clf()
848 
849  axes = self.fig.add_axes((0.1, 0.1, 0.85, 0.80))
850 
851  axes.plot(ref[good], dmag[good] - center, "b+")
852  axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
853  linestyle='', color='b')
854 
855  bad = np.logical_not(good)
856  if len(ref[bad]) > 0:
857  axes.plot(ref[bad], dmag[bad] - center, "r+")
858  axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
859  linestyle='', color='r')
860 
861  axes.plot((-100, 100), (0, 0), "g-")
862  for x in (-1, 1):
863  axes.plot((-100, 100), x*0.05*np.ones(2), "g--")
864 
865  axes.set_ylim(-1.1, 1.1)
866  axes.set_xlim(24, 13)
867  axes.set_xlabel("Reference")
868  axes.set_ylabel("Reference - Instrumental")
869 
870  self.fig.show()
871 
872  if self.scatterPlot > 1:
873  reply = None
874  while i == 0 or reply != "c":
875  try:
876  reply = input("Next iteration? [ynhpc] ")
877  except EOFError:
878  reply = "n"
879 
880  if reply == "h":
881  print("Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
882  continue
883 
884  if reply in ("", "c", "n", "p", "y"):
885  break
886  else:
887  print("Unrecognised response: %s" % reply, file=sys.stderr)
888 
889  if reply == "n":
890  break
891  elif reply == "p":
892  import pdb
893  pdb.set_trace()
894  except Exception as e:
895  print("Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
896 
897  # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
898 
899  old_ngood = ngood
900  ngood = sum(good)
901  if ngood == 0:
902  msg = "PhotoCal.getZeroPoint: no good stars remain"
903 
904  if i == 0: # failed the first time round -- probably all fell in one bin
905  center = np.average(dmag, weights=dmagErr)
906  msg += " on first iteration; using average of all calibration stars"
907 
908  self.log.warn(msg)
909 
910  return pipeBase.Struct(
911  zp=center,
912  sigma=sig,
913  ngood=len(dmag))
914  elif ngood == old_ngood:
915  break
916 
917  if False:
918  ref = ref[good]
919  dmag = dmag[good]
920  dmagErr = dmagErr[good]
921 
922  dmag = dmag[good]
923  dmagErr = dmagErr[good]
924  zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=True)
925  sigma = np.sqrt(1.0/weightSum)
926  return pipeBase.Struct(
927  zp=zp,
928  sigma=sigma,
929  ngood=len(dmag),
930  )
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
Definition: photoCal.py:296
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
Definition: photoCal.py:596
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
Definition: photoCal.py:314
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Definition: photoCal.py:481
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
Definition: photoCal.py:168
def checkSourceFlags(source, sourceKeys)
Return True if the given source has all good flags set and none of the bad flags set.
Definition: photoCal.py:46
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
Definition: photoCal.py:731
def isUnresolved(self, source, starGalKey=None)
Return whether the provided source is unresolved or not.
Definition: photoCal.py:335
def selectMatches(self, matches, sourceKeys, filterName, frame=None)
Select reference/source matches according the criteria specified in the config.
Definition: photoCal.py:352