27import astropy.units
as u
31from lsst.afw.image import abMagErrFromFluxErr, makePhotoCalibFromCalibZeroPoint
36from .colorterms
import ColortermLibrary
38__all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
42 """Config for PhotoCal"""
43 match = pexConf.ConfigField(
"Match to reference catalog",
44 DirectMatchConfigWithoutLoader)
45 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
46 fluxField = pexConf.Field(
48 default=
"slot_CalibFlux_instFlux",
49 doc=(
"Name of the source instFlux field to use. The associated flag field\n"
50 "('<name>_flags') will be implicitly included in badFlags."),
52 applyColorTerms = pexConf.Field(
55 doc=(
"Apply photometric color terms to reference stars? One of:\n"
56 "None: apply if colorterms and photoCatName are not None;\n"
57 " fail if color term data is not available for the specified ref catalog and filter.\n"
58 "True: always apply colorterms; fail if color term data is not available for the\n"
59 " specified reference catalog and filter.\n"
60 "False: do not apply."),
63 sigmaMax = pexConf.Field(
66 doc=
"maximum sigma to use when clipping",
69 nSigma = pexConf.Field(
74 useMedian = pexConf.Field(
77 doc=
"use median instead of mean to compute zeropoint",
79 nIter = pexConf.Field(
82 doc=
"number of iterations",
84 colorterms = pexConf.ConfigField(
85 dtype=ColortermLibrary,
86 doc=
"Library of photometric reference catalog name: color term dict",
88 photoCatName = pexConf.Field(
91 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms."
92 " see also applyColorTerms"),
94 magErrFloor = pexConf.RangeField(
97 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
102 pexConf.Config.validate(self)
104 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
106 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
109 pexConf.Config.setDefaults(self)
110 self.
match.sourceSelection.doFlags =
True
111 self.
match.sourceSelection.flags.bad = [
112 "base_PixelFlags_flag_edge",
113 "base_PixelFlags_flag_interpolated",
114 "base_PixelFlags_flag_saturated",
116 self.
match.sourceSelection.doUnresolved =
True
130@brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
132@section pipe_tasks_photocal_Contents Contents
134 - @ref pipe_tasks_photocal_Purpose
135 - @ref pipe_tasks_photocal_Initialize
136 - @ref pipe_tasks_photocal_IO
137 - @ref pipe_tasks_photocal_Config
138 - @ref pipe_tasks_photocal_Debug
139 - @ref pipe_tasks_photocal_Example
141@section pipe_tasks_photocal_Purpose Description
143@copybrief PhotoCalTask
145Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue.
146The type of flux to use is specified by PhotoCalConfig.fluxField.
148The algorithm clips outliers iteratively, with parameters set in the configuration.
150@note This task can adds fields to the schema, so any code calling this task must ensure that
151these columns are indeed present in the input match list; see @ref pipe_tasks_photocal_Example
153@section pipe_tasks_photocal_Initialize Task initialisation
157@section pipe_tasks_photocal_IO Inputs/Outputs to the run method
161@section pipe_tasks_photocal_Config Configuration parameters
163See
@ref PhotoCalConfig
165@section pipe_tasks_photocal_Debug Debug variables
167The
@link lsst.pipe.base.cmdLineTask.CmdLineTask command line task
@endlink interface supports a
168flag
@c -d to
import @b debug.py
from your
@c PYTHONPATH; see
@ref baseDebug
for more about
@b debug.py files.
170The available variables
in PhotoCalTask are:
173 <DD> If
True enable other debug outputs
174 <DT>
@c displaySources
175 <DD> If
True, display the exposure on the display
's frame 1 and overlay the source catalogue.
178 <DD> Reserved objects
180 <DD> Objects used in the photometric calibration
183 <DD> Make a scatter plot of flux v. reference magnitude
as a function of reference magnitude.
184 - good objects
in blue
185 - rejected objects
in red
186 (
if @c scatterPlot
is 2
or more, prompt to
continue after each iteration)
189@section pipe_tasks_photocal_Example A complete example of using PhotoCalTask
191This code
is in @link examples/photoCalTask.py
@endlink,
and can be run
as @em e.g.
193examples/photoCalTask.py
195@dontinclude photoCalTask.py
197Import the tasks (there are some other standard imports; read the file
for details)
198@skipline from lsst.pipe.tasks.astrometry
199@skipline measPhotocal
201We need to create both our tasks before processing any data
as the task constructors
202can add extra columns to the schema which we get
from the input catalogue,
@c scrCat:
206@skip AstrometryTask.ConfigClass
208(that
@c filterMap line
is because our test code doesn
't use a filter that the reference catalogue recognises,
209so we tell it to use the @c r band)
215If the schema has indeed changed we need to add the new columns to the source table
216(yes; this should be easier!)
220We
're now ready to process the data (we could loop over multiple exposures/catalogues using the same
225We can then unpack
and use the results:
230To investigate the
@ref pipe_tasks_photocal_Debug, put something like
235 if name.endswith(
".PhotoCal"):
242into your debug.py file
and run photoCalTask.py
with the
@c --debug flag.
244 ConfigClass = PhotoCalConfig
245 _DefaultName = "photoCal"
247 def __init__(self, refObjLoader, schema=None, **kwds):
248 """!Create the photometric calibration task. See PhotoCalTask.init for documentation
250 pipeBase.Task.__init__(self, **kwds)
253 if schema
is not None:
254 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
255 doc=
"set if source was used in photometric calibration")
258 self.
match = DirectMatchTask(config=self.config.match, refObjLoader=refObjLoader,
259 name=
"match", parentTask=self)
260 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
261 doc=
"set if source was reserved from photometric calibration")
264 """Return a struct containing the source catalog keys for fields used
270 schema : `lsst.afw.table.schema`
271 Schema of the catalog to get keys from.
275 result : `lsst.pipe.base.Struct`
276 Result struct
with components:
278 - ``instFlux``: Instrument flux key.
279 - ``instFluxErr``: Instrument flux error key.
281 instFlux = schema.find(self.config.fluxField).key
282 instFluxErr = schema.find(self.config.fluxField + "Err").key
283 return pipeBase.Struct(instFlux=instFlux, instFluxErr=instFluxErr)
287 """!Extract magnitude and magnitude error arrays from the given matches.
289 @param[
in] matches Reference/source matches, a
@link lsst::afw::table::ReferenceMatchVector
@endlink
290 @param[
in] filterLabel Label of filter being calibrated
291 @param[
in] sourceKeys Struct of source catalog keys,
as returned by
getSourceKeys()
293 @return Struct containing srcMag, refMag, srcMagErr, refMagErr,
and magErr numpy arrays
294 where magErr
is an error
in the magnitude; the error
in srcMag - refMag
295 If nonzero, config.magErrFloor will be added to magErr *only* (
not srcMagErr
or refMagErr),
as
296 magErr
is what
is later used to determine the zero point.
297 Struct also contains refFluxFieldList: a list of field names of the reference catalog used
for fluxes
299 @note These magnitude arrays are the
@em inputs to the photometric calibration, some may have been
300 discarded by clipping
while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
302 srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m
in matches])
303 srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr)
for m
in matches])
304 if not np.all(np.isfinite(srcInstFluxErrArr)):
306 self.log.warning(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
307 srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
310 referenceFlux = (0*u.ABmag).to_value(u.nJy)
311 srcInstFluxArr = srcInstFluxArr * referenceFlux
312 srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
315 raise RuntimeError(
"No reference stars are available")
316 refSchema = matches[0].first.schema
318 applyColorTerms = self.config.applyColorTerms
319 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
320 if self.config.applyColorTerms
is None:
322 ctDataAvail = len(self.config.colorterms.data) > 0
323 photoCatSpecified = self.config.photoCatName
is not None
324 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
325 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
326 applyColorTerms = ctDataAvail
and photoCatSpecified
329 self.log.info(
"Applying color terms for filter=%r, config.photoCatName=%s because %s",
330 filterLabel.physicalLabel, self.config.photoCatName, applyCTReason)
331 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
332 self.config.photoCatName,
334 refCat = afwTable.SimpleCatalog(matches[0].first.schema)
337 refCat.reserve(len(matches))
339 record = refCat.addNew()
340 record.assign(x.first)
342 refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat)
343 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (colorterm.primary,
344 colorterm.secondary)]
347 self.log.info(
"Not applying color terms because %s", applyCTReason)
350 fluxFieldList = [getRefFluxField(refSchema, filterLabel.bandLabel)]
351 fluxField = getRefFluxField(refSchema, filterLabel.bandLabel)
352 fluxKey = refSchema.find(fluxField).key
353 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
356 fluxErrKey = refSchema.find(fluxField +
"Err").key
357 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
360 self.log.warning(
"Reference catalog does not have flux uncertainties for %s;"
361 " using sqrt(flux).", fluxField)
362 refFluxErrArr = np.sqrt(refFluxArr)
364 refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
366 refMagErrArr = abMagErrFromFluxErr(refFluxErrArr*1e-9, refFluxArr*1e-9)
369 srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
373 magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
374 if self.config.magErrFloor != 0.0:
375 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
377 srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
379 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
381 return pipeBase.Struct(
382 srcMag=srcMagArr[good],
383 refMag=refMagArr[good],
384 magErr=magErrArr[good],
385 srcMagErr=srcMagErrArr[good],
386 refMagErr=refMagErrArr[good],
387 refFluxFieldList=fluxFieldList,
391 def run(self, exposure, sourceCat, expId=0):
392 """!Do photometric calibration - select matches to use and (possibly iteratively) compute
395 @param[
in] exposure Exposure upon which the sources
in the matches were detected.
396 @param[
in] sourceCat A catalog of sources to use
in the calibration
399 the reference object
and matched object respectively).
400 (will
not be modified
except to set the outputField
if requested.).
403 - photoCalib --
@link lsst::afw::image::PhotoCalib
@endlink object containing the calibration
404 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays
405 - matches ----- Final ReferenceMatchVector,
as returned by PhotoCalTask.selectMatches.
406 - zp ---------- Photometric zero point (mag)
407 - sigma ------- Standard deviation of fit of photometric zero point (mag)
408 - ngood ------- Number of sources used to fit photometric zero point
410 The exposure
is only used to provide the name of the filter being calibrated (it may also be
411 used to generate debugging plots).
413 The reference objects:
414 - Must include a field
@c photometric;
True for objects which should be considered
as
415 photometric standards
416 - Must include a field
@c flux; the flux used to impose a magnitude limit
and also to calibrate
417 the data to (unless a color term
is specified,
in which case ColorTerm.primary
is used;
418 See https://jira.lsstcorp.org/browse/DM-933)
419 - May include a field
@c stargal;
if present,
True means that the object
is a star
420 - May include a field
@c var;
if present,
True means that the object
is variable
422 The measured sources:
423 - Must include PhotoCalConfig.fluxField; the flux measurement to be used
for calibration
425 @throws RuntimeError
with the following strings:
428 <DT> No matches to use
for photocal
429 <DD> No matches are available (perhaps no sources/references were selected by the matcher).
430 <DT> No reference stars are available
431 <DD> No matches are available
from which to extract magnitudes.
437 displaySources = display
and lsstDebug.Info(__name__).displaySources
441 from matplotlib
import pyplot
445 self.
fig = pyplot.figure()
447 filterLabel = exposure.getFilterLabel()
450 matchResults = self.
match.
run(sourceCat, filterLabel.bandLabel)
451 matches = matchResults.matches
453 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
456 if reserveResults.reserved.sum() > 0:
457 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
458 if len(matches) == 0:
459 raise RuntimeError(
"No matches to use for photocal")
462 mm.second.set(self.
usedKey,
True)
469 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
470 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
473 flux0 = 10**(0.4*r.zp)
474 flux0err = 0.4*math.log(10)*flux0*r.sigma
475 photoCalib = makePhotoCalibFromCalibZeroPoint(flux0, flux0err)
477 return pipeBase.Struct(
478 photoCalib=photoCalib,
487 """Display sources we'll use for photocal
489 Sources that will be actually used will be green.
490 Sources reserved from the fit will be red.
494 exposure : `lsst.afw.image.ExposureF`
496 matches : `list` of `lsst.afw.table.RefMatch`
497 Matches used
for photocal.
498 reserved : `numpy.ndarray` of type `bool`
499 Boolean array indicating sources that are reserved.
501 Frame number
for display.
503 disp = afwDisplay.getDisplay(frame=frame)
504 disp.mtv(exposure, title="photocal")
505 with disp.Buffering():
506 for mm, rr
in zip(matches, reserved):
507 x, y = mm.second.getCentroid()
508 ctype = afwDisplay.RED
if rr
else afwDisplay.GREEN
509 disp.dot(
"o", x, y, size=4, ctype=ctype)
512 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars)
514 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists:
515 1. We use the median/interquartile range to estimate the position to clip around,
and the
517 2. We never allow sigma to go _above_ a critical value sigmaMax ---
if we do, a sufficiently
518 large estimate will prevent the clipping
from ever taking effect.
519 3. Rather than start
with the median we start
with a crude mode. This means that a set of magnitude
520 residuals
with a tight core
and asymmetrical outliers will start
in the core. We use the width of
521 this core to set our maximum sigma (see 2.)
524 - zp ---------- Photometric zero point (mag)
525 - sigma ------- Standard deviation of fit of zero point (mag)
526 - ngood ------- Number of sources used to fit zero point
528 sigmaMax = self.config.sigmaMax
532 indArr = np.argsort(dmag)
535 if srcErr
is not None:
536 dmagErr = srcErr[indArr]
538 dmagErr = np.ones(len(dmag))
541 ind_noNan = np.array([i
for i
in range(len(dmag))
542 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
543 dmag = dmag[ind_noNan]
544 dmagErr = dmagErr[ind_noNan]
546 IQ_TO_STDEV = 0.741301109252802
551 for i
in range(self.config.nIter):
562 hist, edges = np.histogram(dmag, nhist, new=
True)
564 hist, edges = np.histogram(dmag, nhist)
565 imode = np.arange(nhist)[np.where(hist == hist.max())]
567 if imode[-1] - imode[0] + 1 == len(imode):
571 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
573 peak = sum(hist[imode])/len(imode)
577 while j >= 0
and hist[j] > 0.5*peak:
580 q1 = dmag[sum(hist[range(j)])]
583 while j < nhist
and hist[j] > 0.5*peak:
585 j = min(j, nhist - 1)
586 j = min(sum(hist[range(j)]), npt - 1)
590 q1 = dmag[int(0.25*npt)]
591 q3 = dmag[int(0.75*npt)]
598 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
602 sigmaMax = dmag[-1] - dmag[0]
604 center = np.median(dmag)
605 q1 = dmag[int(0.25*npt)]
606 q3 = dmag[int(0.75*npt)]
611 if self.config.useMedian:
612 center = np.median(gdmag)
614 gdmagErr = dmagErr[good]
615 center = np.average(gdmag, weights=gdmagErr)
617 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
618 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
620 sig = IQ_TO_STDEV*(q3 - q1)
622 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
629 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
631 axes.plot(ref[good], dmag[good] - center,
"b+")
632 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
633 linestyle=
'', color=
'b')
635 bad = np.logical_not(good)
636 if len(ref[bad]) > 0:
637 axes.plot(ref[bad], dmag[bad] - center,
"r+")
638 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
639 linestyle=
'', color=
'r')
641 axes.plot((-100, 100), (0, 0),
"g-")
643 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
645 axes.set_ylim(-1.1, 1.1)
646 axes.set_xlim(24, 13)
647 axes.set_xlabel(
"Reference")
648 axes.set_ylabel(
"Reference - Instrumental")
654 while i == 0
or reply !=
"c":
656 reply = input(
"Next iteration? [ynhpc] ")
661 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
664 if reply
in (
"",
"c",
"n",
"p",
"y"):
667 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
674 except Exception
as e:
675 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
682 msg =
"PhotoCal.getZeroPoint: no good stars remain"
685 center = np.average(dmag, weights=dmagErr)
686 msg +=
" on first iteration; using average of all calibration stars"
688 self.log.warning(msg)
690 return pipeBase.Struct(
694 elif ngood == old_ngood:
700 dmagErr = dmagErr[good]
703 dmagErr = dmagErr[good]
704 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
705 sigma = np.sqrt(1.0/weightSum)
706 return pipeBase.Struct(
def getSourceKeys(self, schema)
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars)
def extractMagArrays(self, matches, filterLabel, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
def __init__(self, refObjLoader, schema=None, **kwds)
Create the photometric calibration task.
def displaySources(self, exposure, matches, reserved, frame=1)
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point.