22__all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
28import astropy.units
as u
31import lsst.pipe.base
as pipeBase
32from lsst.afw.image import abMagErrFromFluxErr, makePhotoCalibFromCalibZeroPoint
37from lsst.utils.timer
import timeMethod
38from .colorterms
import ColortermLibrary
42 """Config for PhotoCal."""
44 match = pexConf.ConfigField(
"Match to reference catalog",
45 DirectMatchConfigWithoutLoader)
46 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
47 fluxField = pexConf.Field(
49 default=
"slot_CalibFlux_instFlux",
50 doc=(
"Name of the source instFlux field to use.\nThe associated flag field "
51 "('<name>_flags') will be implicitly included in badFlags."),
53 applyColorTerms = pexConf.Field(
56 doc=(
"Apply photometric color terms to reference stars?\n"
57 "`True`: attempt to apply color terms; fail if color term data is "
58 "not available for the specified reference catalog and filter.\n"
59 "`False`: do not apply color terms."),
62 sigmaMax = pexConf.Field(
65 doc=
"maximum sigma to use when clipping",
68 nSigma = pexConf.Field(
73 useMedian = pexConf.Field(
76 doc=
"use median instead of mean to compute zeropoint",
78 nIter = pexConf.Field(
81 doc=
"number of iterations",
83 colorterms = pexConf.ConfigField(
84 dtype=ColortermLibrary,
85 doc=
"Library of photometric reference catalog name: color term dict (see also applyColorTerms).",
87 photoCatName = pexConf.Field(
90 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms.\n"
91 "See also applyColorTerms."),
93 magErrFloor = pexConf.RangeField(
96 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
101 pexConf.Config.validate(self)
103 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
105 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
108 pexConf.Config.setDefaults(self)
109 self.
match.sourceSelection.doRequirePrimary =
True
110 self.
match.sourceSelection.doFlags =
True
111 self.
match.sourceSelection.flags.bad = [
112 "base_PixelFlags_flag_edge",
113 "base_PixelFlags_flag_interpolated",
114 "base_PixelFlags_flag_saturated",
116 self.
match.sourceSelection.doUnresolved =
True
120 """Calculate an Exposure's zero-point given a set of flux measurements
121 of stars matched to an input catalogue.
125 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
126 An instance of LoadReferenceObjectsTasks that supplies an external reference
129 The schema of the detection catalogs used as input to this task.
131 Additional keyword arguments.
135 The type of flux to use
is specified by PhotoCalConfig.fluxField.
137 The algorithm clips outliers iteratively,
with parameters set
in the configuration.
139 This task can adds fields to the schema, so any code calling this task must ensure that
140 these columns are indeed present
in the input match list; see `pipe_tasks_photocal_Example`.
144 The available `~lsst.base.lsstDebug` variables
in PhotoCalTask are:
147 If
True enable other debug outputs.
149 If
True, display the exposure on ds9
's frame 1 and overlay the source catalogue.
154 Objects used in the photometric calibration.
157 Make a scatter plot of flux v. reference magnitude
as a function of reference magnitude:
159 - good objects
in blue
160 - rejected objects
in red
162 (
if scatterPlot
is 2
or more, prompt to
continue after each iteration)
165 ConfigClass = PhotoCalConfig
166 _DefaultName = "photoCal"
168 def __init__(self, refObjLoader, schema=None, **kwds):
169 pipeBase.Task.__init__(self, **kwds)
172 if schema
is not None:
173 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
174 doc=
"set if source was used in photometric calibration")
177 self.
match = DirectMatchTask(config=self.config.match, refObjLoader=refObjLoader,
178 name=
"match", parentTask=self)
179 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
180 doc=
"set if source was reserved from photometric calibration")
183 """Return a struct containing the source catalog keys for fields used
188 schema : `lsst.afw.table.schema`
189 Schema of the catalog to get keys from.
193 result : `lsst.pipe.base.Struct`
194 Results
as a struct
with attributes:
199 Instrument flux error key.
201 instFlux = schema.find(self.config.fluxField).key
202 instFluxErr = schema.find(self.config.fluxField + "Err").key
203 return pipeBase.Struct(instFlux=instFlux, instFluxErr=instFluxErr)
207 """Extract magnitude and magnitude error arrays from the given matches.
212 Reference/source matches.
214 Label of filter being calibrated.
215 sourceKeys : `lsst.pipe.base.Struct`
216 Struct of source catalog keys, as returned by
getSourceKeys().
220 result : `lsst.pipe.base.Struct`
221 Results
as a struct
with attributes:
224 Source magnitude (`np.array`).
226 Reference magnitude (`np.array`).
228 Source magnitude error (`np.array`).
230 Reference magnitude error (`np.array`).
232 An error
in the magnitude; the error
in ``srcMag`` - ``refMag``.
233 If nonzero, ``config.magErrFloor`` will be added to ``magErr`` only
234 (
not ``srcMagErr``
or ``refMagErr``),
as
235 ``magErr``
is what
is later used to determine the zero point (`np.array`).
237 A list of field names of the reference catalog used
for fluxes (1
or 2 strings) (`list`).
239 srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m
in matches])
240 srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr)
for m
in matches])
241 if not np.all(np.isfinite(srcInstFluxErrArr)):
243 self.log.warning(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
244 srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
247 referenceFlux = (0*u.ABmag).to_value(u.nJy)
248 srcInstFluxArr = srcInstFluxArr * referenceFlux
249 srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
252 raise RuntimeError(
"No reference stars are available")
253 refSchema = matches[0].first.schema
255 if self.config.applyColorTerms:
256 self.log.info(
"Applying color terms for filter=%r, config.photoCatName=%s",
257 filterLabel.physicalLabel, self.config.photoCatName)
258 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
259 self.config.photoCatName,
261 refCat = afwTable.SimpleCatalog(matches[0].first.schema)
264 refCat.reserve(len(matches))
266 record = refCat.addNew()
267 record.assign(x.first)
269 refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat)
270 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (colorterm.primary,
271 colorterm.secondary)]
273 self.log.info(
"Not applying color terms.")
276 fluxFieldList = [getRefFluxField(refSchema, filterLabel.bandLabel)]
277 fluxField = getRefFluxField(refSchema, filterLabel.bandLabel)
278 fluxKey = refSchema.find(fluxField).key
279 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
282 fluxErrKey = refSchema.find(fluxField +
"Err").key
283 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
286 self.log.warning(
"Reference catalog does not have flux uncertainties for %s;"
287 " using sqrt(flux).", fluxField)
288 refFluxErrArr = np.sqrt(refFluxArr)
290 refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
292 refMagErrArr = abMagErrFromFluxErr(refFluxErrArr*1e-9, refFluxArr*1e-9)
295 srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
299 magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
300 if self.config.magErrFloor != 0.0:
301 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
303 srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
305 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
307 return pipeBase.Struct(
308 srcMag=srcMagArr[good],
309 refMag=refMagArr[good],
310 magErr=magErrArr[good],
311 srcMagErr=srcMagErrArr[good],
312 refMagErr=refMagErrArr[good],
313 refFluxFieldList=fluxFieldList,
317 def run(self, exposure, sourceCat, expId=0):
318 """Do photometric calibration - select matches to use and (possibly iteratively) compute
324 Exposure upon which the sources in the matches were detected.
325 sourceCat : `lsst.afw.image.SourceCatalog`
326 A catalog of sources to use
in the calibration
329 the reference object
and matched object respectively).
330 Will
not be modified
except to set the outputField
if requested.
331 expId : `int`, optional
336 result : `lsst.pipe.base.Struct`
337 Results
as a struct
with attributes:
340 Object containing the zero point (`lsst.afw.image.Calib`).
342 Magnitude arrays returned be `PhotoCalTask.extractMagArrays`.
344 ReferenceMatchVector,
as returned by `PhotoCalTask.selectMatches`.
346 Photometric zero point (mag, `float`).
348 Standard deviation of fit of photometric zero point (mag, `float`).
350 Number of sources used to fit photometric zero point (`int`).
355 Raised
if any of the following occur:
356 - No matches to use
for photocal.
357 - No matches are available (perhaps no sources/references were selected by the matcher).
358 - No reference stars are available.
359 - No matches are available
from which to extract magnitudes.
363 The exposure
is only used to provide the name of the filter being calibrated (it may also be
364 used to generate debugging plots).
366 The reference objects:
367 - Must include a field ``photometric``;
True for objects which should be considered
as
368 photometric standards.
369 - Must include a field ``flux``; the flux used to impose a magnitude limit
and also to calibrate
370 the data to (unless a color term
is specified,
in which case ColorTerm.primary
is used;
371 See https://jira.lsstcorp.org/browse/DM-933).
372 - May include a field ``stargal``;
if present,
True means that the object
is a star.
373 - May include a field ``var``;
if present,
True means that the object
is variable.
375 The measured sources:
376 - Must include PhotoCalConfig.fluxField; the flux measurement to be used
for calibration.
381 displaySources = display
and lsstDebug.Info(__name__).displaySources
385 from matplotlib
import pyplot
389 self.
fig = pyplot.figure()
391 filterLabel = exposure.getFilter()
394 matchResults = self.
match.run(sourceCat, filterLabel.bandLabel)
395 matches = matchResults.matches
397 reserveResults = self.reserve.run([mm.second
for mm
in matches], expId=expId)
400 if reserveResults.reserved.sum() > 0:
401 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
402 if len(matches) == 0:
403 raise RuntimeError(
"No matches to use for photocal")
406 mm.second.set(self.
usedKey,
True)
413 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
414 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
417 flux0 = 10**(0.4*r.zp)
418 flux0err = 0.4*math.log(10)*flux0*r.sigma
419 photoCalib = makePhotoCalibFromCalibZeroPoint(flux0, flux0err)
421 return pipeBase.Struct(
422 photoCalib=photoCalib,
431 """Display sources we'll use for photocal.
433 Sources that will be actually used will be green.
434 Sources reserved from the fit will be red.
438 exposure : `lsst.afw.image.ExposureF`
440 matches : `list` of `lsst.afw.table.RefMatch`
441 Matches used
for photocal.
442 reserved : `numpy.ndarray` of type `bool`
443 Boolean array indicating sources that are reserved.
444 frame : `int`, optional
445 Frame number
for display.
447 disp = afwDisplay.getDisplay(frame=frame)
448 disp.mtv(exposure, title="photocal")
449 with disp.Buffering():
450 for mm, rr
in zip(matches, reserved):
451 x, y = mm.second.getCentroid()
452 ctype = afwDisplay.RED
if rr
else afwDisplay.GREEN
453 disp.dot(
"o", x, y, size=4, ctype=ctype)
456 """Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars).
460 result : `lsst.pipe.base.Struct`
461 Results as a struct
with attributes:
464 Photometric zero point (mag, `float`).
466 Standard deviation of fit of photometric zero point (mag, `float`).
468 Number of sources used to fit photometric zero point (`int`).
472 We perform nIter iterations of a simple sigma-clipping algorithm
with a couple of twists:
473 - We use the median/interquartile range to estimate the position to clip around,
and the
475 - We never allow sigma to go _above_ a critical value sigmaMax ---
if we do, a sufficiently
476 large estimate will prevent the clipping
from ever taking effect.
477 - Rather than start
with the median we start
with a crude mode. This means that a set of magnitude
478 residuals
with a tight core
and asymmetrical outliers will start
in the core. We use the width of
479 this core to set our maximum sigma (see second bullet).
481 sigmaMax = self.config.sigmaMax
485 indArr = np.argsort(dmag)
488 if srcErr
is not None:
489 dmagErr = srcErr[indArr]
491 dmagErr = np.ones(len(dmag))
494 ind_noNan = np.array([i
for i
in range(len(dmag))
495 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
496 dmag = dmag[ind_noNan]
497 dmagErr = dmagErr[ind_noNan]
499 IQ_TO_STDEV = 0.741301109252802
504 for i
in range(self.config.nIter):
515 hist, edges = np.histogram(dmag, nhist, new=
True)
517 hist, edges = np.histogram(dmag, nhist)
518 imode = np.arange(nhist)[np.where(hist == hist.max())]
520 if imode[-1] - imode[0] + 1 == len(imode):
524 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
526 peak = sum(hist[imode])/len(imode)
530 while j >= 0
and hist[j] > 0.5*peak:
533 q1 = dmag[sum(hist[range(j)])]
536 while j < nhist
and hist[j] > 0.5*peak:
538 j = min(j, nhist - 1)
539 j = min(sum(hist[range(j)]), npt - 1)
543 q1 = dmag[int(0.25*npt)]
544 q3 = dmag[int(0.75*npt)]
551 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
555 sigmaMax = dmag[-1] - dmag[0]
557 center = np.median(dmag)
558 q1 = dmag[int(0.25*npt)]
559 q3 = dmag[int(0.75*npt)]
564 if self.config.useMedian:
565 center = np.median(gdmag)
567 gdmagErr = dmagErr[good]
568 center = np.average(gdmag, weights=gdmagErr)
570 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
571 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
573 sig = IQ_TO_STDEV*(q3 - q1)
575 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
582 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
584 axes.plot(ref[good], dmag[good] - center,
"b+")
585 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
586 linestyle=
'', color=
'b')
588 bad = np.logical_not(good)
589 if len(ref[bad]) > 0:
590 axes.plot(ref[bad], dmag[bad] - center,
"r+")
591 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
592 linestyle=
'', color=
'r')
594 axes.plot((-100, 100), (0, 0),
"g-")
596 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
598 axes.set_ylim(-1.1, 1.1)
599 axes.set_xlim(24, 13)
600 axes.set_xlabel(
"Reference")
601 axes.set_ylabel(
"Reference - Instrumental")
607 while i == 0
or reply !=
"c":
609 reply = input(
"Next iteration? [ynhpc] ")
614 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
617 if reply
in (
"",
"c",
"n",
"p",
"y"):
620 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
627 except Exception
as e:
628 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
635 msg =
"PhotoCal.getZeroPoint: no good stars remain"
638 center = np.average(dmag, weights=dmagErr)
639 msg +=
" on first iteration; using average of all calibration stars"
641 self.log.warning(msg)
643 return pipeBase.Struct(
647 elif ngood == old_ngood:
653 dmagErr = dmagErr[good]
656 dmagErr = dmagErr[good]
657 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
658 sigma = np.sqrt(1.0/weightSum)
659 return pipeBase.Struct(
displaySources(self, exposure, matches, reserved, frame=1)
__init__(self, refObjLoader, schema=None, **kwds)
extractMagArrays(self, matches, filterLabel, sourceKeys)
getSourceKeys(self, schema)
getZeroPoint(self, src, ref, srcErr=None, zp0=None)