22__all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
28import astropy.units
as u
32from lsst.afw.image import abMagErrFromFluxErr, makePhotoCalibFromCalibZeroPoint
37from lsst.utils.timer
import timeMethod
38from .colorterms
import ColortermLibrary
42 """Config for PhotoCal."""
44 match = pexConf.ConfigField(
"Match to reference catalog",
45 DirectMatchConfigWithoutLoader)
46 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
47 fluxField = pexConf.Field(
49 default=
"slot_CalibFlux_instFlux",
50 doc=(
"Name of the source instFlux field to use. The associated flag field\n"
51 "('<name>_flags') will be implicitly included in badFlags."),
53 applyColorTerms = pexConf.Field(
56 doc=(
"Apply photometric color terms to reference stars? One of:\n"
57 "None: apply if colorterms and photoCatName are not None;\n"
58 " fail if color term data is not available for the specified ref catalog and filter.\n"
59 "True: always apply colorterms; fail if color term data is not available for the\n"
60 " specified reference catalog and filter.\n"
61 "False: do not apply."),
64 sigmaMax = pexConf.Field(
67 doc=
"maximum sigma to use when clipping",
70 nSigma = pexConf.Field(
75 useMedian = pexConf.Field(
78 doc=
"use median instead of mean to compute zeropoint",
80 nIter = pexConf.Field(
83 doc=
"number of iterations",
85 colorterms = pexConf.ConfigField(
86 dtype=ColortermLibrary,
87 doc=
"Library of photometric reference catalog name: color term dict",
89 photoCatName = pexConf.Field(
92 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms."
93 " see also applyColorTerms"),
95 magErrFloor = pexConf.RangeField(
98 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
103 pexConf.Config.validate(self)
105 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
107 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
110 pexConf.Config.setDefaults(self)
111 self.
match.sourceSelection.doFlags =
True
112 self.
match.sourceSelection.flags.bad = [
113 "base_PixelFlags_flag_edge",
114 "base_PixelFlags_flag_interpolated",
115 "base_PixelFlags_flag_saturated",
117 self.
match.sourceSelection.doUnresolved =
True
121 """Calculate an Exposure's zero-point given a set of flux measurements
122 of stars matched to an input catalogue.
126 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
127 An instance of LoadReferenceObjectsTasks that supplies an external reference
130 The schema of the detection catalogs used as input to this task.
132 Additional keyword arguments.
136 The type of flux to use
is specified by PhotoCalConfig.fluxField.
138 The algorithm clips outliers iteratively,
with parameters set
in the configuration.
140 This task can adds fields to the schema, so any code calling this task must ensure that
141 these columns are indeed present
in the input match list; see `pipe_tasks_photocal_Example`.
145 The available `~lsst.base.lsstDebug` variables
in PhotoCalTask are:
148 If
True enable other debug outputs.
150 If
True, display the exposure on ds9
's frame 1 and overlay the source catalogue.
155 Objects used in the photometric calibration.
158 Make a scatter plot of flux v. reference magnitude
as a function of reference magnitude:
160 - good objects
in blue
161 - rejected objects
in red
163 (
if scatterPlot
is 2
or more, prompt to
continue after each iteration)
166 ConfigClass = PhotoCalConfig
167 _DefaultName = "photoCal"
169 def __init__(self, refObjLoader, schema=None, **kwds):
170 pipeBase.Task.__init__(self, **kwds)
173 if schema
is not None:
174 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
175 doc=
"set if source was used in photometric calibration")
178 self.
match = DirectMatchTask(config=self.config.match, refObjLoader=refObjLoader,
179 name=
"match", parentTask=self)
180 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
181 doc=
"set if source was reserved from photometric calibration")
184 """Return a struct containing the source catalog keys for fields used
189 schema : `lsst.afw.table.schema`
190 Schema of the catalog to get keys from.
194 result : `lsst.pipe.base.Struct`
195 Results
as a struct
with attributes:
200 Instrument flux error key.
202 instFlux = schema.find(self.config.fluxField).key
203 instFluxErr = schema.find(self.config.fluxField + "Err").key
204 return pipeBase.Struct(instFlux=instFlux, instFluxErr=instFluxErr)
208 """Extract magnitude and magnitude error arrays from the given matches.
213 Reference/source matches.
215 Label of filter being calibrated.
216 sourceKeys : `lsst.pipe.base.Struct`
217 Struct of source catalog keys, as returned by
getSourceKeys().
221 result : `lsst.pipe.base.Struct`
222 Results
as a struct
with attributes:
225 Source magnitude (`np.array`).
227 Reference magnitude (`np.array`).
229 Source magnitude error (`np.array`).
231 Reference magnitude error (`np.array`).
233 An error
in the magnitude; the error
in ``srcMag`` - ``refMag``.
234 If nonzero, ``config.magErrFloor`` will be added to ``magErr`` only
235 (
not ``srcMagErr``
or ``refMagErr``),
as
236 ``magErr``
is what
is later used to determine the zero point (`np.array`).
238 A list of field names of the reference catalog used
for fluxes (1
or 2 strings) (`list`).
240 srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m
in matches])
241 srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr)
for m
in matches])
242 if not np.all(np.isfinite(srcInstFluxErrArr)):
244 self.log.warning(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
245 srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
248 referenceFlux = (0*u.ABmag).to_value(u.nJy)
249 srcInstFluxArr = srcInstFluxArr * referenceFlux
250 srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
253 raise RuntimeError(
"No reference stars are available")
254 refSchema = matches[0].first.schema
256 applyColorTerms = self.config.applyColorTerms
257 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
258 if self.config.applyColorTerms
is None:
260 ctDataAvail = len(self.config.colorterms.data) > 0
261 photoCatSpecified = self.config.photoCatName
is not None
262 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
263 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
264 applyColorTerms = ctDataAvail
and photoCatSpecified
267 self.log.info(
"Applying color terms for filter=%r, config.photoCatName=%s because %s",
268 filterLabel.physicalLabel, self.config.photoCatName, applyCTReason)
269 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
270 self.config.photoCatName,
272 refCat = afwTable.SimpleCatalog(matches[0].first.schema)
275 refCat.reserve(len(matches))
277 record = refCat.addNew()
278 record.assign(x.first)
280 refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat)
281 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (colorterm.primary,
282 colorterm.secondary)]
285 self.log.info(
"Not applying color terms because %s", applyCTReason)
288 fluxFieldList = [getRefFluxField(refSchema, filterLabel.bandLabel)]
289 fluxField = getRefFluxField(refSchema, filterLabel.bandLabel)
290 fluxKey = refSchema.find(fluxField).key
291 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
294 fluxErrKey = refSchema.find(fluxField +
"Err").key
295 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
298 self.log.warning(
"Reference catalog does not have flux uncertainties for %s;"
299 " using sqrt(flux).", fluxField)
300 refFluxErrArr = np.sqrt(refFluxArr)
302 refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
304 refMagErrArr = abMagErrFromFluxErr(refFluxErrArr*1e-9, refFluxArr*1e-9)
307 srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
311 magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
312 if self.config.magErrFloor != 0.0:
313 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
315 srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
317 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
319 return pipeBase.Struct(
320 srcMag=srcMagArr[good],
321 refMag=refMagArr[good],
322 magErr=magErrArr[good],
323 srcMagErr=srcMagErrArr[good],
324 refMagErr=refMagErrArr[good],
325 refFluxFieldList=fluxFieldList,
329 def run(self, exposure, sourceCat, expId=0):
330 """Do photometric calibration - select matches to use and (possibly iteratively) compute
336 Exposure upon which the sources in the matches were detected.
337 sourceCat : `lsst.afw.image.SourceCatalog`
338 A catalog of sources to use
in the calibration
341 the reference object
and matched object respectively).
342 Will
not be modified
except to set the outputField
if requested.
343 expId : `int`, optional
348 result : `lsst.pipe.base.Struct`
349 Results
as a struct
with attributes:
352 Object containing the zero point (`lsst.afw.image.Calib`).
354 Magnitude arrays returned be `PhotoCalTask.extractMagArrays`.
356 ReferenceMatchVector,
as returned by `PhotoCalTask.selectMatches`.
358 Photometric zero point (mag, `float`).
360 Standard deviation of fit of photometric zero point (mag, `float`).
362 Number of sources used to fit photometric zero point (`int`).
367 Raised
if any of the following occur:
368 - No matches to use
for photocal.
369 - No matches are available (perhaps no sources/references were selected by the matcher).
370 - No reference stars are available.
371 - No matches are available
from which to extract magnitudes.
375 The exposure
is only used to provide the name of the filter being calibrated (it may also be
376 used to generate debugging plots).
378 The reference objects:
379 - Must include a field ``photometric``;
True for objects which should be considered
as
380 photometric standards.
381 - Must include a field ``flux``; the flux used to impose a magnitude limit
and also to calibrate
382 the data to (unless a color term
is specified,
in which case ColorTerm.primary
is used;
383 See https://jira.lsstcorp.org/browse/DM-933).
384 - May include a field ``stargal``;
if present,
True means that the object
is a star.
385 - May include a field ``var``;
if present,
True means that the object
is variable.
387 The measured sources:
388 - Must include PhotoCalConfig.fluxField; the flux measurement to be used
for calibration.
393 displaySources = display
and lsstDebug.Info(__name__).displaySources
397 from matplotlib
import pyplot
401 self.
fig = pyplot.figure()
403 filterLabel = exposure.getFilter()
406 matchResults = self.
match.run(sourceCat, filterLabel.bandLabel)
407 matches = matchResults.matches
409 reserveResults = self.reserve.run([mm.second
for mm
in matches], expId=expId)
412 if reserveResults.reserved.sum() > 0:
413 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
414 if len(matches) == 0:
415 raise RuntimeError(
"No matches to use for photocal")
418 mm.second.set(self.
usedKey,
True)
425 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
426 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
429 flux0 = 10**(0.4*r.zp)
430 flux0err = 0.4*math.log(10)*flux0*r.sigma
431 photoCalib = makePhotoCalibFromCalibZeroPoint(flux0, flux0err)
433 return pipeBase.Struct(
434 photoCalib=photoCalib,
443 """Display sources we'll use for photocal.
445 Sources that will be actually used will be green.
446 Sources reserved from the fit will be red.
450 exposure : `lsst.afw.image.ExposureF`
452 matches : `list` of `lsst.afw.table.RefMatch`
453 Matches used
for photocal.
454 reserved : `numpy.ndarray` of type `bool`
455 Boolean array indicating sources that are reserved.
456 frame : `int`, optional
457 Frame number
for display.
459 disp = afwDisplay.getDisplay(frame=frame)
460 disp.mtv(exposure, title="photocal")
461 with disp.Buffering():
462 for mm, rr
in zip(matches, reserved):
463 x, y = mm.second.getCentroid()
464 ctype = afwDisplay.RED
if rr
else afwDisplay.GREEN
465 disp.dot(
"o", x, y, size=4, ctype=ctype)
468 """Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars).
472 result : `lsst.pipe.base.Struct`
473 Results as a struct
with attributes:
476 Photometric zero point (mag, `float`).
478 Standard deviation of fit of photometric zero point (mag, `float`).
480 Number of sources used to fit photometric zero point (`int`).
484 We perform nIter iterations of a simple sigma-clipping algorithm
with a couple of twists:
485 - We use the median/interquartile range to estimate the position to clip around,
and the
487 - We never allow sigma to go _above_ a critical value sigmaMax ---
if we do, a sufficiently
488 large estimate will prevent the clipping
from ever taking effect.
489 - Rather than start
with the median we start
with a crude mode. This means that a set of magnitude
490 residuals
with a tight core
and asymmetrical outliers will start
in the core. We use the width of
491 this core to set our maximum sigma (see second bullet).
493 sigmaMax = self.config.sigmaMax
497 indArr = np.argsort(dmag)
500 if srcErr
is not None:
501 dmagErr = srcErr[indArr]
503 dmagErr = np.ones(len(dmag))
506 ind_noNan = np.array([i
for i
in range(len(dmag))
507 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
508 dmag = dmag[ind_noNan]
509 dmagErr = dmagErr[ind_noNan]
511 IQ_TO_STDEV = 0.741301109252802
516 for i
in range(self.config.nIter):
527 hist, edges = np.histogram(dmag, nhist, new=
True)
529 hist, edges = np.histogram(dmag, nhist)
530 imode = np.arange(nhist)[np.where(hist == hist.max())]
532 if imode[-1] - imode[0] + 1 == len(imode):
536 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
538 peak = sum(hist[imode])/len(imode)
542 while j >= 0
and hist[j] > 0.5*peak:
545 q1 = dmag[sum(hist[range(j)])]
548 while j < nhist
and hist[j] > 0.5*peak:
550 j = min(j, nhist - 1)
551 j = min(sum(hist[range(j)]), npt - 1)
555 q1 = dmag[int(0.25*npt)]
556 q3 = dmag[int(0.75*npt)]
563 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
567 sigmaMax = dmag[-1] - dmag[0]
569 center = np.median(dmag)
570 q1 = dmag[int(0.25*npt)]
571 q3 = dmag[int(0.75*npt)]
576 if self.config.useMedian:
577 center = np.median(gdmag)
579 gdmagErr = dmagErr[good]
580 center = np.average(gdmag, weights=gdmagErr)
582 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
583 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
585 sig = IQ_TO_STDEV*(q3 - q1)
587 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
594 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
596 axes.plot(ref[good], dmag[good] - center,
"b+")
597 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
598 linestyle=
'', color=
'b')
600 bad = np.logical_not(good)
601 if len(ref[bad]) > 0:
602 axes.plot(ref[bad], dmag[bad] - center,
"r+")
603 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
604 linestyle=
'', color=
'r')
606 axes.plot((-100, 100), (0, 0),
"g-")
608 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
610 axes.set_ylim(-1.1, 1.1)
611 axes.set_xlim(24, 13)
612 axes.set_xlabel(
"Reference")
613 axes.set_ylabel(
"Reference - Instrumental")
619 while i == 0
or reply !=
"c":
621 reply = input(
"Next iteration? [ynhpc] ")
626 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
629 if reply
in (
"",
"c",
"n",
"p",
"y"):
632 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
639 except Exception
as e:
640 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
647 msg =
"PhotoCal.getZeroPoint: no good stars remain"
650 center = np.average(dmag, weights=dmagErr)
651 msg +=
" on first iteration; using average of all calibration stars"
653 self.log.warning(msg)
655 return pipeBase.Struct(
659 elif ngood == old_ngood:
665 dmagErr = dmagErr[good]
668 dmagErr = dmagErr[good]
669 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
670 sigma = np.sqrt(1.0/weightSum)
671 return pipeBase.Struct(
def getSourceKeys(self, schema)
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
def extractMagArrays(self, matches, filterLabel, sourceKeys)
def __init__(self, refObjLoader, schema=None, **kwds)
def displaySources(self, exposure, matches, reserved, frame=1)