30 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, fluxFromABMag, Calib
35 from .colorterms
import ColortermLibrary
37 __all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
41 """Config for PhotoCal""" 42 match = pexConf.ConfigField(
"Match to reference catalog",
43 DirectMatchConfigWithoutLoader)
44 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
45 fluxField = pexConf.Field(
47 default=
"slot_CalibFlux_flux",
48 doc=(
"Name of the source flux field to use. The associated flag field\n" 49 "('<name>_flags') will be implicitly included in badFlags."),
51 applyColorTerms = pexConf.Field(
54 doc=(
"Apply photometric color terms to reference stars? One of:\n" 55 "None: apply if colorterms and photoCatName are not None;\n" 56 " fail if color term data is not available for the specified ref catalog and filter.\n" 57 "True: always apply colorterms; fail if color term data is not available for the\n" 58 " specified reference catalog and filter.\n" 59 "False: do not apply."),
62 sigmaMax = pexConf.Field(
65 doc=
"maximum sigma to use when clipping",
68 nSigma = pexConf.Field(
73 useMedian = pexConf.Field(
76 doc=
"use median instead of mean to compute zeropoint",
78 nIter = pexConf.Field(
81 doc=
"number of iterations",
83 colorterms = pexConf.ConfigField(
84 dtype=ColortermLibrary,
85 doc=
"Library of photometric reference catalog name: color term dict",
87 photoCatName = pexConf.Field(
90 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms." 91 " see also applyColorTerms"),
93 magErrFloor = pexConf.RangeField(
96 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
101 pexConf.Config.validate(self)
103 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
105 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
108 pexConf.Config.setDefaults(self)
109 self.
match.sourceSelection.doFlags =
True 110 self.
match.sourceSelection.flags.bad = [
111 "base_PixelFlags_flag_edge",
112 "base_PixelFlags_flag_interpolated",
113 "base_PixelFlags_flag_saturated",
115 self.
match.sourceSelection.doUnresolved =
True 127 \anchor PhotoCalTask_ 129 \brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. 131 \section pipe_tasks_photocal_Contents Contents 133 - \ref pipe_tasks_photocal_Purpose 134 - \ref pipe_tasks_photocal_Initialize 135 - \ref pipe_tasks_photocal_IO 136 - \ref pipe_tasks_photocal_Config 137 - \ref pipe_tasks_photocal_Debug 138 - \ref pipe_tasks_photocal_Example 140 \section pipe_tasks_photocal_Purpose Description 142 \copybrief PhotoCalTask 144 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. 145 The type of flux to use is specified by PhotoCalConfig.fluxField. 147 The algorithm clips outliers iteratively, with parameters set in the configuration. 149 \note This task can adds fields to the schema, so any code calling this task must ensure that 150 these columns are indeed present in the input match list; see \ref pipe_tasks_photocal_Example 152 \section pipe_tasks_photocal_Initialize Task initialisation 154 \copydoc \_\_init\_\_ 156 \section pipe_tasks_photocal_IO Inputs/Outputs to the run method 160 \section pipe_tasks_photocal_Config Configuration parameters 162 See \ref PhotoCalConfig 164 \section pipe_tasks_photocal_Debug Debug variables 166 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 167 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 169 The available variables in PhotoCalTask are: 172 <DD> If True enable other debug outputs 173 <DT> \c displaySources 174 <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue. 177 <DD> Reserved objects 179 <DD> Objects used in the photometric calibration 182 <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. 183 - good objects in blue 184 - rejected objects in red 185 (if \c scatterPlot is 2 or more, prompt to continue after each iteration) 188 \section pipe_tasks_photocal_Example A complete example of using PhotoCalTask 190 This code is in \link examples/photoCalTask.py\endlink, and can be run as \em e.g. 192 examples/photoCalTask.py 194 \dontinclude photoCalTask.py 196 Import the tasks (there are some other standard imports; read the file for details) 197 \skipline from lsst.pipe.tasks.astrometry 198 \skipline measPhotocal 200 We need to create both our tasks before processing any data as the task constructors 201 can add extra columns to the schema which we get from the input catalogue, \c scrCat: 205 \skip AstrometryTask.ConfigClass 207 (that \c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, 208 so we tell it to use the \c r band) 214 If the schema has indeed changed we need to add the new columns to the source table 215 (yes; this should be easier!) 219 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same 224 We can then unpack and use the results: 229 To investigate the \ref pipe_tasks_photocal_Debug, put something like 233 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 234 if name.endswith(".PhotoCal"): 239 lsstDebug.Info = DebugInfo 241 into your debug.py file and run photoCalTask.py with the \c --debug flag. 243 ConfigClass = PhotoCalConfig
244 _DefaultName =
"photoCal" 246 def __init__(self, refObjLoader, schema=None, **kwds):
247 """!Create the photometric calibration task. See PhotoCalTask.init for documentation 249 pipeBase.Task.__init__(self, **kwds)
252 if schema
is not None:
253 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
254 doc=
"set if source was used in photometric calibration")
257 self.
match = DirectMatchTask(self.config.match, refObjLoader=refObjLoader,
258 name=
"match", parentTask=self)
259 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
260 doc=
"set if source was reserved from photometric calibration")
263 """!Return a struct containing the source catalog keys for fields used by PhotoCalTask. 265 Returned fields include: 269 flux = schema.find(self.config.fluxField).key
270 fluxErr = schema.find(self.config.fluxField +
"Sigma").key
271 return pipeBase.Struct(flux=flux, fluxErr=fluxErr)
275 """!Extract magnitude and magnitude error arrays from the given matches. 277 \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink 278 \param[in] filterName Name of filter being calibrated 279 \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() 281 \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays 282 where magErr is an error in the magnitude; the error in srcMag - refMag 283 If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as 284 magErr is what is later used to determine the zero point. 285 Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes 287 \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been 288 discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) 290 srcFluxArr = np.array([m.second.get(sourceKeys.flux)
for m
in matches])
291 srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr)
for m
in matches])
292 if not np.all(np.isfinite(srcFluxErrArr)):
294 self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
295 srcFluxErrArr = np.sqrt(srcFluxArr)
298 JanskysPerABFlux = 3631.0
299 srcFluxArr = srcFluxArr * JanskysPerABFlux
300 srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
303 raise RuntimeError(
"No reference stars are available")
304 refSchema = matches[0].first.schema
306 applyColorTerms = self.config.applyColorTerms
307 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
308 if self.config.applyColorTerms
is None:
310 ctDataAvail = len(self.config.colorterms.data) > 0
311 photoCatSpecified = self.config.photoCatName
is not None 312 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
313 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
314 applyColorTerms = ctDataAvail
and photoCatSpecified
317 self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
318 filterName, self.config.photoCatName, applyCTReason)
319 ct = self.config.colorterms.getColorterm(
320 filterName=filterName, photoCatName=self.config.photoCatName, doRaise=
True)
322 self.log.info(
"Not applying color terms because %s", applyCTReason)
326 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (ct.primary, ct.secondary)]
327 missingFluxFieldList = []
328 for fluxField
in fluxFieldList:
330 refSchema.find(fluxField).key
332 missingFluxFieldList.append(fluxField)
334 if missingFluxFieldList:
335 self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
336 " ".join(missingFluxFieldList))
340 fluxFieldList = [getRefFluxField(refSchema, filterName)]
343 refFluxErrArrList = []
344 for fluxField
in fluxFieldList:
345 fluxKey = refSchema.find(fluxField).key
346 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
348 fluxErrKey = refSchema.find(fluxField +
"Sigma").key
349 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
352 self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
354 refFluxErrArr = np.sqrt(refFluxArr)
356 refFluxArrList.append(refFluxArr)
357 refFluxErrArrList.append(refFluxErrArr)
360 refMagArr1 = np.array([abMagFromFlux(rf1)
for rf1
in refFluxArrList[0]])
361 refMagArr2 = np.array([abMagFromFlux(rf2)
for rf2
in refFluxArrList[1]])
363 refMagArr = ct.transformMags(refMagArr1, refMagArr2)
364 refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
366 refMagArr = np.array([abMagFromFlux(rf)
for rf
in refFluxArrList[0]])
368 srcMagArr = np.array([abMagFromFlux(sf)
for sf
in srcFluxArr])
372 magErrArr = np.array([abMagErrFromFluxErr(fe, sf)
for fe, sf
in zip(srcFluxErrArr, srcFluxArr)])
373 if self.config.magErrFloor != 0.0:
374 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
376 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf)
for sfe, sf
in zip(srcFluxErrArr, srcFluxArr)])
377 refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf)
for rfe, rf
in zip(refFluxErrArr, refFluxArr)])
379 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
381 return pipeBase.Struct(
382 srcMag=srcMagArr[good],
383 refMag=refMagArr[good],
384 magErr=magErrArr[good],
385 srcMagErr=srcMagErrArr[good],
386 refMagErr=refMagErrArr[good],
387 refFluxFieldList=fluxFieldList,
391 def run(self, exposure, sourceCat, expId=0):
392 """!Do photometric calibration - select matches to use and (possibly iteratively) compute 395 \param[in] exposure Exposure upon which the sources in the matches were detected. 396 \param[in] sourceCat A catalog of sources to use in the calibration 397 (\em i.e. a list of lsst.afw.table.Match with 398 \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord --- 399 the reference object and matched object respectively). 400 (will not be modified except to set the outputField if requested.). 403 - calib ------- \link lsst::afw::image::Calib\endlink object containing the zero point 404 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays 405 - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. 406 - zp ---------- Photometric zero point (mag) 407 - sigma ------- Standard deviation of fit of photometric zero point (mag) 408 - ngood ------- Number of sources used to fit photometric zero point 410 The exposure is only used to provide the name of the filter being calibrated (it may also be 411 used to generate debugging plots). 413 The reference objects: 414 - Must include a field \c photometric; True for objects which should be considered as 415 photometric standards 416 - Must include a field \c flux; the flux used to impose a magnitude limit and also to calibrate 417 the data to (unless a color term is specified, in which case ColorTerm.primary is used; 418 See https://jira.lsstcorp.org/browse/DM-933) 419 - May include a field \c stargal; if present, True means that the object is a star 420 - May include a field \c var; if present, True means that the object is variable 422 The measured sources: 423 - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration 425 \throws RuntimeError with the following strings: 428 <DT> No matches to use for photocal 429 <DD> No matches are available (perhaps no sources/references were selected by the matcher). 430 <DT> No reference stars are available 431 <DD> No matches are available from which to extract magnitudes. 437 displaySources = display
and lsstDebug.Info(__name__).displaySources
441 from matplotlib
import pyplot
445 self.
fig = pyplot.figure()
447 filterName = exposure.getFilter().getName()
450 matchResults = self.
match.
run(sourceCat, filterName)
451 matches = matchResults.matches
452 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
455 if reserveResults.reserved.sum() > 0:
456 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
457 if len(matches) == 0:
458 raise RuntimeError(
"No matches to use for photocal")
461 mm.second.set(self.
usedKey,
True)
465 arrays = self.
extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
468 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
469 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
472 flux0 = 10**(0.4*r.zp)
473 flux0err = 0.4*math.log(10)*flux0*r.sigma
475 calib.setFluxMag0(flux0, flux0err)
477 return pipeBase.Struct(
487 """Display sources we'll use for photocal 489 Sources that will be actually used will be green. 490 Sources reserved from the fit will be red. 494 exposure : `lsst.afw.image.ExposureF` 496 matches : `list` of `lsst.afw.table.RefMatch` 497 Matches used for photocal. 498 reserved : `numpy.ndarray` of type `bool` 499 Boolean array indicating sources that are reserved. 501 Frame number for display. 503 ds9.mtv(exposure, frame=frame, title=
"photocal")
504 with ds9.Buffering():
505 for mm, rr
in zip(matches, reserved):
506 x, y = mm.second.getCentroid()
507 ctype = ds9.RED
if rr
else ds9.GREEN
508 ds9.dot(
"o", x, y, size=4, frame=frame, ctype=ctype)
512 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) 514 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: 515 1. We use the median/interquartile range to estimate the position to clip around, and the 517 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently 518 large estimate will prevent the clipping from ever taking effect. 519 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude 520 residuals with a tight core and asymmetrical outliers will start in the core. We use the width of 521 this core to set our maximum sigma (see 2.) 524 - zp ---------- Photometric zero point (mag) 525 - sigma ------- Standard deviation of fit of zero point (mag) 526 - ngood ------- Number of sources used to fit zero point 528 sigmaMax = self.config.sigmaMax
532 indArr = np.argsort(dmag)
535 if srcErr
is not None:
536 dmagErr = srcErr[indArr]
538 dmagErr = np.ones(len(dmag))
541 ind_noNan = np.array([i
for i
in range(len(dmag))
542 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
543 dmag = dmag[ind_noNan]
544 dmagErr = dmagErr[ind_noNan]
546 IQ_TO_STDEV = 0.741301109252802
551 for i
in range(self.config.nIter):
562 hist, edges = np.histogram(dmag, nhist, new=
True)
564 hist, edges = np.histogram(dmag, nhist)
565 imode = np.arange(nhist)[np.where(hist == hist.max())]
567 if imode[-1] - imode[0] + 1 == len(imode):
571 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
573 peak = sum(hist[imode])/len(imode)
577 while j >= 0
and hist[j] > 0.5*peak:
580 q1 = dmag[sum(hist[range(j)])]
583 while j < nhist
and hist[j] > 0.5*peak:
585 j = min(j, nhist - 1)
586 j = min(sum(hist[range(j)]), npt - 1)
590 q1 = dmag[int(0.25*npt)]
591 q3 = dmag[int(0.75*npt)]
598 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
602 sigmaMax = dmag[-1] - dmag[0]
604 center = np.median(dmag)
605 q1 = dmag[int(0.25*npt)]
606 q3 = dmag[int(0.75*npt)]
611 if self.config.useMedian:
612 center = np.median(gdmag)
614 gdmagErr = dmagErr[good]
615 center = np.average(gdmag, weights=gdmagErr)
617 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
618 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
620 sig = IQ_TO_STDEV*(q3 - q1)
622 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
629 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
631 axes.plot(ref[good], dmag[good] - center,
"b+")
632 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
633 linestyle=
'', color=
'b')
635 bad = np.logical_not(good)
636 if len(ref[bad]) > 0:
637 axes.plot(ref[bad], dmag[bad] - center,
"r+")
638 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
639 linestyle=
'', color=
'r') 641 axes.plot((-100, 100), (0, 0), "g-")
643 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
645 axes.set_ylim(-1.1, 1.1)
646 axes.set_xlim(24, 13)
647 axes.set_xlabel(
"Reference")
648 axes.set_ylabel(
"Reference - Instrumental")
654 while i == 0
or reply !=
"c":
656 reply = input(
"Next iteration? [ynhpc] ")
661 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
664 if reply
in (
"",
"c",
"n",
"p",
"y"):
667 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
674 except Exception
as e:
675 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
682 msg =
"PhotoCal.getZeroPoint: no good stars remain" 685 center = np.average(dmag, weights=dmagErr)
686 msg +=
" on first iteration; using average of all calibration stars" 690 return pipeBase.Struct(
694 elif ngood == old_ngood:
700 dmagErr = dmagErr[good]
703 dmagErr = dmagErr[good]
704 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
705 sigma = np.sqrt(1.0/weightSum)
706 return pipeBase.Struct(
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
def displaySources(self, exposure, matches, reserved, frame=1)