24 from __future__
import absolute_import, division, print_function
25 from builtins
import zip
26 from builtins
import input
27 from builtins
import range
36 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, fluxFromABMag, Calib
41 from .colorterms
import ColortermLibrary
43 __all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
47 """Config for PhotoCal""" 48 match = pexConf.ConfigurableField(target=DirectMatchTask, doc=
"Match to reference catalog")
49 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
50 fluxField = pexConf.Field(
52 default=
"slot_CalibFlux_flux",
53 doc=(
"Name of the source flux field to use. The associated flag field\n" 54 "('<name>_flags') will be implicitly included in badFlags."),
56 applyColorTerms = pexConf.Field(
59 doc=(
"Apply photometric color terms to reference stars? One of:\n" 60 "None: apply if colorterms and photoCatName are not None;\n" 61 " fail if color term data is not available for the specified ref catalog and filter.\n" 62 "True: always apply colorterms; fail if color term data is not available for the\n" 63 " specified reference catalog and filter.\n" 64 "False: do not apply."),
67 sigmaMax = pexConf.Field(
70 doc=
"maximum sigma to use when clipping",
73 nSigma = pexConf.Field(
78 useMedian = pexConf.Field(
81 doc=
"use median instead of mean to compute zeropoint",
83 nIter = pexConf.Field(
86 doc=
"number of iterations",
88 colorterms = pexConf.ConfigField(
89 dtype=ColortermLibrary,
90 doc=
"Library of photometric reference catalog name: color term dict",
92 photoCatName = pexConf.Field(
95 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms." 96 " see also applyColorTerms"),
98 magErrFloor = pexConf.RangeField(
101 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
106 pexConf.Config.validate(self)
108 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
110 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
113 pexConf.Config.setDefaults(self)
114 self.
match.sourceSelection.doFlags =
True 115 self.
match.sourceSelection.flags.bad = [
116 "base_PixelFlags_flag_edge",
117 "base_PixelFlags_flag_interpolated",
118 "base_PixelFlags_flag_saturated",
120 self.
match.sourceSelection.doUnresolved =
True 132 \anchor PhotoCalTask_ 134 \brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. 136 \section pipe_tasks_photocal_Contents Contents 138 - \ref pipe_tasks_photocal_Purpose 139 - \ref pipe_tasks_photocal_Initialize 140 - \ref pipe_tasks_photocal_IO 141 - \ref pipe_tasks_photocal_Config 142 - \ref pipe_tasks_photocal_Debug 143 - \ref pipe_tasks_photocal_Example 145 \section pipe_tasks_photocal_Purpose Description 147 \copybrief PhotoCalTask 149 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. 150 The type of flux to use is specified by PhotoCalConfig.fluxField. 152 The algorithm clips outliers iteratively, with parameters set in the configuration. 154 \note This task can adds fields to the schema, so any code calling this task must ensure that 155 these columns are indeed present in the input match list; see \ref pipe_tasks_photocal_Example 157 \section pipe_tasks_photocal_Initialize Task initialisation 159 \copydoc \_\_init\_\_ 161 \section pipe_tasks_photocal_IO Inputs/Outputs to the run method 165 \section pipe_tasks_photocal_Config Configuration parameters 167 See \ref PhotoCalConfig 169 \section pipe_tasks_photocal_Debug Debug variables 171 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 172 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 174 The available variables in PhotoCalTask are: 177 <DD> If True enable other debug outputs 178 <DT> \c displaySources 179 <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue. 182 <DD> Reserved objects 184 <DD> Objects used in the photometric calibration 187 <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. 188 - good objects in blue 189 - rejected objects in red 190 (if \c scatterPlot is 2 or more, prompt to continue after each iteration) 193 \section pipe_tasks_photocal_Example A complete example of using PhotoCalTask 195 This code is in \link examples/photoCalTask.py\endlink, and can be run as \em e.g. 197 examples/photoCalTask.py 199 \dontinclude photoCalTask.py 201 Import the tasks (there are some other standard imports; read the file for details) 202 \skipline from lsst.pipe.tasks.astrometry 203 \skipline measPhotocal 205 We need to create both our tasks before processing any data as the task constructors 206 can add extra columns to the schema which we get from the input catalogue, \c scrCat: 210 \skip AstrometryTask.ConfigClass 212 (that \c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, 213 so we tell it to use the \c r band) 219 If the schema has indeed changed we need to add the new columns to the source table 220 (yes; this should be easier!) 224 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same 229 We can then unpack and use the results: 234 To investigate the \ref pipe_tasks_photocal_Debug, put something like 238 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 239 if name.endswith(".PhotoCal"): 244 lsstDebug.Info = DebugInfo 246 into your debug.py file and run photoCalTask.py with the \c --debug flag. 248 ConfigClass = PhotoCalConfig
249 _DefaultName =
"photoCal" 251 def __init__(self, refObjLoader, schema=None, **kwds):
252 """!Create the photometric calibration task. See PhotoCalTask.init for documentation 254 pipeBase.Task.__init__(self, **kwds)
257 if schema
is not None:
258 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
259 doc=
"set if source was used in photometric calibration")
262 self.makeSubtask(
"match", refObjLoader=refObjLoader)
263 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
264 doc=
"set if source was reserved from photometric calibration")
267 """!Return a struct containing the source catalog keys for fields used by PhotoCalTask. 269 Returned fields include: 273 flux = schema.find(self.config.fluxField).key
274 fluxErr = schema.find(self.config.fluxField +
"Sigma").key
275 return pipeBase.Struct(flux=flux, fluxErr=fluxErr)
279 """!Extract magnitude and magnitude error arrays from the given matches. 281 \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink 282 \param[in] filterName Name of filter being calibrated 283 \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() 285 \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays 286 where magErr is an error in the magnitude; the error in srcMag - refMag 287 If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as 288 magErr is what is later used to determine the zero point. 289 Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes 291 \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been 292 discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) 294 srcFluxArr = np.array([m.second.get(sourceKeys.flux)
for m
in matches])
295 srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr)
for m
in matches])
296 if not np.all(np.isfinite(srcFluxErrArr)):
298 self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
299 srcFluxErrArr = np.sqrt(srcFluxArr)
302 JanskysPerABFlux = 3631.0
303 srcFluxArr = srcFluxArr * JanskysPerABFlux
304 srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
307 raise RuntimeError(
"No reference stars are available")
308 refSchema = matches[0].first.schema
310 applyColorTerms = self.config.applyColorTerms
311 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
312 if self.config.applyColorTerms
is None:
314 ctDataAvail = len(self.config.colorterms.data) > 0
315 photoCatSpecified = self.config.photoCatName
is not None 316 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
317 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
318 applyColorTerms = ctDataAvail
and photoCatSpecified
321 self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
322 filterName, self.config.photoCatName, applyCTReason)
323 ct = self.config.colorterms.getColorterm(
324 filterName=filterName, photoCatName=self.config.photoCatName, doRaise=
True)
326 self.log.info(
"Not applying color terms because %s", applyCTReason)
330 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (ct.primary, ct.secondary)]
331 missingFluxFieldList = []
332 for fluxField
in fluxFieldList:
334 refSchema.find(fluxField).key
336 missingFluxFieldList.append(fluxField)
338 if missingFluxFieldList:
339 self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
340 " ".join(missingFluxFieldList))
344 fluxFieldList = [getRefFluxField(refSchema, filterName)]
347 refFluxErrArrList = []
348 for fluxField
in fluxFieldList:
349 fluxKey = refSchema.find(fluxField).key
350 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
352 fluxErrKey = refSchema.find(fluxField +
"Sigma").key
353 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
356 self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
358 refFluxErrArr = np.sqrt(refFluxArr)
360 refFluxArrList.append(refFluxArr)
361 refFluxErrArrList.append(refFluxErrArr)
364 refMagArr1 = np.array([abMagFromFlux(rf1)
for rf1
in refFluxArrList[0]])
365 refMagArr2 = np.array([abMagFromFlux(rf2)
for rf2
in refFluxArrList[1]])
367 refMagArr = ct.transformMags(refMagArr1, refMagArr2)
368 refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
370 refMagArr = np.array([abMagFromFlux(rf)
for rf
in refFluxArrList[0]])
372 srcMagArr = np.array([abMagFromFlux(sf)
for sf
in srcFluxArr])
376 magErrArr = np.array([abMagErrFromFluxErr(fe, sf)
for fe, sf
in zip(srcFluxErrArr, srcFluxArr)])
377 if self.config.magErrFloor != 0.0:
378 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
380 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf)
for sfe, sf
in zip(srcFluxErrArr, srcFluxArr)])
381 refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf)
for rfe, rf
in zip(refFluxErrArr, refFluxArr)])
383 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
385 return pipeBase.Struct(
386 srcMag=srcMagArr[good],
387 refMag=refMagArr[good],
388 magErr=magErrArr[good],
389 srcMagErr=srcMagErrArr[good],
390 refMagErr=refMagErrArr[good],
391 refFluxFieldList=fluxFieldList,
395 def run(self, exposure, sourceCat, expId=0):
396 """!Do photometric calibration - select matches to use and (possibly iteratively) compute 399 \param[in] exposure Exposure upon which the sources in the matches were detected. 400 \param[in] sourceCat A catalog of sources to use in the calibration 401 (\em i.e. a list of lsst.afw.table.Match with 402 \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord --- 403 the reference object and matched object respectively). 404 (will not be modified except to set the outputField if requested.). 407 - calib ------- \link lsst::afw::image::Calib\endlink object containing the zero point 408 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays 409 - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. 410 - zp ---------- Photometric zero point (mag) 411 - sigma ------- Standard deviation of fit of photometric zero point (mag) 412 - ngood ------- Number of sources used to fit photometric zero point 414 The exposure is only used to provide the name of the filter being calibrated (it may also be 415 used to generate debugging plots). 417 The reference objects: 418 - Must include a field \c photometric; True for objects which should be considered as 419 photometric standards 420 - Must include a field \c flux; the flux used to impose a magnitude limit and also to calibrate 421 the data to (unless a color term is specified, in which case ColorTerm.primary is used; 422 See https://jira.lsstcorp.org/browse/DM-933) 423 - May include a field \c stargal; if present, True means that the object is a star 424 - May include a field \c var; if present, True means that the object is variable 426 The measured sources: 427 - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration 429 \throws RuntimeError with the following strings: 432 <DT> No matches to use for photocal 433 <DD> No matches are available (perhaps no sources/references were selected by the matcher). 434 <DT> No reference stars are available 435 <DD> No matches are available from which to extract magnitudes. 441 displaySources = display
and lsstDebug.Info(__name__).displaySources
445 from matplotlib
import pyplot
449 self.
fig = pyplot.figure()
451 filterName = exposure.getFilter().getName()
454 matchResults = self.match.
run(sourceCat, filterName)
455 matches = matchResults.matches
456 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
459 if reserveResults.reserved.sum() > 0:
460 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
461 if len(matches) == 0:
462 raise RuntimeError(
"No matches to use for photocal")
465 mm.second.set(self.
usedKey,
True)
469 arrays = self.
extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
472 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
473 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
476 flux0 = 10**(0.4*r.zp)
477 flux0err = 0.4*math.log(10)*flux0*r.sigma
479 calib.setFluxMag0(flux0, flux0err)
481 return pipeBase.Struct(
491 """Display sources we'll use for photocal 493 Sources that will be actually used will be green. 494 Sources reserved from the fit will be red. 498 exposure : `lsst.afw.image.ExposureF` 500 matches : `list` of `lsst.afw.table.RefMatch` 501 Matches used for photocal. 502 reserved : `numpy.ndarray` of type `bool` 503 Boolean array indicating sources that are reserved. 505 Frame number for display. 507 ds9.mtv(exposure, frame=frame, title=
"photocal")
508 with ds9.Buffering():
509 for mm, rr
in zip(matches, reserved):
510 x, y = mm.second.getCentroid()
511 ctype = ds9.RED
if rr
else ds9.GREEN
512 ds9.dot(
"o", x, y, size=4, frame=frame, ctype=ctype)
516 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) 518 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: 519 1. We use the median/interquartile range to estimate the position to clip around, and the 521 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently 522 large estimate will prevent the clipping from ever taking effect. 523 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude 524 residuals with a tight core and asymmetrical outliers will start in the core. We use the width of 525 this core to set our maximum sigma (see 2.) 528 - zp ---------- Photometric zero point (mag) 529 - sigma ------- Standard deviation of fit of zero point (mag) 530 - ngood ------- Number of sources used to fit zero point 532 sigmaMax = self.config.sigmaMax
536 indArr = np.argsort(dmag)
539 if srcErr
is not None:
540 dmagErr = srcErr[indArr]
542 dmagErr = np.ones(len(dmag))
545 ind_noNan = np.array([i
for i
in range(len(dmag))
546 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
547 dmag = dmag[ind_noNan]
548 dmagErr = dmagErr[ind_noNan]
550 IQ_TO_STDEV = 0.741301109252802
555 for i
in range(self.config.nIter):
566 hist, edges = np.histogram(dmag, nhist, new=
True)
568 hist, edges = np.histogram(dmag, nhist)
569 imode = np.arange(nhist)[np.where(hist == hist.max())]
571 if imode[-1] - imode[0] + 1 == len(imode):
575 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
577 peak = sum(hist[imode])/len(imode)
581 while j >= 0
and hist[j] > 0.5*peak:
584 q1 = dmag[sum(hist[range(j)])]
587 while j < nhist
and hist[j] > 0.5*peak:
589 j = min(j, nhist - 1)
590 j = min(sum(hist[range(j)]), npt - 1)
594 q1 = dmag[int(0.25*npt)]
595 q3 = dmag[int(0.75*npt)]
602 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
606 sigmaMax = dmag[-1] - dmag[0]
608 center = np.median(dmag)
609 q1 = dmag[int(0.25*npt)]
610 q3 = dmag[int(0.75*npt)]
615 if self.config.useMedian:
616 center = np.median(gdmag)
618 gdmagErr = dmagErr[good]
619 center = np.average(gdmag, weights=gdmagErr)
621 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
622 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
624 sig = IQ_TO_STDEV*(q3 - q1)
626 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
633 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
635 axes.plot(ref[good], dmag[good] - center,
"b+")
636 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
637 linestyle=
'', color=
'b')
639 bad = np.logical_not(good)
640 if len(ref[bad]) > 0:
641 axes.plot(ref[bad], dmag[bad] - center,
"r+")
642 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
643 linestyle=
'', color=
'r') 645 axes.plot((-100, 100), (0, 0), "g-")
647 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
649 axes.set_ylim(-1.1, 1.1)
650 axes.set_xlim(24, 13)
651 axes.set_xlabel(
"Reference")
652 axes.set_ylabel(
"Reference - Instrumental")
658 while i == 0
or reply !=
"c":
660 reply = input(
"Next iteration? [ynhpc] ")
665 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
668 if reply
in (
"",
"c",
"n",
"p",
"y"):
671 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
678 except Exception
as e:
679 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
686 msg =
"PhotoCal.getZeroPoint: no good stars remain" 689 center = np.average(dmag, weights=dmagErr)
690 msg +=
" on first iteration; using average of all calibration stars" 694 return pipeBase.Struct(
698 elif ngood == old_ngood:
704 dmagErr = dmagErr[good]
707 dmagErr = dmagErr[good]
708 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
709 sigma = np.sqrt(1.0/weightSum)
710 return pipeBase.Struct(
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
def displaySources(self, exposure, matches, reserved, frame=1)