24 from __future__
import absolute_import, division, print_function
25 from builtins
import zip
26 from builtins
import input
27 from builtins
import range
36 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, fluxFromABMag, Calib
41 from .colorterms
import ColortermLibrary
43 __all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
47 """Config for PhotoCal""" 48 match = pexConf.ConfigField(
"Match to reference catalog",
49 DirectMatchConfigWithoutLoader)
50 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
51 fluxField = pexConf.Field(
53 default=
"slot_CalibFlux_flux",
54 doc=(
"Name of the source flux field to use. The associated flag field\n" 55 "('<name>_flags') will be implicitly included in badFlags."),
57 applyColorTerms = pexConf.Field(
60 doc=(
"Apply photometric color terms to reference stars? One of:\n" 61 "None: apply if colorterms and photoCatName are not None;\n" 62 " fail if color term data is not available for the specified ref catalog and filter.\n" 63 "True: always apply colorterms; fail if color term data is not available for the\n" 64 " specified reference catalog and filter.\n" 65 "False: do not apply."),
68 sigmaMax = pexConf.Field(
71 doc=
"maximum sigma to use when clipping",
74 nSigma = pexConf.Field(
79 useMedian = pexConf.Field(
82 doc=
"use median instead of mean to compute zeropoint",
84 nIter = pexConf.Field(
87 doc=
"number of iterations",
89 colorterms = pexConf.ConfigField(
90 dtype=ColortermLibrary,
91 doc=
"Library of photometric reference catalog name: color term dict",
93 photoCatName = pexConf.Field(
96 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms." 97 " see also applyColorTerms"),
99 magErrFloor = pexConf.RangeField(
102 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
107 pexConf.Config.validate(self)
109 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
111 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
114 pexConf.Config.setDefaults(self)
115 self.
match.sourceSelection.doFlags =
True 116 self.
match.sourceSelection.flags.bad = [
117 "base_PixelFlags_flag_edge",
118 "base_PixelFlags_flag_interpolated",
119 "base_PixelFlags_flag_saturated",
121 self.
match.sourceSelection.doUnresolved =
True 133 \anchor PhotoCalTask_ 135 \brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. 137 \section pipe_tasks_photocal_Contents Contents 139 - \ref pipe_tasks_photocal_Purpose 140 - \ref pipe_tasks_photocal_Initialize 141 - \ref pipe_tasks_photocal_IO 142 - \ref pipe_tasks_photocal_Config 143 - \ref pipe_tasks_photocal_Debug 144 - \ref pipe_tasks_photocal_Example 146 \section pipe_tasks_photocal_Purpose Description 148 \copybrief PhotoCalTask 150 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. 151 The type of flux to use is specified by PhotoCalConfig.fluxField. 153 The algorithm clips outliers iteratively, with parameters set in the configuration. 155 \note This task can adds fields to the schema, so any code calling this task must ensure that 156 these columns are indeed present in the input match list; see \ref pipe_tasks_photocal_Example 158 \section pipe_tasks_photocal_Initialize Task initialisation 160 \copydoc \_\_init\_\_ 162 \section pipe_tasks_photocal_IO Inputs/Outputs to the run method 166 \section pipe_tasks_photocal_Config Configuration parameters 168 See \ref PhotoCalConfig 170 \section pipe_tasks_photocal_Debug Debug variables 172 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 173 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 175 The available variables in PhotoCalTask are: 178 <DD> If True enable other debug outputs 179 <DT> \c displaySources 180 <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue. 183 <DD> Reserved objects 185 <DD> Objects used in the photometric calibration 188 <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. 189 - good objects in blue 190 - rejected objects in red 191 (if \c scatterPlot is 2 or more, prompt to continue after each iteration) 194 \section pipe_tasks_photocal_Example A complete example of using PhotoCalTask 196 This code is in \link examples/photoCalTask.py\endlink, and can be run as \em e.g. 198 examples/photoCalTask.py 200 \dontinclude photoCalTask.py 202 Import the tasks (there are some other standard imports; read the file for details) 203 \skipline from lsst.pipe.tasks.astrometry 204 \skipline measPhotocal 206 We need to create both our tasks before processing any data as the task constructors 207 can add extra columns to the schema which we get from the input catalogue, \c scrCat: 211 \skip AstrometryTask.ConfigClass 213 (that \c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, 214 so we tell it to use the \c r band) 220 If the schema has indeed changed we need to add the new columns to the source table 221 (yes; this should be easier!) 225 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same 230 We can then unpack and use the results: 235 To investigate the \ref pipe_tasks_photocal_Debug, put something like 239 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 240 if name.endswith(".PhotoCal"): 245 lsstDebug.Info = DebugInfo 247 into your debug.py file and run photoCalTask.py with the \c --debug flag. 249 ConfigClass = PhotoCalConfig
250 _DefaultName =
"photoCal" 252 def __init__(self, refObjLoader, schema=None, **kwds):
253 """!Create the photometric calibration task. See PhotoCalTask.init for documentation 255 pipeBase.Task.__init__(self, **kwds)
258 if schema
is not None:
259 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
260 doc=
"set if source was used in photometric calibration")
263 self.
match = DirectMatchTask(self.config.match, refObjLoader=refObjLoader,
264 name=
"match", parentTask=self)
265 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
266 doc=
"set if source was reserved from photometric calibration")
269 """!Return a struct containing the source catalog keys for fields used by PhotoCalTask. 271 Returned fields include: 275 flux = schema.find(self.config.fluxField).key
276 fluxErr = schema.find(self.config.fluxField +
"Sigma").key
277 return pipeBase.Struct(flux=flux, fluxErr=fluxErr)
281 """!Extract magnitude and magnitude error arrays from the given matches. 283 \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink 284 \param[in] filterName Name of filter being calibrated 285 \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() 287 \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays 288 where magErr is an error in the magnitude; the error in srcMag - refMag 289 If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as 290 magErr is what is later used to determine the zero point. 291 Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes 293 \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been 294 discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) 296 srcFluxArr = np.array([m.second.get(sourceKeys.flux)
for m
in matches])
297 srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr)
for m
in matches])
298 if not np.all(np.isfinite(srcFluxErrArr)):
300 self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
301 srcFluxErrArr = np.sqrt(srcFluxArr)
304 JanskysPerABFlux = 3631.0
305 srcFluxArr = srcFluxArr * JanskysPerABFlux
306 srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
309 raise RuntimeError(
"No reference stars are available")
310 refSchema = matches[0].first.schema
312 applyColorTerms = self.config.applyColorTerms
313 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
314 if self.config.applyColorTerms
is None:
316 ctDataAvail = len(self.config.colorterms.data) > 0
317 photoCatSpecified = self.config.photoCatName
is not None 318 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
319 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
320 applyColorTerms = ctDataAvail
and photoCatSpecified
323 self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
324 filterName, self.config.photoCatName, applyCTReason)
325 ct = self.config.colorterms.getColorterm(
326 filterName=filterName, photoCatName=self.config.photoCatName, doRaise=
True)
328 self.log.info(
"Not applying color terms because %s", applyCTReason)
332 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (ct.primary, ct.secondary)]
333 missingFluxFieldList = []
334 for fluxField
in fluxFieldList:
336 refSchema.find(fluxField).key
338 missingFluxFieldList.append(fluxField)
340 if missingFluxFieldList:
341 self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
342 " ".join(missingFluxFieldList))
346 fluxFieldList = [getRefFluxField(refSchema, filterName)]
349 refFluxErrArrList = []
350 for fluxField
in fluxFieldList:
351 fluxKey = refSchema.find(fluxField).key
352 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
354 fluxErrKey = refSchema.find(fluxField +
"Sigma").key
355 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
358 self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
360 refFluxErrArr = np.sqrt(refFluxArr)
362 refFluxArrList.append(refFluxArr)
363 refFluxErrArrList.append(refFluxErrArr)
366 refMagArr1 = np.array([abMagFromFlux(rf1)
for rf1
in refFluxArrList[0]])
367 refMagArr2 = np.array([abMagFromFlux(rf2)
for rf2
in refFluxArrList[1]])
369 refMagArr = ct.transformMags(refMagArr1, refMagArr2)
370 refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
372 refMagArr = np.array([abMagFromFlux(rf)
for rf
in refFluxArrList[0]])
374 srcMagArr = np.array([abMagFromFlux(sf)
for sf
in srcFluxArr])
378 magErrArr = np.array([abMagErrFromFluxErr(fe, sf)
for fe, sf
in zip(srcFluxErrArr, srcFluxArr)])
379 if self.config.magErrFloor != 0.0:
380 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
382 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf)
for sfe, sf
in zip(srcFluxErrArr, srcFluxArr)])
383 refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf)
for rfe, rf
in zip(refFluxErrArr, refFluxArr)])
385 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
387 return pipeBase.Struct(
388 srcMag=srcMagArr[good],
389 refMag=refMagArr[good],
390 magErr=magErrArr[good],
391 srcMagErr=srcMagErrArr[good],
392 refMagErr=refMagErrArr[good],
393 refFluxFieldList=fluxFieldList,
397 def run(self, exposure, sourceCat, expId=0):
398 """!Do photometric calibration - select matches to use and (possibly iteratively) compute 401 \param[in] exposure Exposure upon which the sources in the matches were detected. 402 \param[in] sourceCat A catalog of sources to use in the calibration 403 (\em i.e. a list of lsst.afw.table.Match with 404 \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord --- 405 the reference object and matched object respectively). 406 (will not be modified except to set the outputField if requested.). 409 - calib ------- \link lsst::afw::image::Calib\endlink object containing the zero point 410 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays 411 - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. 412 - zp ---------- Photometric zero point (mag) 413 - sigma ------- Standard deviation of fit of photometric zero point (mag) 414 - ngood ------- Number of sources used to fit photometric zero point 416 The exposure is only used to provide the name of the filter being calibrated (it may also be 417 used to generate debugging plots). 419 The reference objects: 420 - Must include a field \c photometric; True for objects which should be considered as 421 photometric standards 422 - Must include a field \c flux; the flux used to impose a magnitude limit and also to calibrate 423 the data to (unless a color term is specified, in which case ColorTerm.primary is used; 424 See https://jira.lsstcorp.org/browse/DM-933) 425 - May include a field \c stargal; if present, True means that the object is a star 426 - May include a field \c var; if present, True means that the object is variable 428 The measured sources: 429 - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration 431 \throws RuntimeError with the following strings: 434 <DT> No matches to use for photocal 435 <DD> No matches are available (perhaps no sources/references were selected by the matcher). 436 <DT> No reference stars are available 437 <DD> No matches are available from which to extract magnitudes. 443 displaySources = display
and lsstDebug.Info(__name__).displaySources
447 from matplotlib
import pyplot
451 self.
fig = pyplot.figure()
453 filterName = exposure.getFilter().getName()
456 matchResults = self.
match.
run(sourceCat, filterName)
457 matches = matchResults.matches
458 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
461 if reserveResults.reserved.sum() > 0:
462 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
463 if len(matches) == 0:
464 raise RuntimeError(
"No matches to use for photocal")
467 mm.second.set(self.
usedKey,
True)
471 arrays = self.
extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
474 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
475 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
478 flux0 = 10**(0.4*r.zp)
479 flux0err = 0.4*math.log(10)*flux0*r.sigma
481 calib.setFluxMag0(flux0, flux0err)
483 return pipeBase.Struct(
493 """Display sources we'll use for photocal 495 Sources that will be actually used will be green. 496 Sources reserved from the fit will be red. 500 exposure : `lsst.afw.image.ExposureF` 502 matches : `list` of `lsst.afw.table.RefMatch` 503 Matches used for photocal. 504 reserved : `numpy.ndarray` of type `bool` 505 Boolean array indicating sources that are reserved. 507 Frame number for display. 509 ds9.mtv(exposure, frame=frame, title=
"photocal")
510 with ds9.Buffering():
511 for mm, rr
in zip(matches, reserved):
512 x, y = mm.second.getCentroid()
513 ctype = ds9.RED
if rr
else ds9.GREEN
514 ds9.dot(
"o", x, y, size=4, frame=frame, ctype=ctype)
518 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) 520 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: 521 1. We use the median/interquartile range to estimate the position to clip around, and the 523 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently 524 large estimate will prevent the clipping from ever taking effect. 525 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude 526 residuals with a tight core and asymmetrical outliers will start in the core. We use the width of 527 this core to set our maximum sigma (see 2.) 530 - zp ---------- Photometric zero point (mag) 531 - sigma ------- Standard deviation of fit of zero point (mag) 532 - ngood ------- Number of sources used to fit zero point 534 sigmaMax = self.config.sigmaMax
538 indArr = np.argsort(dmag)
541 if srcErr
is not None:
542 dmagErr = srcErr[indArr]
544 dmagErr = np.ones(len(dmag))
547 ind_noNan = np.array([i
for i
in range(len(dmag))
548 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
549 dmag = dmag[ind_noNan]
550 dmagErr = dmagErr[ind_noNan]
552 IQ_TO_STDEV = 0.741301109252802
557 for i
in range(self.config.nIter):
568 hist, edges = np.histogram(dmag, nhist, new=
True)
570 hist, edges = np.histogram(dmag, nhist)
571 imode = np.arange(nhist)[np.where(hist == hist.max())]
573 if imode[-1] - imode[0] + 1 == len(imode):
577 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
579 peak = sum(hist[imode])/len(imode)
583 while j >= 0
and hist[j] > 0.5*peak:
586 q1 = dmag[sum(hist[range(j)])]
589 while j < nhist
and hist[j] > 0.5*peak:
591 j = min(j, nhist - 1)
592 j = min(sum(hist[range(j)]), npt - 1)
596 q1 = dmag[int(0.25*npt)]
597 q3 = dmag[int(0.75*npt)]
604 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
608 sigmaMax = dmag[-1] - dmag[0]
610 center = np.median(dmag)
611 q1 = dmag[int(0.25*npt)]
612 q3 = dmag[int(0.75*npt)]
617 if self.config.useMedian:
618 center = np.median(gdmag)
620 gdmagErr = dmagErr[good]
621 center = np.average(gdmag, weights=gdmagErr)
623 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
624 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
626 sig = IQ_TO_STDEV*(q3 - q1)
628 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
635 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
637 axes.plot(ref[good], dmag[good] - center,
"b+")
638 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
639 linestyle=
'', color=
'b')
641 bad = np.logical_not(good)
642 if len(ref[bad]) > 0:
643 axes.plot(ref[bad], dmag[bad] - center,
"r+")
644 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
645 linestyle=
'', color=
'r') 647 axes.plot((-100, 100), (0, 0), "g-")
649 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
651 axes.set_ylim(-1.1, 1.1)
652 axes.set_xlim(24, 13)
653 axes.set_xlabel(
"Reference")
654 axes.set_ylabel(
"Reference - Instrumental")
660 while i == 0
or reply !=
"c":
662 reply = input(
"Next iteration? [ynhpc] ")
667 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
670 if reply
in (
"",
"c",
"n",
"p",
"y"):
673 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
680 except Exception
as e:
681 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
688 msg =
"PhotoCal.getZeroPoint: no good stars remain" 691 center = np.average(dmag, weights=dmagErr)
692 msg +=
" on first iteration; using average of all calibration stars" 696 return pipeBase.Struct(
700 elif ngood == old_ngood:
706 dmagErr = dmagErr[good]
709 dmagErr = dmagErr[good]
710 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
711 sigma = np.sqrt(1.0/weightSum)
712 return pipeBase.Struct(
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
def displaySources(self, exposure, matches, reserved, frame=1)