30 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, Calib
34 from .colorterms
import ColortermLibrary
36 __all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
40 """Config for PhotoCal""" 41 match = pexConf.ConfigField(
"Match to reference catalog",
42 DirectMatchConfigWithoutLoader)
43 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
44 fluxField = pexConf.Field(
46 default=
"slot_CalibFlux_flux",
47 doc=(
"Name of the source flux field to use. The associated flag field\n" 48 "('<name>_flags') will be implicitly included in badFlags."),
50 applyColorTerms = pexConf.Field(
53 doc=(
"Apply photometric color terms to reference stars? One of:\n" 54 "None: apply if colorterms and photoCatName are not None;\n" 55 " fail if color term data is not available for the specified ref catalog and filter.\n" 56 "True: always apply colorterms; fail if color term data is not available for the\n" 57 " specified reference catalog and filter.\n" 58 "False: do not apply."),
61 sigmaMax = pexConf.Field(
64 doc=
"maximum sigma to use when clipping",
67 nSigma = pexConf.Field(
72 useMedian = pexConf.Field(
75 doc=
"use median instead of mean to compute zeropoint",
77 nIter = pexConf.Field(
80 doc=
"number of iterations",
82 colorterms = pexConf.ConfigField(
83 dtype=ColortermLibrary,
84 doc=
"Library of photometric reference catalog name: color term dict",
86 photoCatName = pexConf.Field(
89 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms." 90 " see also applyColorTerms"),
92 magErrFloor = pexConf.RangeField(
95 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
100 pexConf.Config.validate(self)
102 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
104 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
107 pexConf.Config.setDefaults(self)
108 self.
match.sourceSelection.doFlags =
True 109 self.
match.sourceSelection.flags.bad = [
110 "base_PixelFlags_flag_edge",
111 "base_PixelFlags_flag_interpolated",
112 "base_PixelFlags_flag_saturated",
114 self.
match.sourceSelection.doUnresolved =
True 126 @anchor PhotoCalTask_ 128 @brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. 130 @section pipe_tasks_photocal_Contents Contents 132 - @ref pipe_tasks_photocal_Purpose 133 - @ref pipe_tasks_photocal_Initialize 134 - @ref pipe_tasks_photocal_IO 135 - @ref pipe_tasks_photocal_Config 136 - @ref pipe_tasks_photocal_Debug 137 - @ref pipe_tasks_photocal_Example 139 @section pipe_tasks_photocal_Purpose Description 141 @copybrief PhotoCalTask 143 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. 144 The type of flux to use is specified by PhotoCalConfig.fluxField. 146 The algorithm clips outliers iteratively, with parameters set in the configuration. 148 @note This task can adds fields to the schema, so any code calling this task must ensure that 149 these columns are indeed present in the input match list; see @ref pipe_tasks_photocal_Example 151 @section pipe_tasks_photocal_Initialize Task initialisation 153 @copydoc \_\_init\_\_ 155 @section pipe_tasks_photocal_IO Inputs/Outputs to the run method 159 @section pipe_tasks_photocal_Config Configuration parameters 161 See @ref PhotoCalConfig 163 @section pipe_tasks_photocal_Debug Debug variables 165 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 166 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. 168 The available variables in PhotoCalTask are: 171 <DD> If True enable other debug outputs 172 <DT> @c displaySources 173 <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue. 176 <DD> Reserved objects 178 <DD> Objects used in the photometric calibration 181 <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. 182 - good objects in blue 183 - rejected objects in red 184 (if @c scatterPlot is 2 or more, prompt to continue after each iteration) 187 @section pipe_tasks_photocal_Example A complete example of using PhotoCalTask 189 This code is in @link examples/photoCalTask.py@endlink, and can be run as @em e.g. 191 examples/photoCalTask.py 193 @dontinclude photoCalTask.py 195 Import the tasks (there are some other standard imports; read the file for details) 196 @skipline from lsst.pipe.tasks.astrometry 197 @skipline measPhotocal 199 We need to create both our tasks before processing any data as the task constructors 200 can add extra columns to the schema which we get from the input catalogue, @c scrCat: 204 @skip AstrometryTask.ConfigClass 206 (that @c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, 207 so we tell it to use the @c r band) 213 If the schema has indeed changed we need to add the new columns to the source table 214 (yes; this should be easier!) 218 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same 223 We can then unpack and use the results: 228 To investigate the @ref pipe_tasks_photocal_Debug, put something like 232 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 233 if name.endswith(".PhotoCal"): 238 lsstDebug.Info = DebugInfo 240 into your debug.py file and run photoCalTask.py with the @c --debug flag. 242 ConfigClass = PhotoCalConfig
243 _DefaultName =
"photoCal" 245 def __init__(self, refObjLoader, schema=None, **kwds):
246 """!Create the photometric calibration task. See PhotoCalTask.init for documentation 248 pipeBase.Task.__init__(self, **kwds)
251 if schema
is not None:
252 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
253 doc=
"set if source was used in photometric calibration")
256 self.
match = DirectMatchTask(self.config.match, refObjLoader=refObjLoader,
257 name=
"match", parentTask=self)
258 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
259 doc=
"set if source was reserved from photometric calibration")
262 """!Return a struct containing the source catalog keys for fields used by PhotoCalTask. 264 Returned fields include: 268 flux = schema.find(self.config.fluxField).key
269 fluxErr = schema.find(self.config.fluxField +
"Sigma").key
270 return pipeBase.Struct(flux=flux, fluxErr=fluxErr)
274 """!Extract magnitude and magnitude error arrays from the given matches. 276 @param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink 277 @param[in] filterName Name of filter being calibrated 278 @param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() 280 @return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays 281 where magErr is an error in the magnitude; the error in srcMag - refMag 282 If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as 283 magErr is what is later used to determine the zero point. 284 Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes 286 @note These magnitude arrays are the @em inputs to the photometric calibration, some may have been 287 discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) 289 srcFluxArr = np.array([m.second.get(sourceKeys.flux)
for m
in matches])
290 srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr)
for m
in matches])
291 if not np.all(np.isfinite(srcFluxErrArr)):
293 self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
294 srcFluxErrArr = np.sqrt(srcFluxArr)
297 JanskysPerABFlux = 3631.0
298 srcFluxArr = srcFluxArr * JanskysPerABFlux
299 srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
302 raise RuntimeError(
"No reference stars are available")
303 refSchema = matches[0].first.schema
305 applyColorTerms = self.config.applyColorTerms
306 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
307 if self.config.applyColorTerms
is None:
309 ctDataAvail = len(self.config.colorterms.data) > 0
310 photoCatSpecified = self.config.photoCatName
is not None 311 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
312 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
313 applyColorTerms = ctDataAvail
and photoCatSpecified
316 self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
317 filterName, self.config.photoCatName, applyCTReason)
318 ct = self.config.colorterms.getColorterm(
319 filterName=filterName, photoCatName=self.config.photoCatName, doRaise=
True)
321 self.log.info(
"Not applying color terms because %s", applyCTReason)
325 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (ct.primary, ct.secondary)]
326 missingFluxFieldList = []
327 for fluxField
in fluxFieldList:
329 refSchema.find(fluxField).key
331 missingFluxFieldList.append(fluxField)
333 if missingFluxFieldList:
334 self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
335 " ".join(missingFluxFieldList))
339 fluxFieldList = [getRefFluxField(refSchema, filterName)]
342 refFluxErrArrList = []
343 for fluxField
in fluxFieldList:
344 fluxKey = refSchema.find(fluxField).key
345 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
347 fluxErrKey = refSchema.find(fluxField +
"Sigma").key
348 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
351 self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
353 refFluxErrArr = np.sqrt(refFluxArr)
355 refFluxArrList.append(refFluxArr)
356 refFluxErrArrList.append(refFluxErrArr)
359 refMagArr1 = np.array([abMagFromFlux(rf1)
for rf1
in refFluxArrList[0]])
360 refMagArr2 = np.array([abMagFromFlux(rf2)
for rf2
in refFluxArrList[1]])
362 refMagArr = ct.transformMags(refMagArr1, refMagArr2)
363 refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
365 refMagArr = np.array([abMagFromFlux(rf)
for rf
in refFluxArrList[0]])
367 srcMagArr = np.array([abMagFromFlux(sf)
for sf
in srcFluxArr])
371 magErrArr = np.array([abMagErrFromFluxErr(fe, sf)
for fe, sf
in zip(srcFluxErrArr, srcFluxArr)])
372 if self.config.magErrFloor != 0.0:
373 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
375 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf)
for sfe, sf
in zip(srcFluxErrArr, srcFluxArr)])
376 refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf)
for rfe, rf
in zip(refFluxErrArr, refFluxArr)])
378 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
380 return pipeBase.Struct(
381 srcMag=srcMagArr[good],
382 refMag=refMagArr[good],
383 magErr=magErrArr[good],
384 srcMagErr=srcMagErrArr[good],
385 refMagErr=refMagErrArr[good],
386 refFluxFieldList=fluxFieldList,
390 def run(self, exposure, sourceCat, expId=0):
391 """!Do photometric calibration - select matches to use and (possibly iteratively) compute 394 @param[in] exposure Exposure upon which the sources in the matches were detected. 395 @param[in] sourceCat A catalog of sources to use in the calibration 396 (@em i.e. a list of lsst.afw.table.Match with 397 @c first being of type lsst.afw.table.SimpleRecord and @c second type lsst.afw.table.SourceRecord --- 398 the reference object and matched object respectively). 399 (will not be modified except to set the outputField if requested.). 402 - calib ------- @link lsst::afw::image::Calib@endlink object containing the zero point 403 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays 404 - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. 405 - zp ---------- Photometric zero point (mag) 406 - sigma ------- Standard deviation of fit of photometric zero point (mag) 407 - ngood ------- Number of sources used to fit photometric zero point 409 The exposure is only used to provide the name of the filter being calibrated (it may also be 410 used to generate debugging plots). 412 The reference objects: 413 - Must include a field @c photometric; True for objects which should be considered as 414 photometric standards 415 - Must include a field @c flux; the flux used to impose a magnitude limit and also to calibrate 416 the data to (unless a color term is specified, in which case ColorTerm.primary is used; 417 See https://jira.lsstcorp.org/browse/DM-933) 418 - May include a field @c stargal; if present, True means that the object is a star 419 - May include a field @c var; if present, True means that the object is variable 421 The measured sources: 422 - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration 424 @throws RuntimeError with the following strings: 427 <DT> No matches to use for photocal 428 <DD> No matches are available (perhaps no sources/references were selected by the matcher). 429 <DT> No reference stars are available 430 <DD> No matches are available from which to extract magnitudes. 436 displaySources = display
and lsstDebug.Info(__name__).displaySources
440 from matplotlib
import pyplot
444 self.
fig = pyplot.figure()
446 filterName = exposure.getFilter().getName()
449 matchResults = self.
match.
run(sourceCat, filterName)
450 matches = matchResults.matches
451 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
454 if reserveResults.reserved.sum() > 0:
455 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
456 if len(matches) == 0:
457 raise RuntimeError(
"No matches to use for photocal")
460 mm.second.set(self.
usedKey,
True)
464 arrays = self.
extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
467 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
468 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
471 flux0 = 10**(0.4*r.zp)
472 flux0err = 0.4*math.log(10)*flux0*r.sigma
474 calib.setFluxMag0(flux0, flux0err)
476 return pipeBase.Struct(
486 """Display sources we'll use for photocal 488 Sources that will be actually used will be green. 489 Sources reserved from the fit will be red. 493 exposure : `lsst.afw.image.ExposureF` 495 matches : `list` of `lsst.afw.table.RefMatch` 496 Matches used for photocal. 497 reserved : `numpy.ndarray` of type `bool` 498 Boolean array indicating sources that are reserved. 500 Frame number for display. 502 ds9.mtv(exposure, frame=frame, title=
"photocal")
503 with ds9.Buffering():
504 for mm, rr
in zip(matches, reserved):
505 x, y = mm.second.getCentroid()
506 ctype = ds9.RED
if rr
else ds9.GREEN
507 ds9.dot(
"o", x, y, size=4, frame=frame, ctype=ctype)
510 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) 512 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: 513 1. We use the median/interquartile range to estimate the position to clip around, and the 515 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently 516 large estimate will prevent the clipping from ever taking effect. 517 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude 518 residuals with a tight core and asymmetrical outliers will start in the core. We use the width of 519 this core to set our maximum sigma (see 2.) 522 - zp ---------- Photometric zero point (mag) 523 - sigma ------- Standard deviation of fit of zero point (mag) 524 - ngood ------- Number of sources used to fit zero point 526 sigmaMax = self.config.sigmaMax
530 indArr = np.argsort(dmag)
533 if srcErr
is not None:
534 dmagErr = srcErr[indArr]
536 dmagErr = np.ones(len(dmag))
539 ind_noNan = np.array([i
for i
in range(len(dmag))
540 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
541 dmag = dmag[ind_noNan]
542 dmagErr = dmagErr[ind_noNan]
544 IQ_TO_STDEV = 0.741301109252802
549 for i
in range(self.config.nIter):
560 hist, edges = np.histogram(dmag, nhist, new=
True)
562 hist, edges = np.histogram(dmag, nhist)
563 imode = np.arange(nhist)[np.where(hist == hist.max())]
565 if imode[-1] - imode[0] + 1 == len(imode):
569 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
571 peak = sum(hist[imode])/len(imode)
575 while j >= 0
and hist[j] > 0.5*peak:
578 q1 = dmag[sum(hist[range(j)])]
581 while j < nhist
and hist[j] > 0.5*peak:
583 j = min(j, nhist - 1)
584 j = min(sum(hist[range(j)]), npt - 1)
588 q1 = dmag[int(0.25*npt)]
589 q3 = dmag[int(0.75*npt)]
596 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
600 sigmaMax = dmag[-1] - dmag[0]
602 center = np.median(dmag)
603 q1 = dmag[int(0.25*npt)]
604 q3 = dmag[int(0.75*npt)]
609 if self.config.useMedian:
610 center = np.median(gdmag)
612 gdmagErr = dmagErr[good]
613 center = np.average(gdmag, weights=gdmagErr)
615 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
616 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
618 sig = IQ_TO_STDEV*(q3 - q1)
620 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
627 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
629 axes.plot(ref[good], dmag[good] - center,
"b+")
630 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
631 linestyle=
'', color=
'b')
633 bad = np.logical_not(good)
634 if len(ref[bad]) > 0:
635 axes.plot(ref[bad], dmag[bad] - center,
"r+")
636 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
637 linestyle=
'', color=
'r') 639 axes.plot((-100, 100), (0, 0), "g-")
641 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
643 axes.set_ylim(-1.1, 1.1)
644 axes.set_xlim(24, 13)
645 axes.set_xlabel(
"Reference")
646 axes.set_ylabel(
"Reference - Instrumental")
652 while i == 0
or reply !=
"c":
654 reply = input(
"Next iteration? [ynhpc] ")
659 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
662 if reply
in (
"",
"c",
"n",
"p",
"y"):
665 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
672 except Exception
as e:
673 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
680 msg =
"PhotoCal.getZeroPoint: no good stars remain" 683 center = np.average(dmag, weights=dmagErr)
684 msg +=
" on first iteration; using average of all calibration stars" 688 return pipeBase.Struct(
692 elif ngood == old_ngood:
698 dmagErr = dmagErr[good]
701 dmagErr = dmagErr[good]
702 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
703 sigma = np.sqrt(1.0/weightSum)
704 return pipeBase.Struct(
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
def getSourceKeys(self, schema)
Return a struct containing the source catalog keys for fields used by PhotoCalTask.
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
def displaySources(self, exposure, matches, reserved, frame=1)