30 from lsst.afw.image import abMagFromFlux, abMagErrFromFluxErr, Calib
34 from .colorterms
import ColortermLibrary
36 __all__ = [
"PhotoCalTask",
"PhotoCalConfig"]
40 """Config for PhotoCal""" 41 match = pexConf.ConfigField(
"Match to reference catalog",
42 DirectMatchConfigWithoutLoader)
43 reserve = pexConf.ConfigurableField(target=ReserveSourcesTask, doc=
"Reserve sources from fitting")
44 fluxField = pexConf.Field(
46 default=
"slot_CalibFlux_instFlux",
47 doc=(
"Name of the source instFlux field to use. The associated flag field\n" 48 "('<name>_flags') will be implicitly included in badFlags."),
50 applyColorTerms = pexConf.Field(
53 doc=(
"Apply photometric color terms to reference stars? One of:\n" 54 "None: apply if colorterms and photoCatName are not None;\n" 55 " fail if color term data is not available for the specified ref catalog and filter.\n" 56 "True: always apply colorterms; fail if color term data is not available for the\n" 57 " specified reference catalog and filter.\n" 58 "False: do not apply."),
61 sigmaMax = pexConf.Field(
64 doc=
"maximum sigma to use when clipping",
67 nSigma = pexConf.Field(
72 useMedian = pexConf.Field(
75 doc=
"use median instead of mean to compute zeropoint",
77 nIter = pexConf.Field(
80 doc=
"number of iterations",
82 colorterms = pexConf.ConfigField(
83 dtype=ColortermLibrary,
84 doc=
"Library of photometric reference catalog name: color term dict",
86 photoCatName = pexConf.Field(
89 doc=(
"Name of photometric reference catalog; used to select a color term dict in colorterms." 90 " see also applyColorTerms"),
92 magErrFloor = pexConf.RangeField(
95 doc=
"Additional magnitude uncertainty to be added in quadrature with measurement errors.",
100 pexConf.Config.validate(self)
102 raise RuntimeError(
"applyColorTerms=True requires photoCatName is non-None")
104 raise RuntimeError(
"applyColorTerms=True requires colorterms be provided")
107 pexConf.Config.setDefaults(self)
108 self.
match.sourceSelection.doFlags =
True 109 self.
match.sourceSelection.flags.bad = [
110 "base_PixelFlags_flag_edge",
111 "base_PixelFlags_flag_interpolated",
112 "base_PixelFlags_flag_saturated",
114 self.
match.sourceSelection.doUnresolved =
True 126 @anchor PhotoCalTask_ 128 @brief Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector. 130 @section pipe_tasks_photocal_Contents Contents 132 - @ref pipe_tasks_photocal_Purpose 133 - @ref pipe_tasks_photocal_Initialize 134 - @ref pipe_tasks_photocal_IO 135 - @ref pipe_tasks_photocal_Config 136 - @ref pipe_tasks_photocal_Debug 137 - @ref pipe_tasks_photocal_Example 139 @section pipe_tasks_photocal_Purpose Description 141 @copybrief PhotoCalTask 143 Calculate an Exposure's zero-point given a set of flux measurements of stars matched to an input catalogue. 144 The type of flux to use is specified by PhotoCalConfig.fluxField. 146 The algorithm clips outliers iteratively, with parameters set in the configuration. 148 @note This task can adds fields to the schema, so any code calling this task must ensure that 149 these columns are indeed present in the input match list; see @ref pipe_tasks_photocal_Example 151 @section pipe_tasks_photocal_Initialize Task initialisation 153 @copydoc \_\_init\_\_ 155 @section pipe_tasks_photocal_IO Inputs/Outputs to the run method 159 @section pipe_tasks_photocal_Config Configuration parameters 161 See @ref PhotoCalConfig 163 @section pipe_tasks_photocal_Debug Debug variables 165 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 166 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. 168 The available variables in PhotoCalTask are: 171 <DD> If True enable other debug outputs 172 <DT> @c displaySources 173 <DD> If True, display the exposure on ds9's frame 1 and overlay the source catalogue. 176 <DD> Reserved objects 178 <DD> Objects used in the photometric calibration 181 <DD> Make a scatter plot of flux v. reference magnitude as a function of reference magnitude. 182 - good objects in blue 183 - rejected objects in red 184 (if @c scatterPlot is 2 or more, prompt to continue after each iteration) 187 @section pipe_tasks_photocal_Example A complete example of using PhotoCalTask 189 This code is in @link examples/photoCalTask.py@endlink, and can be run as @em e.g. 191 examples/photoCalTask.py 193 @dontinclude photoCalTask.py 195 Import the tasks (there are some other standard imports; read the file for details) 196 @skipline from lsst.pipe.tasks.astrometry 197 @skipline measPhotocal 199 We need to create both our tasks before processing any data as the task constructors 200 can add extra columns to the schema which we get from the input catalogue, @c scrCat: 204 @skip AstrometryTask.ConfigClass 206 (that @c filterMap line is because our test code doesn't use a filter that the reference catalogue recognises, 207 so we tell it to use the @c r band) 213 If the schema has indeed changed we need to add the new columns to the source table 214 (yes; this should be easier!) 218 We're now ready to process the data (we could loop over multiple exposures/catalogues using the same 223 We can then unpack and use the results: 228 To investigate the @ref pipe_tasks_photocal_Debug, put something like 232 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 233 if name.endswith(".PhotoCal"): 238 lsstDebug.Info = DebugInfo 240 into your debug.py file and run photoCalTask.py with the @c --debug flag. 242 ConfigClass = PhotoCalConfig
243 _DefaultName =
"photoCal" 245 def __init__(self, refObjLoader, schema=None, **kwds):
246 """!Create the photometric calibration task. See PhotoCalTask.init for documentation 248 pipeBase.Task.__init__(self, **kwds)
251 if schema
is not None:
252 self.
usedKey = schema.addField(
"calib_photometry_used", type=
"Flag",
253 doc=
"set if source was used in photometric calibration")
256 self.
match = DirectMatchTask(config=self.config.match, refObjLoader=refObjLoader,
257 name=
"match", parentTask=self)
258 self.makeSubtask(
"reserve", columnName=
"calib_photometry", schema=schema,
259 doc=
"set if source was reserved from photometric calibration")
262 """Return a struct containing the source catalog keys for fields used 268 schema : `lsst.afw.table.schema` 269 Schema of the catalog to get keys from. 273 result : `lsst.pipe.base.Struct` 274 Result struct with components: 276 - ``instFlux``: Instrument flux key. 277 - ``instFluxErr``: Instrument flux error key. 279 instFlux = schema.find(self.config.fluxField).key
280 instFluxErr = schema.find(self.config.fluxField +
"Err").key
281 return pipeBase.Struct(instFlux=instFlux, instFluxErr=instFluxErr)
285 """!Extract magnitude and magnitude error arrays from the given matches. 287 @param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink 288 @param[in] filterName Name of filter being calibrated 289 @param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() 291 @return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays 292 where magErr is an error in the magnitude; the error in srcMag - refMag 293 If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as 294 magErr is what is later used to determine the zero point. 295 Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes 297 @note These magnitude arrays are the @em inputs to the photometric calibration, some may have been 298 discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) 300 srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux)
for m
in matches])
301 srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr)
for m
in matches])
302 if not np.all(np.isfinite(srcInstFluxErrArr)):
304 self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux).")
305 srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
308 JanskysPerABFlux = 3631.0
309 srcInstFluxArr = srcInstFluxArr * JanskysPerABFlux
310 srcInstFluxErrArr = srcInstFluxErrArr * JanskysPerABFlux
313 raise RuntimeError(
"No reference stars are available")
314 refSchema = matches[0].first.schema
316 applyColorTerms = self.config.applyColorTerms
317 applyCTReason =
"config.applyColorTerms is %s" % (self.config.applyColorTerms,)
318 if self.config.applyColorTerms
is None:
320 ctDataAvail = len(self.config.colorterms.data) > 0
321 photoCatSpecified = self.config.photoCatName
is not None 322 applyCTReason +=
" and data %s available" % (
"is" if ctDataAvail
else "is not")
323 applyCTReason +=
" and photoRefCat %s provided" % (
"is" if photoCatSpecified
else "is not")
324 applyColorTerms = ctDataAvail
and photoCatSpecified
327 self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
328 filterName, self.config.photoCatName, applyCTReason)
329 ct = self.config.colorterms.getColorterm(
330 filterName=filterName, photoCatName=self.config.photoCatName, doRaise=
True)
332 self.log.info(
"Not applying color terms because %s", applyCTReason)
336 fluxFieldList = [getRefFluxField(refSchema, filt)
for filt
in (ct.primary, ct.secondary)]
337 missingFluxFieldList = []
338 for fluxField
in fluxFieldList:
340 refSchema.find(fluxField).key
342 missingFluxFieldList.append(fluxField)
344 if missingFluxFieldList:
345 self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
346 " ".join(missingFluxFieldList))
350 fluxFieldList = [getRefFluxField(refSchema, filterName)]
353 refFluxErrArrList = []
354 for fluxField
in fluxFieldList:
355 fluxKey = refSchema.find(fluxField).key
356 refFluxArr = np.array([m.first.get(fluxKey)
for m
in matches])
358 fluxErrKey = refSchema.find(fluxField +
"Err").key
359 refFluxErrArr = np.array([m.first.get(fluxErrKey)
for m
in matches])
362 self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
364 refFluxErrArr = np.sqrt(refFluxArr)
366 refFluxArrList.append(refFluxArr)
367 refFluxErrArrList.append(refFluxErrArr)
370 refMagArr1 = np.array([abMagFromFlux(rf1)
for rf1
in refFluxArrList[0]])
371 refMagArr2 = np.array([abMagFromFlux(rf2)
for rf2
in refFluxArrList[1]])
373 refMagArr = ct.transformMags(refMagArr1, refMagArr2)
374 refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
376 refMagArr = np.array([abMagFromFlux(rf)
for rf
in refFluxArrList[0]])
378 srcMagArr = np.array([abMagFromFlux(sf)
for sf
in srcInstFluxArr])
382 magErrArr = np.array([abMagErrFromFluxErr(fe, sf)
383 for fe, sf
in zip(srcInstFluxErrArr, srcInstFluxArr)])
384 if self.config.magErrFloor != 0.0:
385 magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
387 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf)
388 for sfe, sf
in zip(srcInstFluxErrArr, srcInstFluxArr)])
389 refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf)
390 for rfe, rf
in zip(refFluxErrArr, refFluxArr)])
392 good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
394 return pipeBase.Struct(
395 srcMag=srcMagArr[good],
396 refMag=refMagArr[good],
397 magErr=magErrArr[good],
398 srcMagErr=srcMagErrArr[good],
399 refMagErr=refMagErrArr[good],
400 refFluxFieldList=fluxFieldList,
404 def run(self, exposure, sourceCat, expId=0):
405 """!Do photometric calibration - select matches to use and (possibly iteratively) compute 408 @param[in] exposure Exposure upon which the sources in the matches were detected. 409 @param[in] sourceCat A catalog of sources to use in the calibration 410 (@em i.e. a list of lsst.afw.table.Match with 411 @c first being of type lsst.afw.table.SimpleRecord and @c second type lsst.afw.table.SourceRecord --- 412 the reference object and matched object respectively). 413 (will not be modified except to set the outputField if requested.). 416 - calib ------- @link lsst::afw::image::Calib@endlink object containing the zero point 417 - arrays ------ Magnitude arrays returned be PhotoCalTask.extractMagArrays 418 - matches ----- Final ReferenceMatchVector, as returned by PhotoCalTask.selectMatches. 419 - zp ---------- Photometric zero point (mag) 420 - sigma ------- Standard deviation of fit of photometric zero point (mag) 421 - ngood ------- Number of sources used to fit photometric zero point 423 The exposure is only used to provide the name of the filter being calibrated (it may also be 424 used to generate debugging plots). 426 The reference objects: 427 - Must include a field @c photometric; True for objects which should be considered as 428 photometric standards 429 - Must include a field @c flux; the flux used to impose a magnitude limit and also to calibrate 430 the data to (unless a color term is specified, in which case ColorTerm.primary is used; 431 See https://jira.lsstcorp.org/browse/DM-933) 432 - May include a field @c stargal; if present, True means that the object is a star 433 - May include a field @c var; if present, True means that the object is variable 435 The measured sources: 436 - Must include PhotoCalConfig.fluxField; the flux measurement to be used for calibration 438 @throws RuntimeError with the following strings: 441 <DT> No matches to use for photocal 442 <DD> No matches are available (perhaps no sources/references were selected by the matcher). 443 <DT> No reference stars are available 444 <DD> No matches are available from which to extract magnitudes. 450 displaySources = display
and lsstDebug.Info(__name__).displaySources
454 from matplotlib
import pyplot
458 self.
fig = pyplot.figure()
460 filterName = exposure.getFilter().getName()
463 matchResults = self.
match.
run(sourceCat, filterName)
464 matches = matchResults.matches
465 reserveResults = self.reserve.
run([mm.second
for mm
in matches], expId=expId)
468 if reserveResults.reserved.sum() > 0:
469 matches = [mm
for mm, use
in zip(matches, reserveResults.use)
if use]
470 if len(matches) == 0:
471 raise RuntimeError(
"No matches to use for photocal")
474 mm.second.set(self.
usedKey,
True)
478 arrays = self.
extractMagArrays(matches=matches, filterName=filterName, sourceKeys=sourceKeys)
481 r = self.
getZeroPoint(arrays.srcMag, arrays.refMag, arrays.magErr)
482 self.log.info(
"Magnitude zero point: %f +/- %f from %d stars", r.zp, r.sigma, r.ngood)
485 flux0 = 10**(0.4*r.zp)
486 flux0err = 0.4*math.log(10)*flux0*r.sigma
488 calib.setFluxMag0(flux0, flux0err)
490 return pipeBase.Struct(
500 """Display sources we'll use for photocal 502 Sources that will be actually used will be green. 503 Sources reserved from the fit will be red. 507 exposure : `lsst.afw.image.ExposureF` 509 matches : `list` of `lsst.afw.table.RefMatch` 510 Matches used for photocal. 511 reserved : `numpy.ndarray` of type `bool` 512 Boolean array indicating sources that are reserved. 514 Frame number for display. 516 ds9.mtv(exposure, frame=frame, title=
"photocal")
517 with ds9.Buffering():
518 for mm, rr
in zip(matches, reserved):
519 x, y = mm.second.getCentroid()
520 ctype = ds9.RED
if rr
else ds9.GREEN
521 ds9.dot(
"o", x, y, size=4, frame=frame, ctype=ctype)
524 """!Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) 526 We perform nIter iterations of a simple sigma-clipping algorithm with a couple of twists: 527 1. We use the median/interquartile range to estimate the position to clip around, and the 529 2. We never allow sigma to go _above_ a critical value sigmaMax --- if we do, a sufficiently 530 large estimate will prevent the clipping from ever taking effect. 531 3. Rather than start with the median we start with a crude mode. This means that a set of magnitude 532 residuals with a tight core and asymmetrical outliers will start in the core. We use the width of 533 this core to set our maximum sigma (see 2.) 536 - zp ---------- Photometric zero point (mag) 537 - sigma ------- Standard deviation of fit of zero point (mag) 538 - ngood ------- Number of sources used to fit zero point 540 sigmaMax = self.config.sigmaMax
544 indArr = np.argsort(dmag)
547 if srcErr
is not None:
548 dmagErr = srcErr[indArr]
550 dmagErr = np.ones(len(dmag))
553 ind_noNan = np.array([i
for i
in range(len(dmag))
554 if (
not np.isnan(dmag[i])
and not np.isnan(dmagErr[i]))])
555 dmag = dmag[ind_noNan]
556 dmagErr = dmagErr[ind_noNan]
558 IQ_TO_STDEV = 0.741301109252802
563 for i
in range(self.config.nIter):
574 hist, edges = np.histogram(dmag, nhist, new=
True)
576 hist, edges = np.histogram(dmag, nhist)
577 imode = np.arange(nhist)[np.where(hist == hist.max())]
579 if imode[-1] - imode[0] + 1 == len(imode):
583 center = 0.5*(edges[imode[0]] + edges[imode[-1] + 1])
585 peak = sum(hist[imode])/len(imode)
589 while j >= 0
and hist[j] > 0.5*peak:
592 q1 = dmag[sum(hist[range(j)])]
595 while j < nhist
and hist[j] > 0.5*peak:
597 j = min(j, nhist - 1)
598 j = min(sum(hist[range(j)]), npt - 1)
602 q1 = dmag[int(0.25*npt)]
603 q3 = dmag[int(0.75*npt)]
610 self.log.debug(
"Photo calibration histogram: center = %.2f, sig = %.2f", center, sig)
614 sigmaMax = dmag[-1] - dmag[0]
616 center = np.median(dmag)
617 q1 = dmag[int(0.25*npt)]
618 q3 = dmag[int(0.75*npt)]
623 if self.config.useMedian:
624 center = np.median(gdmag)
626 gdmagErr = dmagErr[good]
627 center = np.average(gdmag, weights=gdmagErr)
629 q3 = gdmag[min(int(0.75*npt + 0.5), npt - 1)]
630 q1 = gdmag[min(int(0.25*npt + 0.5), npt - 1)]
632 sig = IQ_TO_STDEV*(q3 - q1)
634 good = abs(dmag - center) < self.config.nSigma*min(sig, sigmaMax)
641 axes = self.
fig.add_axes((0.1, 0.1, 0.85, 0.80))
643 axes.plot(ref[good], dmag[good] - center,
"b+")
644 axes.errorbar(ref[good], dmag[good] - center, yerr=dmagErr[good],
645 linestyle=
'', color=
'b')
647 bad = np.logical_not(good)
648 if len(ref[bad]) > 0:
649 axes.plot(ref[bad], dmag[bad] - center,
"r+")
650 axes.errorbar(ref[bad], dmag[bad] - center, yerr=dmagErr[bad],
651 linestyle=
'', color=
'r') 653 axes.plot((-100, 100), (0, 0), "g-")
655 axes.plot((-100, 100), x*0.05*np.ones(2),
"g--")
657 axes.set_ylim(-1.1, 1.1)
658 axes.set_xlim(24, 13)
659 axes.set_xlabel(
"Reference")
660 axes.set_ylabel(
"Reference - Instrumental")
666 while i == 0
or reply !=
"c":
668 reply = input(
"Next iteration? [ynhpc] ")
673 print(
"Options: c[ontinue] h[elp] n[o] p[db] y[es]", file=sys.stderr)
676 if reply
in (
"",
"c",
"n",
"p",
"y"):
679 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
686 except Exception
as e:
687 print(
"Error plotting in PhotoCal.getZeroPoint: %s" % e, file=sys.stderr)
694 msg =
"PhotoCal.getZeroPoint: no good stars remain" 697 center = np.average(dmag, weights=dmagErr)
698 msg +=
" on first iteration; using average of all calibration stars" 702 return pipeBase.Struct(
706 elif ngood == old_ngood:
712 dmagErr = dmagErr[good]
715 dmagErr = dmagErr[good]
716 zp, weightSum = np.average(dmag, weights=1/dmagErr**2, returned=
True)
717 sigma = np.sqrt(1.0/weightSum)
718 return pipeBase.Struct(
def __init__(self, refObjLoader, schema=None, kwds)
Create the photometric calibration task.
def run(self, exposure, sourceCat, expId=0)
Do photometric calibration - select matches to use and (possibly iteratively) compute the zero point...
def getSourceKeys(self, schema)
def extractMagArrays(self, matches, filterName, sourceKeys)
Extract magnitude and magnitude error arrays from the given matches.
Calculate the zero point of an exposure given a lsst.afw.table.ReferenceMatchVector.
def getZeroPoint(self, src, ref, srcErr=None, zp0=None)
Flux calibration code, returning (ZeroPoint, Distribution Width, Number of stars) ...
def displaySources(self, exposure, matches, reserved, frame=1)