26 import lsst.pex.config
as pexConfig
33 from lsst.meas.base import ForcedMeasurementTask, EvaluateLocalCalibrationTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
38 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
39 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
40 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
41 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
46 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 """Config for ImageDifferenceTask 53 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
54 doc=
"Add background to calexp before processing it. " 55 "Useful as ipDiffim does background matching.")
56 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
57 doc=
"Use image-to-image registration to align template with " 59 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
60 doc=
"Writing debugging data for doUseRegister")
61 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
62 doc=
"Select stars to use for kernel fitting")
63 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
64 doc=
"Select stars of extreme color as part of the control sample")
65 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
66 doc=
"Select stars that are variable to be part " 67 "of the control sample")
68 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
69 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
70 doc=
"Convolve science image by its PSF before PSF-matching?")
71 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
72 doc=
"Scale variance of the template before PSF matching")
73 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
74 doc=
"Use a simple gaussian PSF model for pre-convolution " 75 "(else use fit PSF)? Ignored if doPreConvolve false.")
76 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
77 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
78 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L " 79 "kernel convolution? If True, also update the diffim PSF.")
80 doMerge = pexConfig.Field(dtype=bool, default=
True,
81 doc=
"Merge positive and negative diaSources with grow radius " 82 "set by growFootprint")
83 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
84 doc=
"Match diaSources with input calexp sources and ref catalog sources")
85 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
86 doEvalLocCalibration = pexConfig.Field(
89 doc=
"Store calibration products (local wcs and photoCalib) in the " 90 "output DiaSource catalog.")
91 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
92 doForcedMeasurement = pexConfig.Field(
95 doc=
"Force photometer diaSource locations on PVI?")
96 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
97 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
98 doc=
"Write warped and PSF-matched template coadd exposure?")
99 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
100 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
101 doc=
"Add columns to the source table to hold analysis metrics?")
103 coaddName = pexConfig.Field(
104 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
108 convolveTemplate = pexConfig.Field(
109 doc=
"Which image gets convolved (default = template)",
113 refObjLoader = pexConfig.ConfigurableField(
114 target=LoadIndexedReferenceObjectsTask,
115 doc=
"reference object loader",
117 astrometer = pexConfig.ConfigurableField(
118 target=AstrometryTask,
119 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
121 sourceSelector = pexConfig.ConfigurableField(
122 target=ObjectSizeStarSelectorTask,
123 doc=
"Source selection algorithm",
125 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
126 decorrelate = pexConfig.ConfigurableField(
127 target=DecorrelateALKernelSpatialTask,
128 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 129 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 132 doSpatiallyVarying = pexConfig.Field(
135 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the " 136 "image in order to allow for spatial variations" 138 detection = pexConfig.ConfigurableField(
139 target=SourceDetectionTask,
140 doc=
"Low-threshold detection for final measurement",
142 measurement = pexConfig.ConfigurableField(
143 target=DipoleFitTask,
144 doc=
"Enable updated dipole fitting method",
146 evalLocCalib = pexConfig.ConfigurableField(
147 target=EvaluateLocalCalibrationTask,
148 doc=
"Task to strip calibrations from an exposure and store their " 149 "local values in the output DiaSource catalog." 151 forcedMeasurement = pexConfig.ConfigurableField(
152 target=ForcedMeasurementTask,
153 doc=
"Subtask to force photometer PVI at diaSource location.",
155 getTemplate = pexConfig.ConfigurableField(
156 target=GetCoaddAsTemplateTask,
157 doc=
"Subtask to retrieve template exposure and sources",
159 scaleVariance = pexConfig.ConfigurableField(
160 target=ScaleVarianceTask,
161 doc=
"Subtask to rescale the variance of the template " 162 "to the statistically expected level" 164 controlStepSize = pexConfig.Field(
165 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
169 controlRandomSeed = pexConfig.Field(
170 doc=
"Random seed for shuffing the control sample",
174 register = pexConfig.ConfigurableField(
176 doc=
"Task to enable image-to-image image registration (warping)",
178 kernelSourcesFromRef = pexConfig.Field(
179 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
183 templateSipOrder = pexConfig.Field(
184 dtype=int, default=2,
185 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)" 187 growFootprint = pexConfig.Field(
188 dtype=int, default=2,
189 doc=
"Grow positive and negative footprints by this amount before merging" 191 diaSourceMatchRadius = pexConfig.Field(
192 dtype=float, default=0.5,
193 doc=
"Match radius (in arcseconds) for DiaSource to Source association" 199 self.
subtract[
'al'].kernel.name =
"AL" 200 self.
subtract[
'al'].kernel.active.fitForBackground =
True 201 self.
subtract[
'al'].kernel.active.spatialKernelOrder = 1
202 self.
subtract[
'al'].kernel.active.spatialBgOrder = 2
209 self.
detection.thresholdPolarity =
"both" 211 self.
detection.reEstimateBackground =
False 212 self.
detection.thresholdType =
"pixel_stdev" 218 self.
measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
222 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
230 pexConfig.Config.validate(self)
232 raise ValueError(
"Cannot run source measurement without source detection.")
234 raise ValueError(
"Cannot run source merging without source detection.")
236 raise ValueError(
"doUseRegister=True and doSelectSources=False. " 237 "Cannot run RegisterTask without selecting sources.")
239 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and " 240 "convolveTemplate=False is not supported.")
243 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
250 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
255 """Subtract an image from a template and measure the result 257 ConfigClass = ImageDifferenceConfig
258 RunnerClass = ImageDifferenceTaskRunner
259 _DefaultName =
"imageDifference" 262 """!Construct an ImageDifference Task 264 @param[in] butler Butler object to use in constructing reference object loaders 266 pipeBase.CmdLineTask.__init__(self, **kwargs)
267 self.makeSubtask(
"getTemplate")
269 self.makeSubtask(
"subtract")
271 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
272 self.makeSubtask(
"decorrelate")
274 if self.config.doScaleTemplateVariance:
275 self.makeSubtask(
"scaleVariance")
277 if self.config.doUseRegister:
278 self.makeSubtask(
"register")
279 self.
schema = afwTable.SourceTable.makeMinimalSchema()
281 if self.config.doSelectSources:
282 self.makeSubtask(
"sourceSelector")
283 if self.config.kernelSourcesFromRef:
284 self.makeSubtask(
'refObjLoader', butler=butler)
285 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
288 if self.config.doDetection:
289 self.makeSubtask(
"detection", schema=self.
schema)
290 if self.config.doMeasurement:
291 self.makeSubtask(
"measurement", schema=self.
schema,
293 if self.config.doEvalLocCalibration
and self.config.doMeasurement:
294 self.makeSubtask(
"evalLocCalib", schema=self.
schema)
295 if self.config.doForcedMeasurement:
297 "ip_diffim_forced_PsfFlux_instFlux",
"D",
298 "Forced PSF flux measured on the direct image.")
300 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
301 "Forced PSF flux error measured on the direct image.")
303 "ip_diffim_forced_PsfFlux_area",
"F",
304 "Forced PSF flux effective area of PSF.",
307 "ip_diffim_forced_PsfFlux_flag",
"Flag",
308 "Forced PSF flux general failure flag.")
310 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
311 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
313 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
314 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
315 self.makeSubtask(
"forcedMeasurement", refSchema=self.
schema)
316 if self.config.doMatchSources:
317 self.
schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
318 self.
schema.addField(
"srcMatchId",
"L",
"unique id of source match")
322 """Subtract an image from a template coadd and measure the result 325 - warp template coadd to match WCS of image 326 - PSF match image to warped template 327 - subtract image from PSF-matched, warped template 328 - persist difference image 332 @param sensorRef: sensor-level butler data reference, used for the following data products: 338 - self.config.coaddName + "Coadd_skyMap" 339 - self.config.coaddName + "Coadd" 340 Input or output, depending on config: 341 - self.config.coaddName + "Diff_subtractedExp" 342 Output, depending on config: 343 - self.config.coaddName + "Diff_matchedExp" 344 - self.config.coaddName + "Diff_src" 346 @return pipe_base Struct containing these fields: 347 - subtractedExposure: exposure after subtracting template; 348 the unpersisted version if subtraction not run but detection run 349 None if neither subtraction nor detection run (i.e. nothing useful done) 350 - subtractRes: results of subtraction task; None if subtraction not run 351 - sources: detected and possibly measured sources; None if detection not run 353 self.log.info(
"Processing %s" % (sensorRef.dataId))
356 subtractedExposure =
None 360 controlSources =
None 366 expBits = sensorRef.get(
"ccdExposureId_bits")
367 expId = int(sensorRef.get(
"ccdExposureId"))
368 idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
371 exposure = sensorRef.get(
"calexp", immediate=
True)
372 if self.config.doAddCalexpBackground:
373 mi = exposure.getMaskedImage()
374 mi += sensorRef.get(
"calexpBackground").getImage()
375 if not exposure.hasPsf():
376 raise pipeBase.TaskError(
"Exposure has no psf")
377 sciencePsf = exposure.getPsf()
379 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp" 380 templateExposure =
None 381 templateSources =
None 382 if self.config.doSubtract:
383 template = self.getTemplate.
run(exposure, sensorRef, templateIdList=templateIdList)
384 templateExposure = template.exposure
385 templateSources = template.sources
387 if self.config.doScaleTemplateVariance:
388 templateVarFactor = self.scaleVariance.
run(
389 templateExposure.getMaskedImage())
390 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
392 if self.config.subtract.name ==
'zogy':
393 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
395 spatiallyVarying=self.config.doSpatiallyVarying,
396 doPreConvolve=self.config.doPreConvolve)
397 subtractedExposure = subtractRes.subtractedExposure
399 elif self.config.subtract.name ==
'al':
401 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
404 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
411 if self.config.doPreConvolve:
412 convControl = afwMath.ConvolutionControl()
414 srcMI = exposure.getMaskedImage()
415 destMI = srcMI.Factory(srcMI.getDimensions())
417 if self.config.useGaussianForPreConvolution:
419 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
424 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
425 exposure.setMaskedImage(destMI)
426 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
428 scienceSigmaPost = scienceSigmaOrig
431 if self.config.doSelectSources:
432 if not sensorRef.datasetExists(
"src"):
433 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
435 selectSources = self.subtract.getSelectSources(
437 sigma=scienceSigmaPost,
438 doSmooth=
not self.doPreConvolve,
442 self.log.info(
"Source selection via src product")
444 selectSources = sensorRef.get(
"src")
447 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
448 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
449 targetFwhmPix=templateSigma*FwhmPerSigma))
451 if self.config.doAddMetrics:
453 kcQa = KernelCandidateQa(nparam)
454 selectSources = kcQa.addToSchema(selectSources)
456 if self.config.kernelSourcesFromRef:
458 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
459 matches = astromRet.matches
460 elif templateSources:
462 mc = afwTable.MatchControl()
463 mc.findOnlyClosest =
False 464 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
467 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False," 468 "but template sources not available. Cannot match science " 469 "sources with template sources. Run process* on data from " 470 "which templates are built.")
472 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
473 matches=matches).sourceCat
475 random.shuffle(kernelSources, random.random)
476 controlSources = kernelSources[::self.config.controlStepSize]
477 kernelSources = [k
for i, k
in enumerate(kernelSources)
478 if i % self.config.controlStepSize]
480 if self.config.doSelectDcrCatalog:
481 redSelector = DiaCatalogSourceSelectorTask(
482 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
484 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
485 controlSources.extend(redSources)
487 blueSelector = DiaCatalogSourceSelectorTask(
488 DiaCatalogSourceSelectorConfig(grMin=-99.999,
489 grMax=self.sourceSelector.config.grMin))
490 blueSources = blueSelector.selectStars(exposure, selectSources,
491 matches=matches).starCat
492 controlSources.extend(blueSources)
494 if self.config.doSelectVariableCatalog:
495 varSelector = DiaCatalogSourceSelectorTask(
496 DiaCatalogSourceSelectorConfig(includeVariable=
True))
497 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
498 controlSources.extend(varSources)
500 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)" 501 % (len(kernelSources), len(selectSources), len(controlSources)))
503 if self.config.doUseRegister:
504 self.log.info(
"Registering images")
506 if templateSources
is None:
509 templateSources = self.subtract.getSelectSources(
518 wcsResults = self.
fitAstrometry(templateSources, templateExposure, selectSources)
519 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
520 exposure.getWcs(), exposure.getBBox())
521 templateExposure = warpedExp
526 if self.config.doDebugRegister:
528 srcToMatch = {x.second.getId(): x.first
for x
in matches}
530 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
531 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
532 sids = [m.first.getId()
for m
in wcsResults.matches]
533 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
534 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
535 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
536 allresids = dict(zip(sids, zip(positions, residuals)))
538 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
539 wcsResults.wcs.pixelToSky(
540 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
541 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g")) +
542 2.5*numpy.log10(srcToMatch[x].get(
"r")) 543 for x
in sids
if x
in srcToMatch.keys()])
544 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
545 if s
in srcToMatch.keys()])
546 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
547 if s
in srcToMatch.keys()])
548 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
549 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
550 (colors <= self.sourceSelector.config.grMax))
551 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
552 rms1Long = IqrToSigma*(
553 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
554 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
555 numpy.percentile(dlat[idx1], 25))
556 rms2Long = IqrToSigma*(
557 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
558 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
559 numpy.percentile(dlat[idx2], 25))
560 rms3Long = IqrToSigma*(
561 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
562 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
563 numpy.percentile(dlat[idx3], 25))
564 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
565 (numpy.median(dlong[idx1]), rms1Long,
566 numpy.median(dlat[idx1]), rms1Lat))
567 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
568 (numpy.median(dlong[idx2]), rms2Long,
569 numpy.median(dlat[idx2]), rms2Lat))
570 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
571 (numpy.median(dlong[idx3]), rms3Long,
572 numpy.median(dlat[idx3]), rms3Lat))
574 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
575 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
576 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
577 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
578 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
579 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
581 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
582 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
583 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
584 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
585 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
586 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
593 self.log.info(
"Subtracting images")
594 subtractRes = self.subtract.subtractExposures(
595 templateExposure=templateExposure,
596 scienceExposure=exposure,
597 candidateList=kernelSources,
598 convolveTemplate=self.config.convolveTemplate,
599 doWarping=
not self.config.doUseRegister
601 subtractedExposure = subtractRes.subtractedExposure
603 if self.config.doWriteMatchedExp:
604 sensorRef.put(subtractRes.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
606 if self.config.doDetection:
607 self.log.info(
"Computing diffim PSF")
608 if subtractedExposure
is None:
609 subtractedExposure = sensorRef.get(subtractedExposureName)
612 if not subtractedExposure.hasPsf():
613 if self.config.convolveTemplate:
614 subtractedExposure.setPsf(exposure.getPsf())
616 if templateExposure
is None:
617 template = self.getTemplate.
run(exposure, sensorRef,
618 templateIdList=templateIdList)
619 subtractedExposure.setPsf(template.exposure.getPsf())
624 if self.config.doDecorrelation
and self.config.doSubtract:
626 if preConvPsf
is not None:
627 preConvKernel = preConvPsf.getLocalKernel()
628 if self.config.convolveTemplate:
629 self.log.info(
"Decorrelation after template image convolution")
630 decorrResult = self.decorrelate.
run(exposure, templateExposure,
632 subtractRes.psfMatchingKernel,
633 spatiallyVarying=self.config.doSpatiallyVarying,
634 preConvKernel=preConvKernel)
636 self.log.info(
"Decorrelation after science image convolution")
637 decorrResult = self.decorrelate.
run(templateExposure, exposure,
639 subtractRes.psfMatchingKernel,
640 spatiallyVarying=self.config.doSpatiallyVarying,
641 preConvKernel=preConvKernel)
642 subtractedExposure = decorrResult.correctedExposure
646 if self.config.doDetection:
647 self.log.info(
"Running diaSource detection")
649 mask = subtractedExposure.getMaskedImage().getMask()
650 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
652 table = afwTable.SourceTable.make(self.
schema, idFactory)
654 results = self.detection.makeSourceCatalog(
656 exposure=subtractedExposure,
657 doSmooth=
not self.config.doPreConvolve
660 if self.config.doMerge:
661 fpSet = results.fpSets.positive
662 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
663 self.config.growFootprint,
False)
664 diaSources = afwTable.SourceCatalog(table)
665 fpSet.makeSources(diaSources)
666 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
668 diaSources = results.sources
670 if self.config.doMeasurement:
671 newDipoleFitting = self.config.doDipoleFitting
672 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
673 if not newDipoleFitting:
675 self.measurement.
run(diaSources, subtractedExposure)
678 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
679 self.measurement.
run(diaSources, subtractedExposure, exposure,
680 subtractRes.matchedExposure)
682 self.measurement.
run(diaSources, subtractedExposure, exposure)
684 if self.config.doEvalLocCalibration
and self.config.doMeasurement:
685 self.evalLocCalib.
run(diaSources, subtractedExposure)
687 if self.config.doForcedMeasurement:
690 forcedSources = self.forcedMeasurement.generateMeasCat(
691 exposure, diaSources, subtractedExposure.getWcs())
692 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
693 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
694 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
695 "ip_diffim_forced_PsfFlux_instFlux",
True)
696 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
697 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
698 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
699 "ip_diffim_forced_PsfFlux_area",
True)
700 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
701 "ip_diffim_forced_PsfFlux_flag",
True)
702 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
703 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
704 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
705 "ip_diffim_forced_PsfFlux_flag_edge",
True)
706 for diaSource, forcedSource
in zip(diaSources, forcedSources):
707 diaSource.assign(forcedSource, mapper)
710 if self.config.doMatchSources:
711 if sensorRef.datasetExists(
"src"):
713 matchRadAsec = self.config.diaSourceMatchRadius
714 matchRadPixel = matchRadAsec/exposure.getWcs().pixelScale().asArcseconds()
716 srcMatches = afwTable.matchXy(sensorRef.get(
"src"), diaSources, matchRadPixel)
717 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for 718 srcMatch
in srcMatches])
719 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
722 self.log.warn(
"Src product does not exist; cannot match with diaSources")
726 refAstromConfig = AstrometryConfig()
727 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
728 refAstrometer = AstrometryTask(refAstromConfig)
729 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
730 refMatches = astromRet.matches
731 if refMatches
is None:
732 self.log.warn(
"No diaSource matches with reference catalog")
735 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
737 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for 738 refMatch
in refMatches])
741 for diaSource
in diaSources:
742 sid = diaSource.getId()
743 if sid
in srcMatchDict:
744 diaSource.set(
"srcMatchId", srcMatchDict[sid])
745 if sid
in refMatchDict:
746 diaSource.set(
"refMatchId", refMatchDict[sid])
748 if diaSources
is not None and self.config.doWriteSources:
749 sensorRef.put(diaSources, self.config.coaddName +
"Diff_diaSrc")
751 if self.config.doAddMetrics
and self.config.doSelectSources:
752 self.log.info(
"Evaluating metrics and control sample")
755 for cell
in subtractRes.kernelCellSet.getCellList():
756 for cand
in cell.begin(
False):
757 kernelCandList.append(cand)
760 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
763 diffimTools.sourceTableToCandidateList(controlSources,
764 subtractRes.warpedExposure, exposure,
765 self.config.subtract.kernel.active,
766 self.config.subtract.kernel.active.detectionConfig,
767 self.log, doBuild=
True, basisList=basisList))
769 kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
771 kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
773 if self.config.doDetection:
774 kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
776 kcQa.aggregate(selectSources, self.metadata, allresids)
778 sensorRef.put(selectSources, self.config.coaddName +
"Diff_kernelSrc")
780 if self.config.doWriteSubtractedExp:
781 sensorRef.put(subtractedExposure, subtractedExposureName)
783 self.
runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
784 return pipeBase.Struct(
785 subtractedExposure=subtractedExposure,
786 subtractRes=subtractRes,
791 """Fit the relative astrometry between templateSources and selectSources 793 @todo remove this method. It originally fit a new WCS to the template before calling register.run 794 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 795 It remains because a subtask overrides it. 797 results = self.register.
run(templateSources, templateExposure.getWcs(),
798 templateExposure.getBBox(), selectSources)
801 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
802 """@todo Test and update for current debug display and slot names 812 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
813 if not maskTransparency:
815 disp.setMaskTransparency(maskTransparency)
817 if display
and showSubtracted:
818 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
819 mi = subtractRes.subtractedExposure.getMaskedImage()
820 x0, y0 = mi.getX0(), mi.getY0()
821 with disp.Buffering():
823 x, y = s.getX() - x0, s.getY() - y0
824 ctype =
"red" if s.get(
"flags_negative")
else "yellow" 825 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
or 826 s.get(
"base_PixelFlags_flag_saturatedCenter")
or 827 s.get(
"base_PixelFlags_flag_crCenter")):
829 elif (s.get(
"base_PixelFlags_flag_interpolated")
or 830 s.get(
"base_PixelFlags_flag_saturated")
or 831 s.get(
"base_PixelFlags_flag_cr")):
835 disp.dot(ptype, x, y, size=4, ctype=ctype)
838 if display
and showPixelResiduals
and selectSources:
839 nonKernelSources = []
840 for source
in selectSources:
841 if source
not in kernelSources:
842 nonKernelSources.append(source)
844 diUtils.plotPixelResiduals(exposure,
845 subtractRes.warpedExposure,
846 subtractRes.subtractedExposure,
847 subtractRes.kernelCellSet,
848 subtractRes.psfMatchingKernel,
849 subtractRes.backgroundModel,
851 self.subtract.config.kernel.active.detectionConfig,
853 diUtils.plotPixelResiduals(exposure,
854 subtractRes.warpedExposure,
855 subtractRes.subtractedExposure,
856 subtractRes.kernelCellSet,
857 subtractRes.psfMatchingKernel,
858 subtractRes.backgroundModel,
860 self.subtract.config.kernel.active.detectionConfig,
862 if display
and showDiaSources:
863 flagChecker = SourceFlagChecker(diaSources)
864 isFlagged = [flagChecker(x)
for x
in diaSources]
865 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
866 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
867 frame=lsstDebug.frame)
870 if display
and showDipoles:
871 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
872 frame=lsstDebug.frame)
875 def _getConfigName(self):
876 """Return the name of the config dataset 878 return "%sDiff_config" % (self.config.coaddName,)
880 def _getMetadataName(self):
881 """Return the name of the metadata dataset 883 return "%sDiff_metadata" % (self.config.coaddName,)
886 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 887 diaSrc = afwTable.SourceCatalog(self.
schema)
889 return {self.config.coaddName +
"Diff_diaSrc": diaSrc}
892 def _makeArgumentParser(cls):
893 """Create an argument parser 896 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
897 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
898 help=
"Template data ID in case of calexp template," 899 " e.g. --templateId visit=6789")
904 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
905 doc=
"Shift stars going into RegisterTask by this amount")
906 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
907 doc=
"Perturb stars going into RegisterTask by this amount")
910 ImageDifferenceConfig.setDefaults(self)
915 """!Image difference Task used in the Winter 2013 data challege. 916 Enables testing the effects of registration shifts and scatter. 918 For use with winter 2013 simulated images: 919 Use --templateId visit=88868666 for sparse data 920 --templateId visit=22222200 for dense data (g) 921 --templateId visit=11111100 for dense data (i) 923 ConfigClass = Winter2013ImageDifferenceConfig
924 _DefaultName =
"winter2013ImageDifference" 927 ImageDifferenceTask.__init__(self, **kwargs)
930 """Fit the relative astrometry between templateSources and selectSources""" 931 if self.config.winter2013WcsShift > 0.0:
933 self.config.winter2013WcsShift)
934 cKey = templateSources[0].getTable().getCentroidKey()
935 for source
in templateSources:
936 centroid = source.get(cKey)
937 source.set(cKey, centroid + offset)
938 elif self.config.winter2013WcsRms > 0.0:
939 cKey = templateSources[0].getTable().getCentroidKey()
940 for source
in templateSources:
941 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
942 self.config.winter2013WcsRms*numpy.random.normal())
943 centroid = source.get(cKey)
944 source.set(cKey, centroid + offset)
946 results = self.register.
run(templateSources, templateExposure.getWcs(),
947 templateExposure.getBBox(), selectSources)
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def __init__(self, kwargs)
def getSchemaCatalogs(self)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def getTargetList(parsedCmd, kwargs)