26 import lsst.pex.config
as pexConfig
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
38 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
39 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
40 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
41 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
46 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
47 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
52 """Config for ImageDifferenceTask 54 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
55 doc=
"Add background to calexp before processing it. " 56 "Useful as ipDiffim does background matching.")
57 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
58 doc=
"Use image-to-image registration to align template with " 60 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
61 doc=
"Writing debugging data for doUseRegister")
62 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
63 doc=
"Select stars to use for kernel fitting")
64 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
65 doc=
"Select stars of extreme color as part of the control sample")
66 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
67 doc=
"Select stars that are variable to be part " 68 "of the control sample")
69 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
70 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
71 doc=
"Convolve science image by its PSF before PSF-matching?")
72 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
73 doc=
"Scale variance of the template before PSF matching")
74 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
75 doc=
"Use a simple gaussian PSF model for pre-convolution " 76 "(else use fit PSF)? Ignored if doPreConvolve false.")
77 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
78 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
79 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L " 80 "kernel convolution? If True, also update the diffim PSF.")
81 doMerge = pexConfig.Field(dtype=bool, default=
True,
82 doc=
"Merge positive and negative diaSources with grow radius " 83 "set by growFootprint")
84 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
85 doc=
"Match diaSources with input calexp sources and ref catalog sources")
86 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
87 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
88 doForcedMeasurement = pexConfig.Field(
91 doc=
"Force photometer diaSource locations on PVI?")
92 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
93 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
94 doc=
"Write warped and PSF-matched template coadd exposure?")
95 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
96 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
97 doc=
"Add columns to the source table to hold analysis metrics?")
99 coaddName = pexConfig.Field(
100 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
104 convolveTemplate = pexConfig.Field(
105 doc=
"Which image gets convolved (default = template)",
109 refObjLoader = pexConfig.ConfigurableField(
110 target=LoadIndexedReferenceObjectsTask,
111 doc=
"reference object loader",
113 astrometer = pexConfig.ConfigurableField(
114 target=AstrometryTask,
115 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
117 sourceSelector = pexConfig.ConfigurableField(
118 target=ObjectSizeStarSelectorTask,
119 doc=
"Source selection algorithm",
121 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
122 decorrelate = pexConfig.ConfigurableField(
123 target=DecorrelateALKernelSpatialTask,
124 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 125 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 128 doSpatiallyVarying = pexConfig.Field(
131 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the " 132 "image in order to allow for spatial variations" 134 detection = pexConfig.ConfigurableField(
135 target=SourceDetectionTask,
136 doc=
"Low-threshold detection for final measurement",
138 measurement = pexConfig.ConfigurableField(
139 target=DipoleFitTask,
140 doc=
"Enable updated dipole fitting method",
142 forcedMeasurement = pexConfig.ConfigurableField(
143 target=ForcedMeasurementTask,
144 doc=
"Subtask to force photometer PVI at diaSource location.",
146 getTemplate = pexConfig.ConfigurableField(
147 target=GetCoaddAsTemplateTask,
148 doc=
"Subtask to retrieve template exposure and sources",
150 scaleVariance = pexConfig.ConfigurableField(
151 target=ScaleVarianceTask,
152 doc=
"Subtask to rescale the variance of the template " 153 "to the statistically expected level" 155 controlStepSize = pexConfig.Field(
156 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
160 controlRandomSeed = pexConfig.Field(
161 doc=
"Random seed for shuffing the control sample",
165 register = pexConfig.ConfigurableField(
167 doc=
"Task to enable image-to-image image registration (warping)",
169 kernelSourcesFromRef = pexConfig.Field(
170 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
174 templateSipOrder = pexConfig.Field(
175 dtype=int, default=2,
176 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)" 178 growFootprint = pexConfig.Field(
179 dtype=int, default=2,
180 doc=
"Grow positive and negative footprints by this amount before merging" 182 diaSourceMatchRadius = pexConfig.Field(
183 dtype=float, default=0.5,
184 doc=
"Match radius (in arcseconds) for DiaSource to Source association" 190 self.
subtract[
'al'].kernel.name =
"AL" 191 self.
subtract[
'al'].kernel.active.fitForBackground =
True 192 self.
subtract[
'al'].kernel.active.spatialKernelOrder = 1
193 self.
subtract[
'al'].kernel.active.spatialBgOrder = 2
200 self.
detection.thresholdPolarity =
"both" 202 self.
detection.reEstimateBackground =
False 203 self.
detection.thresholdType =
"pixel_stdev" 209 self.
measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
210 self.
measurement.plugins.names |= [
'base_LocalPhotoCalib',
215 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
223 pexConfig.Config.validate(self)
225 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
227 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
229 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
231 raise ValueError(
"Cannot run source measurement without source detection.")
233 raise ValueError(
"Cannot run source merging without source detection.")
235 raise ValueError(
"doUseRegister=True and doSelectSources=False. " 236 "Cannot run RegisterTask without selecting sources.")
238 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and " 239 "convolveTemplate=False is not supported.")
242 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
249 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
254 """Subtract an image from a template and measure the result 256 ConfigClass = ImageDifferenceConfig
257 RunnerClass = ImageDifferenceTaskRunner
258 _DefaultName =
"imageDifference" 261 """!Construct an ImageDifference Task 263 @param[in] butler Butler object to use in constructing reference object loaders 265 pipeBase.CmdLineTask.__init__(self, **kwargs)
266 self.makeSubtask(
"getTemplate")
268 self.makeSubtask(
"subtract")
270 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
271 self.makeSubtask(
"decorrelate")
273 if self.config.doScaleTemplateVariance:
274 self.makeSubtask(
"scaleVariance")
276 if self.config.doUseRegister:
277 self.makeSubtask(
"register")
278 self.
schema = afwTable.SourceTable.makeMinimalSchema()
280 if self.config.doSelectSources:
281 self.makeSubtask(
"sourceSelector")
282 if self.config.kernelSourcesFromRef:
283 self.makeSubtask(
'refObjLoader', butler=butler)
284 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
287 if self.config.doDetection:
288 self.makeSubtask(
"detection", schema=self.
schema)
289 if self.config.doMeasurement:
290 self.makeSubtask(
"measurement", schema=self.
schema,
292 if self.config.doForcedMeasurement:
294 "ip_diffim_forced_PsfFlux_instFlux",
"D",
295 "Forced PSF flux measured on the direct image.")
297 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
298 "Forced PSF flux error measured on the direct image.")
300 "ip_diffim_forced_PsfFlux_area",
"F",
301 "Forced PSF flux effective area of PSF.",
304 "ip_diffim_forced_PsfFlux_flag",
"Flag",
305 "Forced PSF flux general failure flag.")
307 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
308 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
310 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
311 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
312 self.makeSubtask(
"forcedMeasurement", refSchema=self.
schema)
313 if self.config.doMatchSources:
314 self.
schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
315 self.
schema.addField(
"srcMatchId",
"L",
"unique id of source match")
319 """Create IdFactory instance for unique 64 bit diaSource id-s. 327 Number of used bits in ``expId``. 331 The diasource id-s consists of the ``expId`` stored fixed in the highest value 332 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 333 low value end of the integer. 337 idFactory: `lsst.afw.table.IdFactory` 339 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
343 """Subtract an image from a template coadd and measure the result. 345 Data I/O wrapper around `run` using the butler in Gen2. 349 sensorRef : `lsst.daf.persistence.ButlerDataRef` 350 Sensor-level butler data reference, used for the following data products: 357 - self.config.coaddName + "Coadd_skyMap" 358 - self.config.coaddName + "Coadd" 359 Input or output, depending on config: 360 - self.config.coaddName + "Diff_subtractedExp" 361 Output, depending on config: 362 - self.config.coaddName + "Diff_matchedExp" 363 - self.config.coaddName + "Diff_src" 367 results : `lsst.pipe.base.Struct` 368 Returns the Struct by `run`. 370 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp" 371 subtractedExposure =
None 373 calexpBackgroundExposure =
None 374 self.log.info(
"Processing %s" % (sensorRef.dataId))
379 idFactory = self.
makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
380 expBits=sensorRef.get(
"ccdExposureId_bits"))
381 if self.config.doAddCalexpBackground:
382 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
385 exposure = sensorRef.get(
"calexp", immediate=
True)
388 template = self.getTemplate.
run(exposure, sensorRef, templateIdList=templateIdList)
390 if sensorRef.datasetExists(
"src"):
391 self.log.info(
"Source selection via src product")
393 selectSources = sensorRef.get(
"src")
395 if not self.config.doSubtract
and self.config.doDetection:
397 subtractedExposure = sensorRef.get(subtractedExposureName)
400 results = self.
run(exposure=exposure,
401 selectSources=selectSources,
402 templateExposure=template.exposure,
403 templateSources=template.sources,
405 calexpBackgroundExposure=calexpBackgroundExposure,
406 subtractedExposure=subtractedExposure)
408 if self.config.doWriteSources
and results.diaSources
is not None:
409 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
410 if self.config.doWriteMatchedExp:
411 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
412 if self.config.doAddMetrics
and self.config.doSelectSources:
413 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
414 if self.config.doWriteSubtractedExp:
415 sensorRef.put(results.subtractedExposure, subtractedExposureName)
418 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
419 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
420 """PSF matches, subtract two images and perform detection on the difference image. 424 exposure : `lsst.afw.image.ExposureF`, optional 425 The science exposure, the minuend in the image subtraction. 426 Can be None only if ``config.doSubtract==False``. 427 selectSources : `lsst.afw.table.SourceCatalog`, optional 428 Identified sources on the science exposure. This catalog is used to 429 select sources in order to perform the AL PSF matching on stamp images 430 around them. The selection steps depend on config options and whether 431 ``templateSources`` and ``matchingSources`` specified. 432 templateExposure : `lsst.afw.image.ExposureF`, optional 433 The template to be subtracted from ``exposure`` in the image subtraction. 434 The template exposure should cover the same sky area as the science exposure. 435 It is either a stich of patches of a coadd skymap image or a calexp 436 of the same pointing as the science exposure. Can be None only 437 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 438 templateSources : `lsst.afw.table.SourceCatalog`, optional 439 Identified sources on the template exposure. 440 idFactory : `lsst.afw.table.IdFactory` 441 Generator object to assign ids to detected sources in the difference image. 442 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 443 Background exposure to be added back to the science exposure 444 if ``config.doAddCalexpBackground==True`` 445 subtractedExposure : `lsst.afw.image.ExposureF`, optional 446 If ``config.doSubtract==False`` and ``config.doDetection==True``, 447 performs the post subtraction source detection only on this exposure. 448 Otherwise should be None. 452 results : `lsst.pipe.base.Struct` 453 ``subtractedExposure`` : `lsst.afw.image.ExposureF` 455 ``matchedExposure`` : `lsst.afw.image.ExposureF` 456 The matched PSF exposure. 457 ``subtractRes`` : `lsst.pipe.base.Struct` 458 The returned result structure of the ImagePsfMatchTask subtask. 459 ``diaSources`` : `lsst.afw.table.SourceCatalog` 460 The catalog of detected sources. 461 ``selectSources`` : `lsst.afw.table.SourceCatalog` 462 The input source catalog with optionally added Qa information. 466 The following major steps are included: 468 - warp template coadd to match WCS of image 469 - PSF match image to warped template 470 - subtract image from PSF-matched, warped template 474 For details about the image subtraction configuration modes 475 see `lsst.ip.diffim`. 478 controlSources =
None 482 if self.config.doAddCalexpBackground:
483 mi = exposure.getMaskedImage()
484 mi += calexpBackgroundExposure.getImage()
486 if not exposure.hasPsf():
487 raise pipeBase.TaskError(
"Exposure has no psf")
488 sciencePsf = exposure.getPsf()
490 if self.config.doSubtract:
491 if self.config.doScaleTemplateVariance:
492 templateVarFactor = self.scaleVariance.
run(
493 templateExposure.getMaskedImage())
494 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
496 if self.config.subtract.name ==
'zogy':
497 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
499 spatiallyVarying=self.config.doSpatiallyVarying,
500 doPreConvolve=self.config.doPreConvolve)
501 subtractedExposure = subtractRes.subtractedExposure
503 elif self.config.subtract.name ==
'al':
505 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
506 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
514 if self.config.doPreConvolve:
515 convControl = afwMath.ConvolutionControl()
517 srcMI = exposure.getMaskedImage()
518 destMI = srcMI.Factory(srcMI.getDimensions())
520 if self.config.useGaussianForPreConvolution:
522 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
527 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
528 exposure.setMaskedImage(destMI)
529 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
531 scienceSigmaPost = scienceSigmaOrig
536 if self.config.doSelectSources:
537 if selectSources
is None:
538 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
540 selectSources = self.subtract.getSelectSources(
542 sigma=scienceSigmaPost,
543 doSmooth=
not self.doPreConvolve,
547 if self.config.doAddMetrics:
550 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
551 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
552 targetFwhmPix=templateSigma*FwhmPerSigma))
559 kcQa = KernelCandidateQa(nparam)
560 selectSources = kcQa.addToSchema(selectSources)
561 if self.config.kernelSourcesFromRef:
563 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
564 matches = astromRet.matches
565 elif templateSources:
567 mc = afwTable.MatchControl()
568 mc.findOnlyClosest =
False 569 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
572 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False," 573 "but template sources not available. Cannot match science " 574 "sources with template sources. Run process* on data from " 575 "which templates are built.")
577 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
578 matches=matches).sourceCat
579 random.shuffle(kernelSources, random.random)
580 controlSources = kernelSources[::self.config.controlStepSize]
581 kernelSources = [k
for i, k
in enumerate(kernelSources)
582 if i % self.config.controlStepSize]
584 if self.config.doSelectDcrCatalog:
585 redSelector = DiaCatalogSourceSelectorTask(
586 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
588 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
589 controlSources.extend(redSources)
591 blueSelector = DiaCatalogSourceSelectorTask(
592 DiaCatalogSourceSelectorConfig(grMin=-99.999,
593 grMax=self.sourceSelector.config.grMin))
594 blueSources = blueSelector.selectStars(exposure, selectSources,
595 matches=matches).starCat
596 controlSources.extend(blueSources)
598 if self.config.doSelectVariableCatalog:
599 varSelector = DiaCatalogSourceSelectorTask(
600 DiaCatalogSourceSelectorConfig(includeVariable=
True))
601 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
602 controlSources.extend(varSources)
604 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)" 605 % (len(kernelSources), len(selectSources), len(controlSources)))
609 if self.config.doUseRegister:
610 self.log.info(
"Registering images")
612 if templateSources
is None:
616 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
617 templateSources = self.subtract.getSelectSources(
626 wcsResults = self.
fitAstrometry(templateSources, templateExposure, selectSources)
627 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
628 exposure.getWcs(), exposure.getBBox())
629 templateExposure = warpedExp
634 if self.config.doDebugRegister:
636 srcToMatch = {x.second.getId(): x.first
for x
in matches}
638 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
639 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
640 sids = [m.first.getId()
for m
in wcsResults.matches]
641 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
642 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
643 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
644 allresids = dict(zip(sids, zip(positions, residuals)))
646 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
647 wcsResults.wcs.pixelToSky(
648 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
649 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g")) +
650 2.5*numpy.log10(srcToMatch[x].get(
"r")) 651 for x
in sids
if x
in srcToMatch.keys()])
652 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
653 if s
in srcToMatch.keys()])
654 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
655 if s
in srcToMatch.keys()])
656 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
657 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
658 (colors <= self.sourceSelector.config.grMax))
659 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
660 rms1Long = IqrToSigma*(
661 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
662 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
663 numpy.percentile(dlat[idx1], 25))
664 rms2Long = IqrToSigma*(
665 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
666 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
667 numpy.percentile(dlat[idx2], 25))
668 rms3Long = IqrToSigma*(
669 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
670 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
671 numpy.percentile(dlat[idx3], 25))
672 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
673 (numpy.median(dlong[idx1]), rms1Long,
674 numpy.median(dlat[idx1]), rms1Lat))
675 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
676 (numpy.median(dlong[idx2]), rms2Long,
677 numpy.median(dlat[idx2]), rms2Lat))
678 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
679 (numpy.median(dlong[idx3]), rms3Long,
680 numpy.median(dlat[idx3]), rms3Lat))
682 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
683 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
684 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
685 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
686 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
687 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
689 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
690 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
691 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
692 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
693 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
694 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
701 self.log.info(
"Subtracting images")
702 subtractRes = self.subtract.subtractExposures(
703 templateExposure=templateExposure,
704 scienceExposure=exposure,
705 candidateList=kernelSources,
706 convolveTemplate=self.config.convolveTemplate,
707 doWarping=
not self.config.doUseRegister
709 subtractedExposure = subtractRes.subtractedExposure
711 if self.config.doDetection:
712 self.log.info(
"Computing diffim PSF")
715 if not subtractedExposure.hasPsf():
716 if self.config.convolveTemplate:
717 subtractedExposure.setPsf(exposure.getPsf())
719 subtractedExposure.setPsf(templateExposure.getPsf())
726 if self.config.doDecorrelation
and self.config.doSubtract:
728 if preConvPsf
is not None:
729 preConvKernel = preConvPsf.getLocalKernel()
730 if self.config.convolveTemplate:
731 self.log.info(
"Decorrelation after template image convolution")
732 decorrResult = self.decorrelate.
run(exposure, templateExposure,
734 subtractRes.psfMatchingKernel,
735 spatiallyVarying=self.config.doSpatiallyVarying,
736 preConvKernel=preConvKernel)
738 self.log.info(
"Decorrelation after science image convolution")
739 decorrResult = self.decorrelate.
run(templateExposure, exposure,
741 subtractRes.psfMatchingKernel,
742 spatiallyVarying=self.config.doSpatiallyVarying,
743 preConvKernel=preConvKernel)
744 subtractedExposure = decorrResult.correctedExposure
748 if self.config.doDetection:
749 self.log.info(
"Running diaSource detection")
751 mask = subtractedExposure.getMaskedImage().getMask()
752 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
754 table = afwTable.SourceTable.make(self.
schema, idFactory)
756 results = self.detection.
run(
758 exposure=subtractedExposure,
759 doSmooth=
not self.config.doPreConvolve
762 if self.config.doMerge:
763 fpSet = results.fpSets.positive
764 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
765 self.config.growFootprint,
False)
766 diaSources = afwTable.SourceCatalog(table)
767 fpSet.makeSources(diaSources)
768 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
770 diaSources = results.sources
772 if self.config.doMeasurement:
773 newDipoleFitting = self.config.doDipoleFitting
774 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
775 if not newDipoleFitting:
777 self.measurement.
run(diaSources, subtractedExposure)
780 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
781 self.measurement.
run(diaSources, subtractedExposure, exposure,
782 subtractRes.matchedExposure)
784 self.measurement.
run(diaSources, subtractedExposure, exposure)
786 if self.config.doForcedMeasurement:
789 forcedSources = self.forcedMeasurement.generateMeasCat(
790 exposure, diaSources, subtractedExposure.getWcs())
791 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
792 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
793 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
794 "ip_diffim_forced_PsfFlux_instFlux",
True)
795 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
796 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
797 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
798 "ip_diffim_forced_PsfFlux_area",
True)
799 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
800 "ip_diffim_forced_PsfFlux_flag",
True)
801 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
802 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
803 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
804 "ip_diffim_forced_PsfFlux_flag_edge",
True)
805 for diaSource, forcedSource
in zip(diaSources, forcedSources):
806 diaSource.assign(forcedSource, mapper)
809 if self.config.doMatchSources:
810 if selectSources
is not None:
812 matchRadAsec = self.config.diaSourceMatchRadius
813 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
815 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
816 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for 817 srcMatch
in srcMatches])
818 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
821 self.log.warn(
"Src product does not exist; cannot match with diaSources")
825 refAstromConfig = AstrometryConfig()
826 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
827 refAstrometer = AstrometryTask(refAstromConfig)
828 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
829 refMatches = astromRet.matches
830 if refMatches
is None:
831 self.log.warn(
"No diaSource matches with reference catalog")
834 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
836 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for 837 refMatch
in refMatches])
840 for diaSource
in diaSources:
841 sid = diaSource.getId()
842 if sid
in srcMatchDict:
843 diaSource.set(
"srcMatchId", srcMatchDict[sid])
844 if sid
in refMatchDict:
845 diaSource.set(
"refMatchId", refMatchDict[sid])
847 if self.config.doAddMetrics
and self.config.doSelectSources:
848 self.log.info(
"Evaluating metrics and control sample")
851 for cell
in subtractRes.kernelCellSet.getCellList():
852 for cand
in cell.begin(
False):
853 kernelCandList.append(cand)
856 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
857 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
860 diffimTools.sourceTableToCandidateList(controlSources,
861 subtractRes.warpedExposure, exposure,
862 self.config.subtract.kernel.active,
863 self.config.subtract.kernel.active.detectionConfig,
864 self.log, doBuild=
True, basisList=basisList))
866 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
867 subtractRes.backgroundModel, dof=nparam)
868 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
869 subtractRes.backgroundModel)
871 if self.config.doDetection:
872 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
874 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
876 self.
runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
877 return pipeBase.Struct(
878 subtractedExposure=subtractedExposure,
879 matchedExposure=subtractRes.matchedExposure,
880 subtractRes=subtractRes,
881 diaSources=diaSources,
882 selectSources=selectSources
886 """Fit the relative astrometry between templateSources and selectSources 891 Remove this method. It originally fit a new WCS to the template before calling register.run 892 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 893 It remains because a subtask overrides it. 895 results = self.register.
run(templateSources, templateExposure.getWcs(),
896 templateExposure.getBBox(), selectSources)
899 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
900 """Make debug plots and displays. 904 Test and update for current debug display and slot names 914 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
915 if not maskTransparency:
917 disp.setMaskTransparency(maskTransparency)
919 if display
and showSubtracted:
920 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
921 mi = subtractRes.subtractedExposure.getMaskedImage()
922 x0, y0 = mi.getX0(), mi.getY0()
923 with disp.Buffering():
925 x, y = s.getX() - x0, s.getY() - y0
926 ctype =
"red" if s.get(
"flags_negative")
else "yellow" 927 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
or 928 s.get(
"base_PixelFlags_flag_saturatedCenter")
or 929 s.get(
"base_PixelFlags_flag_crCenter")):
931 elif (s.get(
"base_PixelFlags_flag_interpolated")
or 932 s.get(
"base_PixelFlags_flag_saturated")
or 933 s.get(
"base_PixelFlags_flag_cr")):
937 disp.dot(ptype, x, y, size=4, ctype=ctype)
940 if display
and showPixelResiduals
and selectSources:
941 nonKernelSources = []
942 for source
in selectSources:
943 if source
not in kernelSources:
944 nonKernelSources.append(source)
946 diUtils.plotPixelResiduals(exposure,
947 subtractRes.warpedExposure,
948 subtractRes.subtractedExposure,
949 subtractRes.kernelCellSet,
950 subtractRes.psfMatchingKernel,
951 subtractRes.backgroundModel,
953 self.subtract.config.kernel.active.detectionConfig,
955 diUtils.plotPixelResiduals(exposure,
956 subtractRes.warpedExposure,
957 subtractRes.subtractedExposure,
958 subtractRes.kernelCellSet,
959 subtractRes.psfMatchingKernel,
960 subtractRes.backgroundModel,
962 self.subtract.config.kernel.active.detectionConfig,
964 if display
and showDiaSources:
965 flagChecker = SourceFlagChecker(diaSources)
966 isFlagged = [flagChecker(x)
for x
in diaSources]
967 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
968 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
969 frame=lsstDebug.frame)
972 if display
and showDipoles:
973 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
974 frame=lsstDebug.frame)
977 def _getConfigName(self):
978 """Return the name of the config dataset 980 return "%sDiff_config" % (self.config.coaddName,)
982 def _getMetadataName(self):
983 """Return the name of the metadata dataset 985 return "%sDiff_metadata" % (self.config.coaddName,)
988 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 989 diaSrc = afwTable.SourceCatalog(self.
schema)
991 return {self.config.coaddName +
"Diff_diaSrc": diaSrc}
994 def _makeArgumentParser(cls):
995 """Create an argument parser 998 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
999 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1000 help=
"Template data ID in case of calexp template," 1001 " e.g. --templateId visit=6789")
1006 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1007 doc=
"Shift stars going into RegisterTask by this amount")
1008 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1009 doc=
"Perturb stars going into RegisterTask by this amount")
1012 ImageDifferenceConfig.setDefaults(self)
1013 self.
getTemplate.retarget(GetCalexpAsTemplateTask)
1017 """!Image difference Task used in the Winter 2013 data challege. 1018 Enables testing the effects of registration shifts and scatter. 1020 For use with winter 2013 simulated images: 1021 Use --templateId visit=88868666 for sparse data 1022 --templateId visit=22222200 for dense data (g) 1023 --templateId visit=11111100 for dense data (i) 1025 ConfigClass = Winter2013ImageDifferenceConfig
1026 _DefaultName =
"winter2013ImageDifference" 1029 ImageDifferenceTask.__init__(self, **kwargs)
1032 """Fit the relative astrometry between templateSources and selectSources""" 1033 if self.config.winter2013WcsShift > 0.0:
1035 self.config.winter2013WcsShift)
1036 cKey = templateSources[0].getTable().getCentroidKey()
1037 for source
in templateSources:
1038 centroid = source.get(cKey)
1039 source.set(cKey, centroid + offset)
1040 elif self.config.winter2013WcsRms > 0.0:
1041 cKey = templateSources[0].getTable().getCentroidKey()
1042 for source
in templateSources:
1043 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1044 self.config.winter2013WcsRms*numpy.random.normal())
1045 centroid = source.get(cKey)
1046 source.set(cKey, centroid + offset)
1048 results = self.register.
run(templateSources, templateExposure.getWcs(),
1049 templateExposure.getBBox(), selectSources)
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def __init__(self, kwargs)
def getSchemaCatalogs(self)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def makeIdFactory(expId, expBits)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def getTargetList(parsedCmd, kwargs)
def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None, idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None)