27 import lsst.pex.config
as pexConfig
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
53 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
54 defaultTemplates={
"coaddName":
"deep",
58 exposure = pipeBase.connectionTypes.Input(
59 doc=
"Input science exposure to subtract from.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"ExposureF",
73 skyMap = pipeBase.connectionTypes.Input(
74 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
75 name=
"{coaddName}Coadd_skyMap",
76 dimensions=(
"skymap", ),
77 storageClass=
"SkyMap",
79 coaddExposures = pipeBase.connectionTypes.Input(
80 doc=
"Input template to match and subtract from the exposure",
81 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
82 storageClass=
"ExposureF",
83 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87 subtractedExposure = pipeBase.connectionTypes.Output(
88 doc=
"Output difference image",
89 dimensions=(
"instrument",
"visit",
"detector"),
90 storageClass=
"ExposureF",
91 name=
"{fakesType}{coaddName}Diff_differenceExp",
93 diaSources = pipeBase.connectionTypes.Output(
94 doc=
"Output detected diaSources on the difference image",
95 dimensions=(
"instrument",
"visit",
"detector"),
96 storageClass=
"SourceCatalog",
97 name=
"{fakesType}{coaddName}Diff_diaSrc",
104 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=ImageDifferenceTaskConnections):
106 """Config for ImageDifferenceTask 108 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
109 doc=
"Add background to calexp before processing it. " 110 "Useful as ipDiffim does background matching.")
111 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
112 doc=
"Use image-to-image registration to align template with " 114 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
115 doc=
"Writing debugging data for doUseRegister")
116 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
117 doc=
"Select stars to use for kernel fitting")
118 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
119 doc=
"Select stars of extreme color as part of the control sample")
120 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
121 doc=
"Select stars that are variable to be part " 122 "of the control sample")
123 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
124 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
125 doc=
"Convolve science image by its PSF before PSF-matching?")
126 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
127 doc=
"Scale variance of the template before PSF matching")
128 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
129 doc=
"Use a simple gaussian PSF model for pre-convolution " 130 "(else use fit PSF)? Ignored if doPreConvolve false.")
131 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
132 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
133 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L " 134 "kernel convolution? If True, also update the diffim PSF.")
135 doMerge = pexConfig.Field(dtype=bool, default=
True,
136 doc=
"Merge positive and negative diaSources with grow radius " 137 "set by growFootprint")
138 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
139 doc=
"Match diaSources with input calexp sources and ref catalog sources")
140 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
141 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
142 doForcedMeasurement = pexConfig.Field(
145 doc=
"Force photometer diaSource locations on PVI?")
146 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
147 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Write warped and PSF-matched template coadd exposure?")
149 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
150 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
151 doc=
"Add columns to the source table to hold analysis metrics?")
153 coaddName = pexConfig.Field(
154 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
158 convolveTemplate = pexConfig.Field(
159 doc=
"Which image gets convolved (default = template)",
163 refObjLoader = pexConfig.ConfigurableField(
164 target=LoadIndexedReferenceObjectsTask,
165 doc=
"reference object loader",
167 astrometer = pexConfig.ConfigurableField(
168 target=AstrometryTask,
169 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
171 sourceSelector = pexConfig.ConfigurableField(
172 target=ObjectSizeStarSelectorTask,
173 doc=
"Source selection algorithm",
175 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
176 decorrelate = pexConfig.ConfigurableField(
177 target=DecorrelateALKernelSpatialTask,
178 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 179 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 182 doSpatiallyVarying = pexConfig.Field(
185 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the " 186 "image in order to allow for spatial variations" 188 detection = pexConfig.ConfigurableField(
189 target=SourceDetectionTask,
190 doc=
"Low-threshold detection for final measurement",
192 measurement = pexConfig.ConfigurableField(
193 target=DipoleFitTask,
194 doc=
"Enable updated dipole fitting method",
196 forcedMeasurement = pexConfig.ConfigurableField(
197 target=ForcedMeasurementTask,
198 doc=
"Subtask to force photometer PVI at diaSource location.",
200 getTemplate = pexConfig.ConfigurableField(
201 target=GetCoaddAsTemplateTask,
202 doc=
"Subtask to retrieve template exposure and sources",
204 scaleVariance = pexConfig.ConfigurableField(
205 target=ScaleVarianceTask,
206 doc=
"Subtask to rescale the variance of the template " 207 "to the statistically expected level" 209 controlStepSize = pexConfig.Field(
210 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
214 controlRandomSeed = pexConfig.Field(
215 doc=
"Random seed for shuffing the control sample",
219 register = pexConfig.ConfigurableField(
221 doc=
"Task to enable image-to-image image registration (warping)",
223 kernelSourcesFromRef = pexConfig.Field(
224 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
228 templateSipOrder = pexConfig.Field(
229 dtype=int, default=2,
230 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)" 232 growFootprint = pexConfig.Field(
233 dtype=int, default=2,
234 doc=
"Grow positive and negative footprints by this amount before merging" 236 diaSourceMatchRadius = pexConfig.Field(
237 dtype=float, default=0.5,
238 doc=
"Match radius (in arcseconds) for DiaSource to Source association" 241 def setDefaults(self):
244 self.subtract[
'al'].kernel.name =
"AL" 245 self.subtract[
'al'].kernel.active.fitForBackground =
True 246 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
247 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
248 self.doPreConvolve =
False 249 self.doMatchSources =
False 250 self.doAddMetrics =
False 251 self.doUseRegister =
False 254 self.detection.thresholdPolarity =
"both" 255 self.detection.thresholdValue = 5.5
256 self.detection.reEstimateBackground =
False 257 self.detection.thresholdType =
"pixel_stdev" 263 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
264 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
267 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
268 self.forcedMeasurement.copyColumns = {
269 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
270 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid" 271 self.forcedMeasurement.slots.shape =
None 274 random.seed(self.controlRandomSeed)
277 pexConfig.Config.validate(self)
278 if self.doAddMetrics
and not self.doSubtract:
279 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
280 if not self.doSubtract
and not self.doDetection:
281 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
282 if self.subtract.name ==
'zogy' and self.doAddMetrics:
283 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
284 if self.doMeasurement
and not self.doDetection:
285 raise ValueError(
"Cannot run source measurement without source detection.")
286 if self.doMerge
and not self.doDetection:
287 raise ValueError(
"Cannot run source merging without source detection.")
288 if self.doUseRegister
and not self.doSelectSources:
289 raise ValueError(
"doUseRegister=True and doSelectSources=False. " 290 "Cannot run RegisterTask without selecting sources.")
291 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
292 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and " 293 "convolveTemplate=False is not supported.")
294 if hasattr(self.getTemplate,
"coaddName"):
295 if self.getTemplate.coaddName != self.coaddName:
296 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
299 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
302 def getTargetList(parsedCmd, **kwargs):
303 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
307 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
308 """Subtract an image from a template and measure the result 310 ConfigClass = ImageDifferenceConfig
311 RunnerClass = ImageDifferenceTaskRunner
312 _DefaultName =
"imageDifference" 314 def __init__(self, butler=None, **kwargs):
315 """!Construct an ImageDifference Task 317 @param[in] butler Butler object to use in constructing reference object loaders 319 super().__init__(**kwargs)
320 self.makeSubtask(
"getTemplate")
322 self.makeSubtask(
"subtract")
324 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
325 self.makeSubtask(
"decorrelate")
327 if self.config.doScaleTemplateVariance:
328 self.makeSubtask(
"scaleVariance")
330 if self.config.doUseRegister:
331 self.makeSubtask(
"register")
332 self.schema = afwTable.SourceTable.makeMinimalSchema()
334 if self.config.doSelectSources:
335 self.makeSubtask(
"sourceSelector")
336 if self.config.kernelSourcesFromRef:
337 self.makeSubtask(
'refObjLoader', butler=butler)
338 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
340 self.algMetadata = dafBase.PropertyList()
341 if self.config.doDetection:
342 self.makeSubtask(
"detection", schema=self.schema)
343 if self.config.doMeasurement:
344 self.makeSubtask(
"measurement", schema=self.schema,
345 algMetadata=self.algMetadata)
346 if self.config.doForcedMeasurement:
347 self.schema.addField(
348 "ip_diffim_forced_PsfFlux_instFlux",
"D",
349 "Forced PSF flux measured on the direct image.")
350 self.schema.addField(
351 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
352 "Forced PSF flux error measured on the direct image.")
353 self.schema.addField(
354 "ip_diffim_forced_PsfFlux_area",
"F",
355 "Forced PSF flux effective area of PSF.",
357 self.schema.addField(
358 "ip_diffim_forced_PsfFlux_flag",
"Flag",
359 "Forced PSF flux general failure flag.")
360 self.schema.addField(
361 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
362 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
363 self.schema.addField(
364 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
365 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
366 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
367 if self.config.doMatchSources:
368 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
369 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
372 def makeIdFactory(expId, expBits):
373 """Create IdFactory instance for unique 64 bit diaSource id-s. 381 Number of used bits in ``expId``. 385 The diasource id-s consists of the ``expId`` stored fixed in the highest value 386 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 387 low value end of the integer. 391 idFactory: `lsst.afw.table.IdFactory` 393 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
396 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
397 inputRefs: pipeBase.InputQuantizedConnection,
398 outputRefs: pipeBase.OutputQuantizedConnection):
399 if self.getTemplate.config.coaddName ==
'dcr':
401 raise NotImplementedError(
"TODO DM-22952: Dcr coadd templates are not yet supported in gen3.")
403 inputs = butlerQC.get(inputRefs)
404 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
405 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
407 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
408 templateStruct = self.getTemplate.runQuantum(
409 inputs[
'exposure'], butlerQC, inputRefs.skyMap, inputRefs.coaddExposures
412 outputs = self.run(exposure=inputs[
'exposure'],
413 templateExposure=templateStruct.exposure,
415 butlerQC.put(outputs, outputRefs)
418 def runDataRef(self, sensorRef, templateIdList=None):
419 """Subtract an image from a template coadd and measure the result. 421 Data I/O wrapper around `run` using the butler in Gen2. 425 sensorRef : `lsst.daf.persistence.ButlerDataRef` 426 Sensor-level butler data reference, used for the following data products: 433 - self.config.coaddName + "Coadd_skyMap" 434 - self.config.coaddName + "Coadd" 435 Input or output, depending on config: 436 - self.config.coaddName + "Diff_subtractedExp" 437 Output, depending on config: 438 - self.config.coaddName + "Diff_matchedExp" 439 - self.config.coaddName + "Diff_src" 443 results : `lsst.pipe.base.Struct` 444 Returns the Struct by `run`. 446 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp" 447 subtractedExposure =
None 449 calexpBackgroundExposure =
None 450 self.log.info(
"Processing %s" % (sensorRef.dataId))
455 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
456 expBits=sensorRef.get(
"ccdExposureId_bits"))
457 if self.config.doAddCalexpBackground:
458 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
461 exposure = sensorRef.get(
"calexp", immediate=
True)
464 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
466 if sensorRef.datasetExists(
"src"):
467 self.log.info(
"Source selection via src product")
469 selectSources = sensorRef.get(
"src")
471 if not self.config.doSubtract
and self.config.doDetection:
473 subtractedExposure = sensorRef.get(subtractedExposureName)
476 results = self.run(exposure=exposure,
477 selectSources=selectSources,
478 templateExposure=template.exposure,
479 templateSources=template.sources,
481 calexpBackgroundExposure=calexpBackgroundExposure,
482 subtractedExposure=subtractedExposure)
484 if self.config.doWriteSources
and results.diaSources
is not None:
485 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
486 if self.config.doWriteMatchedExp:
487 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
488 if self.config.doAddMetrics
and self.config.doSelectSources:
489 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
490 if self.config.doWriteSubtractedExp:
491 sensorRef.put(results.subtractedExposure, subtractedExposureName)
494 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
495 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
496 """PSF matches, subtract two images and perform detection on the difference image. 500 exposure : `lsst.afw.image.ExposureF`, optional 501 The science exposure, the minuend in the image subtraction. 502 Can be None only if ``config.doSubtract==False``. 503 selectSources : `lsst.afw.table.SourceCatalog`, optional 504 Identified sources on the science exposure. This catalog is used to 505 select sources in order to perform the AL PSF matching on stamp images 506 around them. The selection steps depend on config options and whether 507 ``templateSources`` and ``matchingSources`` specified. 508 templateExposure : `lsst.afw.image.ExposureF`, optional 509 The template to be subtracted from ``exposure`` in the image subtraction. 510 The template exposure should cover the same sky area as the science exposure. 511 It is either a stich of patches of a coadd skymap image or a calexp 512 of the same pointing as the science exposure. Can be None only 513 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 514 templateSources : `lsst.afw.table.SourceCatalog`, optional 515 Identified sources on the template exposure. 516 idFactory : `lsst.afw.table.IdFactory` 517 Generator object to assign ids to detected sources in the difference image. 518 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 519 Background exposure to be added back to the science exposure 520 if ``config.doAddCalexpBackground==True`` 521 subtractedExposure : `lsst.afw.image.ExposureF`, optional 522 If ``config.doSubtract==False`` and ``config.doDetection==True``, 523 performs the post subtraction source detection only on this exposure. 524 Otherwise should be None. 528 results : `lsst.pipe.base.Struct` 529 ``subtractedExposure`` : `lsst.afw.image.ExposureF` 531 ``matchedExposure`` : `lsst.afw.image.ExposureF` 532 The matched PSF exposure. 533 ``subtractRes`` : `lsst.pipe.base.Struct` 534 The returned result structure of the ImagePsfMatchTask subtask. 535 ``diaSources`` : `lsst.afw.table.SourceCatalog` 536 The catalog of detected sources. 537 ``selectSources`` : `lsst.afw.table.SourceCatalog` 538 The input source catalog with optionally added Qa information. 542 The following major steps are included: 544 - warp template coadd to match WCS of image 545 - PSF match image to warped template 546 - subtract image from PSF-matched, warped template 550 For details about the image subtraction configuration modes 551 see `lsst.ip.diffim`. 554 controlSources =
None 558 if self.config.doAddCalexpBackground:
559 mi = exposure.getMaskedImage()
560 mi += calexpBackgroundExposure.getImage()
562 if not exposure.hasPsf():
563 raise pipeBase.TaskError(
"Exposure has no psf")
564 sciencePsf = exposure.getPsf()
566 if self.config.doSubtract:
567 if self.config.doScaleTemplateVariance:
568 templateVarFactor = self.scaleVariance.
run(
569 templateExposure.getMaskedImage())
570 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
572 if self.config.subtract.name ==
'zogy':
573 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
575 spatiallyVarying=self.config.doSpatiallyVarying,
576 doPreConvolve=self.config.doPreConvolve)
577 subtractedExposure = subtractRes.subtractedExposure
579 elif self.config.subtract.name ==
'al':
581 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
582 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
590 if self.config.doPreConvolve:
591 convControl = afwMath.ConvolutionControl()
593 srcMI = exposure.getMaskedImage()
594 destMI = srcMI.Factory(srcMI.getDimensions())
596 if self.config.useGaussianForPreConvolution:
598 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
603 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
604 exposure.setMaskedImage(destMI)
605 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
607 scienceSigmaPost = scienceSigmaOrig
612 if self.config.doSelectSources:
613 if selectSources
is None:
614 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
616 selectSources = self.subtract.getSelectSources(
618 sigma=scienceSigmaPost,
619 doSmooth=
not self.config.doPreConvolve,
623 if self.config.doAddMetrics:
626 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
627 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
628 targetFwhmPix=templateSigma*FwhmPerSigma))
635 kcQa = KernelCandidateQa(nparam)
636 selectSources = kcQa.addToSchema(selectSources)
637 if self.config.kernelSourcesFromRef:
639 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
640 matches = astromRet.matches
641 elif templateSources:
643 mc = afwTable.MatchControl()
644 mc.findOnlyClosest =
False 645 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
648 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False," 649 "but template sources not available. Cannot match science " 650 "sources with template sources. Run process* on data from " 651 "which templates are built.")
653 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
654 matches=matches).sourceCat
655 random.shuffle(kernelSources, random.random)
656 controlSources = kernelSources[::self.config.controlStepSize]
657 kernelSources = [k
for i, k
in enumerate(kernelSources)
658 if i % self.config.controlStepSize]
660 if self.config.doSelectDcrCatalog:
661 redSelector = DiaCatalogSourceSelectorTask(
662 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
664 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
665 controlSources.extend(redSources)
667 blueSelector = DiaCatalogSourceSelectorTask(
668 DiaCatalogSourceSelectorConfig(grMin=-99.999,
669 grMax=self.sourceSelector.config.grMin))
670 blueSources = blueSelector.selectStars(exposure, selectSources,
671 matches=matches).starCat
672 controlSources.extend(blueSources)
674 if self.config.doSelectVariableCatalog:
675 varSelector = DiaCatalogSourceSelectorTask(
676 DiaCatalogSourceSelectorConfig(includeVariable=
True))
677 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
678 controlSources.extend(varSources)
680 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)" 681 % (len(kernelSources), len(selectSources), len(controlSources)))
685 if self.config.doUseRegister:
686 self.log.info(
"Registering images")
688 if templateSources
is None:
692 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
693 templateSources = self.subtract.getSelectSources(
702 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
703 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
704 exposure.getWcs(), exposure.getBBox())
705 templateExposure = warpedExp
710 if self.config.doDebugRegister:
712 srcToMatch = {x.second.getId(): x.first
for x
in matches}
714 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
715 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
716 sids = [m.first.getId()
for m
in wcsResults.matches]
717 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
718 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
719 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
720 allresids = dict(zip(sids, zip(positions, residuals)))
722 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
723 wcsResults.wcs.pixelToSky(
724 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
725 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g")) +
726 2.5*numpy.log10(srcToMatch[x].get(
"r")) 727 for x
in sids
if x
in srcToMatch.keys()])
728 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
729 if s
in srcToMatch.keys()])
730 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
731 if s
in srcToMatch.keys()])
732 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
733 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
734 (colors <= self.sourceSelector.config.grMax))
735 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
736 rms1Long = IqrToSigma*(
737 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
738 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
739 numpy.percentile(dlat[idx1], 25))
740 rms2Long = IqrToSigma*(
741 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
742 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
743 numpy.percentile(dlat[idx2], 25))
744 rms3Long = IqrToSigma*(
745 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
746 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
747 numpy.percentile(dlat[idx3], 25))
748 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
749 (numpy.median(dlong[idx1]), rms1Long,
750 numpy.median(dlat[idx1]), rms1Lat))
751 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
752 (numpy.median(dlong[idx2]), rms2Long,
753 numpy.median(dlat[idx2]), rms2Lat))
754 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
755 (numpy.median(dlong[idx3]), rms3Long,
756 numpy.median(dlat[idx3]), rms3Lat))
758 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
759 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
760 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
761 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
762 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
763 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
765 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
766 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
767 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
768 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
769 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
770 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
777 self.log.info(
"Subtracting images")
778 subtractRes = self.subtract.subtractExposures(
779 templateExposure=templateExposure,
780 scienceExposure=exposure,
781 candidateList=kernelSources,
782 convolveTemplate=self.config.convolveTemplate,
783 doWarping=
not self.config.doUseRegister
785 subtractedExposure = subtractRes.subtractedExposure
787 if self.config.doDetection:
788 self.log.info(
"Computing diffim PSF")
791 if not subtractedExposure.hasPsf():
792 if self.config.convolveTemplate:
793 subtractedExposure.setPsf(exposure.getPsf())
795 subtractedExposure.setPsf(templateExposure.getPsf())
802 if self.config.doDecorrelation
and self.config.doSubtract:
804 if preConvPsf
is not None:
805 preConvKernel = preConvPsf.getLocalKernel()
806 if self.config.convolveTemplate:
807 self.log.info(
"Decorrelation after template image convolution")
808 decorrResult = self.decorrelate.
run(exposure, templateExposure,
810 subtractRes.psfMatchingKernel,
811 spatiallyVarying=self.config.doSpatiallyVarying,
812 preConvKernel=preConvKernel)
814 self.log.info(
"Decorrelation after science image convolution")
815 decorrResult = self.decorrelate.
run(templateExposure, exposure,
817 subtractRes.psfMatchingKernel,
818 spatiallyVarying=self.config.doSpatiallyVarying,
819 preConvKernel=preConvKernel)
820 subtractedExposure = decorrResult.correctedExposure
824 if self.config.doDetection:
825 self.log.info(
"Running diaSource detection")
827 mask = subtractedExposure.getMaskedImage().getMask()
828 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
830 table = afwTable.SourceTable.make(self.schema, idFactory)
831 table.setMetadata(self.algMetadata)
832 results = self.detection.
run(
834 exposure=subtractedExposure,
835 doSmooth=
not self.config.doPreConvolve
838 if self.config.doMerge:
839 fpSet = results.fpSets.positive
840 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
841 self.config.growFootprint,
False)
842 diaSources = afwTable.SourceCatalog(table)
843 fpSet.makeSources(diaSources)
844 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
846 diaSources = results.sources
848 if self.config.doMeasurement:
849 newDipoleFitting = self.config.doDipoleFitting
850 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
851 if not newDipoleFitting:
853 self.measurement.
run(diaSources, subtractedExposure)
856 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
857 self.measurement.
run(diaSources, subtractedExposure, exposure,
858 subtractRes.matchedExposure)
860 self.measurement.
run(diaSources, subtractedExposure, exposure)
862 if self.config.doForcedMeasurement:
865 forcedSources = self.forcedMeasurement.generateMeasCat(
866 exposure, diaSources, subtractedExposure.getWcs())
867 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
868 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
869 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
870 "ip_diffim_forced_PsfFlux_instFlux",
True)
871 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
872 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
873 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
874 "ip_diffim_forced_PsfFlux_area",
True)
875 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
876 "ip_diffim_forced_PsfFlux_flag",
True)
877 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
878 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
879 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
880 "ip_diffim_forced_PsfFlux_flag_edge",
True)
881 for diaSource, forcedSource
in zip(diaSources, forcedSources):
882 diaSource.assign(forcedSource, mapper)
885 if self.config.doMatchSources:
886 if selectSources
is not None:
888 matchRadAsec = self.config.diaSourceMatchRadius
889 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
891 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
892 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for 893 srcMatch
in srcMatches])
894 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
897 self.log.warn(
"Src product does not exist; cannot match with diaSources")
901 refAstromConfig = AstrometryConfig()
902 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
903 refAstrometer = AstrometryTask(refAstromConfig)
904 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
905 refMatches = astromRet.matches
906 if refMatches
is None:
907 self.log.warn(
"No diaSource matches with reference catalog")
910 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
912 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for 913 refMatch
in refMatches])
916 for diaSource
in diaSources:
917 sid = diaSource.getId()
918 if sid
in srcMatchDict:
919 diaSource.set(
"srcMatchId", srcMatchDict[sid])
920 if sid
in refMatchDict:
921 diaSource.set(
"refMatchId", refMatchDict[sid])
923 if self.config.doAddMetrics
and self.config.doSelectSources:
924 self.log.info(
"Evaluating metrics and control sample")
927 for cell
in subtractRes.kernelCellSet.getCellList():
928 for cand
in cell.begin(
False):
929 kernelCandList.append(cand)
932 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
933 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
936 diffimTools.sourceTableToCandidateList(controlSources,
937 subtractRes.warpedExposure, exposure,
938 self.config.subtract.kernel.active,
939 self.config.subtract.kernel.active.detectionConfig,
940 self.log, doBuild=
True, basisList=basisList))
942 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
943 subtractRes.backgroundModel, dof=nparam)
944 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
945 subtractRes.backgroundModel)
947 if self.config.doDetection:
948 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
950 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
952 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
953 return pipeBase.Struct(
954 subtractedExposure=subtractedExposure,
955 matchedExposure=subtractRes.matchedExposure,
956 subtractRes=subtractRes,
957 diaSources=diaSources,
958 selectSources=selectSources
961 def fitAstrometry(self, templateSources, templateExposure, selectSources):
962 """Fit the relative astrometry between templateSources and selectSources 967 Remove this method. It originally fit a new WCS to the template before calling register.run 968 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 969 It remains because a subtask overrides it. 971 results = self.register.
run(templateSources, templateExposure.getWcs(),
972 templateExposure.getBBox(), selectSources)
975 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
976 """Make debug plots and displays. 980 Test and update for current debug display and slot names 990 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
991 if not maskTransparency:
993 disp.setMaskTransparency(maskTransparency)
995 if display
and showSubtracted:
996 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
997 mi = subtractRes.subtractedExposure.getMaskedImage()
998 x0, y0 = mi.getX0(), mi.getY0()
999 with disp.Buffering():
1000 for s
in diaSources:
1001 x, y = s.getX() - x0, s.getY() - y0
1002 ctype =
"red" if s.get(
"flags_negative")
else "yellow" 1003 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
or 1004 s.get(
"base_PixelFlags_flag_saturatedCenter")
or 1005 s.get(
"base_PixelFlags_flag_crCenter")):
1007 elif (s.get(
"base_PixelFlags_flag_interpolated")
or 1008 s.get(
"base_PixelFlags_flag_saturated")
or 1009 s.get(
"base_PixelFlags_flag_cr")):
1013 disp.dot(ptype, x, y, size=4, ctype=ctype)
1014 lsstDebug.frame += 1
1016 if display
and showPixelResiduals
and selectSources:
1017 nonKernelSources = []
1018 for source
in selectSources:
1019 if source
not in kernelSources:
1020 nonKernelSources.append(source)
1022 diUtils.plotPixelResiduals(exposure,
1023 subtractRes.warpedExposure,
1024 subtractRes.subtractedExposure,
1025 subtractRes.kernelCellSet,
1026 subtractRes.psfMatchingKernel,
1027 subtractRes.backgroundModel,
1029 self.subtract.config.kernel.active.detectionConfig,
1031 diUtils.plotPixelResiduals(exposure,
1032 subtractRes.warpedExposure,
1033 subtractRes.subtractedExposure,
1034 subtractRes.kernelCellSet,
1035 subtractRes.psfMatchingKernel,
1036 subtractRes.backgroundModel,
1038 self.subtract.config.kernel.active.detectionConfig,
1040 if display
and showDiaSources:
1041 flagChecker = SourceFlagChecker(diaSources)
1042 isFlagged = [flagChecker(x)
for x
in diaSources]
1043 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1044 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1045 frame=lsstDebug.frame)
1046 lsstDebug.frame += 1
1048 if display
and showDipoles:
1049 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1050 frame=lsstDebug.frame)
1051 lsstDebug.frame += 1
1053 def _getConfigName(self):
1054 """Return the name of the config dataset 1056 return "%sDiff_config" % (self.config.coaddName,)
1058 def _getMetadataName(self):
1059 """Return the name of the metadata dataset 1061 return "%sDiff_metadata" % (self.config.coaddName,)
1063 def getSchemaCatalogs(self):
1064 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 1065 diaSrc = afwTable.SourceCatalog(self.schema)
1066 diaSrc.getTable().setMetadata(self.algMetadata)
1067 return {self.config.coaddName +
"Diff_diaSrc": diaSrc}
1070 def _makeArgumentParser(cls):
1071 """Create an argument parser 1073 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1074 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1075 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1076 help=
"Template data ID in case of calexp template," 1077 " e.g. --templateId visit=6789")
1081 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1082 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1083 doc=
"Shift stars going into RegisterTask by this amount")
1084 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1085 doc=
"Perturb stars going into RegisterTask by this amount")
1087 def setDefaults(self):
1088 ImageDifferenceConfig.setDefaults(self)
1089 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1092 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1093 """!Image difference Task used in the Winter 2013 data challege. 1094 Enables testing the effects of registration shifts and scatter. 1096 For use with winter 2013 simulated images: 1097 Use --templateId visit=88868666 for sparse data 1098 --templateId visit=22222200 for dense data (g) 1099 --templateId visit=11111100 for dense data (i) 1101 ConfigClass = Winter2013ImageDifferenceConfig
1102 _DefaultName =
"winter2013ImageDifference" 1104 def __init__(self, **kwargs):
1105 ImageDifferenceTask.__init__(self, **kwargs)
1107 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1108 """Fit the relative astrometry between templateSources and selectSources""" 1109 if self.config.winter2013WcsShift > 0.0:
1111 self.config.winter2013WcsShift)
1112 cKey = templateSources[0].getTable().getCentroidKey()
1113 for source
in templateSources:
1114 centroid = source.get(cKey)
1115 source.set(cKey, centroid + offset)
1116 elif self.config.winter2013WcsRms > 0.0:
1117 cKey = templateSources[0].getTable().getCentroidKey()
1118 for source
in templateSources:
1119 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1120 self.config.winter2013WcsRms*numpy.random.normal())
1121 centroid = source.get(cKey)
1122 source.set(cKey, centroid + offset)
1124 results = self.register.
run(templateSources, templateExposure.getWcs(),
1125 templateExposure.getBBox(), selectSources)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)