38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
53 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
54 defaultTemplates={
"coaddName":
"deep",
59 exposure = pipeBase.connectionTypes.Input(
60 doc=
"Input science exposure to subtract from.",
61 dimensions=(
"instrument",
"visit",
"detector"),
62 storageClass=
"ExposureF",
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
76 name=
"{skyMapName}Coadd_skyMap",
77 dimensions=(
"skymap", ),
78 storageClass=
"SkyMap",
80 coaddExposures = pipeBase.connectionTypes.Input(
81 doc=
"Input template to match and subtract from the exposure",
82 dimensions=(
"tract",
"patch",
"skymap",
"band"),
83 storageClass=
"ExposureF",
84 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
88 dcrCoadds = pipeBase.connectionTypes.Input(
89 doc=
"Input DCR template to match and subtract from the exposure",
90 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
91 storageClass=
"ExposureF",
92 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
96 outputSchema = pipeBase.connectionTypes.InitOutput(
97 doc=
"Schema (as an example catalog) for output DIASource catalog.",
98 storageClass=
"SourceCatalog",
99 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
101 subtractedExposure = pipeBase.connectionTypes.Output(
102 doc=
"Output difference image",
103 dimensions=(
"instrument",
"visit",
"detector"),
104 storageClass=
"ExposureF",
105 name=
"{fakesType}{coaddName}Diff_differenceExp",
107 warpedExposure = pipeBase.connectionTypes.Output(
108 doc=
"Warped template used to create `subtractedExposure`.",
109 dimensions=(
"instrument",
"visit",
"detector"),
110 storageClass=
"ExposureF",
111 name=
"{fakesType}{coaddName}Diff_warpedExp",
113 diaSources = pipeBase.connectionTypes.Output(
114 doc=
"Output detected diaSources on the difference image",
115 dimensions=(
"instrument",
"visit",
"detector"),
116 storageClass=
"SourceCatalog",
117 name=
"{fakesType}{coaddName}Diff_diaSrc",
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
122 if config.coaddName ==
'dcr':
123 self.inputs.remove(
"coaddExposures")
125 self.inputs.remove(
"dcrCoadds")
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132 pipelineConnections=ImageDifferenceTaskConnections):
133 """Config for ImageDifferenceTask
135 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
136 doc=
"Add background to calexp before processing it. "
137 "Useful as ipDiffim does background matching.")
138 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
139 doc=
"Use image-to-image registration to align template with "
141 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Writing debugging data for doUseRegister")
143 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
144 doc=
"Select stars to use for kernel fitting")
145 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
146 doc=
"Select stars of extreme color as part of the control sample")
147 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Select stars that are variable to be part "
149 "of the control sample")
150 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
151 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
152 doc=
"Convolve science image by its PSF before PSF-matching?")
153 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
154 doc=
"Scale variance of the template before PSF matching")
155 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
156 doc=
"Use a simple gaussian PSF model for pre-convolution "
157 "(else use fit PSF)? Ignored if doPreConvolve false.")
158 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
159 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
160 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
161 "kernel convolution? If True, also update the diffim PSF.")
162 doMerge = pexConfig.Field(dtype=bool, default=
True,
163 doc=
"Merge positive and negative diaSources with grow radius "
164 "set by growFootprint")
165 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
166 doc=
"Match diaSources with input calexp sources and ref catalog sources")
167 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
168 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
169 doForcedMeasurement = pexConfig.Field(
172 doc=
"Force photometer diaSource locations on PVI?")
173 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
174 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
175 doc=
"Write WCS, warped template coadd exposure?")
176 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
177 doc=
"Write warped and PSF-matched template coadd exposure?")
178 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
179 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
180 doc=
"Add columns to the source table to hold analysis metrics?")
182 coaddName = pexConfig.Field(
183 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
187 convolveTemplate = pexConfig.Field(
188 doc=
"Which image gets convolved (default = template)",
192 refObjLoader = pexConfig.ConfigurableField(
193 target=LoadIndexedReferenceObjectsTask,
194 doc=
"reference object loader",
196 astrometer = pexConfig.ConfigurableField(
197 target=AstrometryTask,
198 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
200 sourceSelector = pexConfig.ConfigurableField(
201 target=ObjectSizeStarSelectorTask,
202 doc=
"Source selection algorithm",
204 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
205 decorrelate = pexConfig.ConfigurableField(
206 target=DecorrelateALKernelSpatialTask,
207 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
211 doSpatiallyVarying = pexConfig.Field(
214 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
215 "image in order to allow for spatial variations"
217 detection = pexConfig.ConfigurableField(
218 target=SourceDetectionTask,
219 doc=
"Low-threshold detection for final measurement",
221 measurement = pexConfig.ConfigurableField(
222 target=DipoleFitTask,
223 doc=
"Enable updated dipole fitting method",
225 forcedMeasurement = pexConfig.ConfigurableField(
226 target=ForcedMeasurementTask,
227 doc=
"Subtask to force photometer PVI at diaSource location.",
229 getTemplate = pexConfig.ConfigurableField(
230 target=GetCoaddAsTemplateTask,
231 doc=
"Subtask to retrieve template exposure and sources",
233 scaleVariance = pexConfig.ConfigurableField(
234 target=ScaleVarianceTask,
235 doc=
"Subtask to rescale the variance of the template "
236 "to the statistically expected level"
238 controlStepSize = pexConfig.Field(
239 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
243 controlRandomSeed = pexConfig.Field(
244 doc=
"Random seed for shuffing the control sample",
248 register = pexConfig.ConfigurableField(
250 doc=
"Task to enable image-to-image image registration (warping)",
252 kernelSourcesFromRef = pexConfig.Field(
253 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
257 templateSipOrder = pexConfig.Field(
258 dtype=int, default=2,
259 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
261 growFootprint = pexConfig.Field(
262 dtype=int, default=2,
263 doc=
"Grow positive and negative footprints by this amount before merging"
265 diaSourceMatchRadius = pexConfig.Field(
266 dtype=float, default=0.5,
267 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
270 def setDefaults(self):
273 self.subtract[
'al'].kernel.name =
"AL"
274 self.subtract[
'al'].kernel.active.fitForBackground =
True
275 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
276 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
277 self.doPreConvolve =
False
278 self.doMatchSources =
False
279 self.doAddMetrics =
False
280 self.doUseRegister =
False
283 self.detection.thresholdPolarity =
"both"
284 self.detection.thresholdValue = 5.5
285 self.detection.reEstimateBackground =
False
286 self.detection.thresholdType =
"pixel_stdev"
292 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
293 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
296 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
297 self.forcedMeasurement.copyColumns = {
298 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
299 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
300 self.forcedMeasurement.slots.shape =
None
303 random.seed(self.controlRandomSeed)
306 pexConfig.Config.validate(self)
307 if self.doAddMetrics
and not self.doSubtract:
308 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
309 if not self.doSubtract
and not self.doDetection:
310 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
311 if self.subtract.name ==
'zogy' and self.doAddMetrics:
312 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
313 if self.doMeasurement
and not self.doDetection:
314 raise ValueError(
"Cannot run source measurement without source detection.")
315 if self.doMerge
and not self.doDetection:
316 raise ValueError(
"Cannot run source merging without source detection.")
317 if self.doUseRegister
and not self.doSelectSources:
318 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
319 "Cannot run RegisterTask without selecting sources.")
320 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
321 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
322 "convolveTemplate=False is not supported.")
323 if hasattr(self.getTemplate,
"coaddName"):
324 if self.getTemplate.coaddName != self.coaddName:
325 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
331 def getTargetList(parsedCmd, **kwargs):
332 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337 """Subtract an image from a template and measure the result
339 ConfigClass = ImageDifferenceConfig
340 RunnerClass = ImageDifferenceTaskRunner
341 _DefaultName =
"imageDifference"
343 def __init__(self, butler=None, **kwargs):
344 """!Construct an ImageDifference Task
346 @param[in] butler Butler object to use in constructing reference object loaders
348 super().__init__(**kwargs)
349 self.makeSubtask(
"getTemplate")
351 self.makeSubtask(
"subtract")
353 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
354 self.makeSubtask(
"decorrelate")
356 if self.config.doScaleTemplateVariance:
357 self.makeSubtask(
"scaleVariance")
359 if self.config.doUseRegister:
360 self.makeSubtask(
"register")
361 self.schema = afwTable.SourceTable.makeMinimalSchema()
363 if self.config.doSelectSources:
364 self.makeSubtask(
"sourceSelector")
365 if self.config.kernelSourcesFromRef:
366 self.makeSubtask(
'refObjLoader', butler=butler)
367 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
369 self.algMetadata = dafBase.PropertyList()
370 if self.config.doDetection:
371 self.makeSubtask(
"detection", schema=self.schema)
372 if self.config.doMeasurement:
373 self.makeSubtask(
"measurement", schema=self.schema,
374 algMetadata=self.algMetadata)
375 if self.config.doForcedMeasurement:
376 self.schema.addField(
377 "ip_diffim_forced_PsfFlux_instFlux",
"D",
378 "Forced PSF flux measured on the direct image.")
379 self.schema.addField(
380 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
381 "Forced PSF flux error measured on the direct image.")
382 self.schema.addField(
383 "ip_diffim_forced_PsfFlux_area",
"F",
384 "Forced PSF flux effective area of PSF.",
386 self.schema.addField(
387 "ip_diffim_forced_PsfFlux_flag",
"Flag",
388 "Forced PSF flux general failure flag.")
389 self.schema.addField(
390 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
391 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392 self.schema.addField(
393 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
394 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
396 if self.config.doMatchSources:
397 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
398 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
401 self.outputSchema = afwTable.SourceCatalog(self.schema)
402 self.outputSchema.getTable().setMetadata(self.algMetadata)
405 def makeIdFactory(expId, expBits):
406 """Create IdFactory instance for unique 64 bit diaSource id-s.
414 Number of used bits in ``expId``.
418 The diasource id-s consists of the ``expId`` stored fixed in the highest value
419 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420 low value end of the integer.
424 idFactory: `lsst.afw.table.IdFactory`
426 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
429 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430 inputRefs: pipeBase.InputQuantizedConnection,
431 outputRefs: pipeBase.OutputQuantizedConnection):
432 inputs = butlerQC.get(inputRefs)
433 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
434 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
436 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437 if self.config.coaddName ==
'dcr':
438 templateExposures = inputRefs.dcrCoadds
440 templateExposures = inputRefs.coaddExposures
441 templateStruct = self.getTemplate.runQuantum(
442 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
445 outputs = self.run(exposure=inputs[
'exposure'],
446 templateExposure=templateStruct.exposure,
448 butlerQC.put(outputs, outputRefs)
451 def runDataRef(self, sensorRef, templateIdList=None):
452 """Subtract an image from a template coadd and measure the result.
454 Data I/O wrapper around `run` using the butler in Gen2.
458 sensorRef : `lsst.daf.persistence.ButlerDataRef`
459 Sensor-level butler data reference, used for the following data products:
466 - self.config.coaddName + "Coadd_skyMap"
467 - self.config.coaddName + "Coadd"
468 Input or output, depending on config:
469 - self.config.coaddName + "Diff_subtractedExp"
470 Output, depending on config:
471 - self.config.coaddName + "Diff_matchedExp"
472 - self.config.coaddName + "Diff_src"
476 results : `lsst.pipe.base.Struct`
477 Returns the Struct by `run`.
479 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
480 subtractedExposure =
None
482 calexpBackgroundExposure =
None
483 self.log.info(
"Processing %s" % (sensorRef.dataId))
488 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
489 expBits=sensorRef.get(
"ccdExposureId_bits"))
490 if self.config.doAddCalexpBackground:
491 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
494 exposure = sensorRef.get(
"calexp", immediate=
True)
497 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
499 if sensorRef.datasetExists(
"src"):
500 self.log.info(
"Source selection via src product")
502 selectSources = sensorRef.get(
"src")
504 if not self.config.doSubtract
and self.config.doDetection:
506 subtractedExposure = sensorRef.get(subtractedExposureName)
509 results = self.run(exposure=exposure,
510 selectSources=selectSources,
511 templateExposure=template.exposure,
512 templateSources=template.sources,
514 calexpBackgroundExposure=calexpBackgroundExposure,
515 subtractedExposure=subtractedExposure)
517 if self.config.doWriteSources
and results.diaSources
is not None:
518 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
519 if self.config.doWriteWarpedExp:
520 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
521 if self.config.doWriteMatchedExp:
522 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
523 if self.config.doAddMetrics
and self.config.doSelectSources:
524 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
525 if self.config.doWriteSubtractedExp:
526 sensorRef.put(results.subtractedExposure, subtractedExposureName)
530 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
531 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
532 """PSF matches, subtract two images and perform detection on the difference image.
536 exposure : `lsst.afw.image.ExposureF`, optional
537 The science exposure, the minuend in the image subtraction.
538 Can be None only if ``config.doSubtract==False``.
539 selectSources : `lsst.afw.table.SourceCatalog`, optional
540 Identified sources on the science exposure. This catalog is used to
541 select sources in order to perform the AL PSF matching on stamp images
542 around them. The selection steps depend on config options and whether
543 ``templateSources`` and ``matchingSources`` specified.
544 templateExposure : `lsst.afw.image.ExposureF`, optional
545 The template to be subtracted from ``exposure`` in the image subtraction.
546 The template exposure should cover the same sky area as the science exposure.
547 It is either a stich of patches of a coadd skymap image or a calexp
548 of the same pointing as the science exposure. Can be None only
549 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
550 templateSources : `lsst.afw.table.SourceCatalog`, optional
551 Identified sources on the template exposure.
552 idFactory : `lsst.afw.table.IdFactory`
553 Generator object to assign ids to detected sources in the difference image.
554 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
555 Background exposure to be added back to the science exposure
556 if ``config.doAddCalexpBackground==True``
557 subtractedExposure : `lsst.afw.image.ExposureF`, optional
558 If ``config.doSubtract==False`` and ``config.doDetection==True``,
559 performs the post subtraction source detection only on this exposure.
560 Otherwise should be None.
564 results : `lsst.pipe.base.Struct`
565 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
567 ``matchedExposure`` : `lsst.afw.image.ExposureF`
568 The matched PSF exposure.
569 ``subtractRes`` : `lsst.pipe.base.Struct`
570 The returned result structure of the ImagePsfMatchTask subtask.
571 ``diaSources`` : `lsst.afw.table.SourceCatalog`
572 The catalog of detected sources.
573 ``selectSources`` : `lsst.afw.table.SourceCatalog`
574 The input source catalog with optionally added Qa information.
578 The following major steps are included:
580 - warp template coadd to match WCS of image
581 - PSF match image to warped template
582 - subtract image from PSF-matched, warped template
586 For details about the image subtraction configuration modes
587 see `lsst.ip.diffim`.
590 controlSources =
None
594 if self.config.doAddCalexpBackground:
595 mi = exposure.getMaskedImage()
596 mi += calexpBackgroundExposure.getImage()
598 if not exposure.hasPsf():
599 raise pipeBase.TaskError(
"Exposure has no psf")
600 sciencePsf = exposure.getPsf()
602 if self.config.doSubtract:
603 if self.config.doScaleTemplateVariance:
604 templateVarFactor = self.scaleVariance.
run(
605 templateExposure.getMaskedImage())
606 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
608 if self.config.subtract.name ==
'zogy':
609 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
610 if self.config.doPreConvolve:
611 subtractedExposure = subtractRes.scoreExp
613 subtractedExposure = subtractRes.diffExp
614 subtractRes.subtractedExposure = subtractedExposure
615 subtractRes.matchedExposure =
None
617 elif self.config.subtract.name ==
'al':
619 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
620 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
628 if self.config.doPreConvolve:
629 convControl = afwMath.ConvolutionControl()
631 srcMI = exposure.getMaskedImage()
632 exposureOrig = exposure.clone()
633 destMI = srcMI.Factory(srcMI.getDimensions())
635 if self.config.useGaussianForPreConvolution:
637 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
642 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
643 exposure.setMaskedImage(destMI)
644 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
646 scienceSigmaPost = scienceSigmaOrig
647 exposureOrig = exposure
652 if self.config.doSelectSources:
653 if selectSources
is None:
654 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
656 selectSources = self.subtract.getSelectSources(
658 sigma=scienceSigmaPost,
659 doSmooth=
not self.config.doPreConvolve,
663 if self.config.doAddMetrics:
666 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
667 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
668 targetFwhmPix=templateSigma*FwhmPerSigma))
675 kcQa = KernelCandidateQa(nparam)
676 selectSources = kcQa.addToSchema(selectSources)
677 if self.config.kernelSourcesFromRef:
679 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
680 matches = astromRet.matches
681 elif templateSources:
683 mc = afwTable.MatchControl()
684 mc.findOnlyClosest =
False
685 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
688 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
689 "but template sources not available. Cannot match science "
690 "sources with template sources. Run process* on data from "
691 "which templates are built.")
693 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
694 matches=matches).sourceCat
695 random.shuffle(kernelSources, random.random)
696 controlSources = kernelSources[::self.config.controlStepSize]
697 kernelSources = [k
for i, k
in enumerate(kernelSources)
698 if i % self.config.controlStepSize]
700 if self.config.doSelectDcrCatalog:
701 redSelector = DiaCatalogSourceSelectorTask(
702 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
704 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
705 controlSources.extend(redSources)
707 blueSelector = DiaCatalogSourceSelectorTask(
708 DiaCatalogSourceSelectorConfig(grMin=-99.999,
709 grMax=self.sourceSelector.config.grMin))
710 blueSources = blueSelector.selectStars(exposure, selectSources,
711 matches=matches).starCat
712 controlSources.extend(blueSources)
714 if self.config.doSelectVariableCatalog:
715 varSelector = DiaCatalogSourceSelectorTask(
716 DiaCatalogSourceSelectorConfig(includeVariable=
True))
717 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
718 controlSources.extend(varSources)
720 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
721 % (len(kernelSources), len(selectSources), len(controlSources)))
725 if self.config.doUseRegister:
726 self.log.info(
"Registering images")
728 if templateSources
is None:
732 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
733 templateSources = self.subtract.getSelectSources(
742 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
743 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
744 exposure.getWcs(), exposure.getBBox())
745 templateExposure = warpedExp
750 if self.config.doDebugRegister:
752 srcToMatch = {x.second.getId(): x.first
for x
in matches}
754 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
755 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
756 sids = [m.first.getId()
for m
in wcsResults.matches]
757 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
758 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
759 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
760 allresids = dict(zip(sids, zip(positions, residuals)))
762 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
763 wcsResults.wcs.pixelToSky(
764 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
765 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
766 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
767 for x
in sids
if x
in srcToMatch.keys()])
768 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
769 if s
in srcToMatch.keys()])
770 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
771 if s
in srcToMatch.keys()])
772 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
773 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
774 & (colors <= self.sourceSelector.config.grMax))
775 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
776 rms1Long = IqrToSigma*(
777 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
778 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
779 - numpy.percentile(dlat[idx1], 25))
780 rms2Long = IqrToSigma*(
781 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
782 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
783 - numpy.percentile(dlat[idx2], 25))
784 rms3Long = IqrToSigma*(
785 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
786 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
787 - numpy.percentile(dlat[idx3], 25))
788 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
789 (numpy.median(dlong[idx1]), rms1Long,
790 numpy.median(dlat[idx1]), rms1Lat))
791 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
792 (numpy.median(dlong[idx2]), rms2Long,
793 numpy.median(dlat[idx2]), rms2Lat))
794 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
795 (numpy.median(dlong[idx3]), rms3Long,
796 numpy.median(dlat[idx3]), rms3Lat))
798 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
799 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
800 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
801 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
802 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
803 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
805 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
806 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
807 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
808 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
809 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
810 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
817 self.log.info(
"Subtracting images")
818 subtractRes = self.subtract.subtractExposures(
819 templateExposure=templateExposure,
820 scienceExposure=exposure,
821 candidateList=kernelSources,
822 convolveTemplate=self.config.convolveTemplate,
823 doWarping=
not self.config.doUseRegister
825 subtractedExposure = subtractRes.subtractedExposure
827 if self.config.doDetection:
828 self.log.info(
"Computing diffim PSF")
831 if not subtractedExposure.hasPsf():
832 if self.config.convolveTemplate:
833 subtractedExposure.setPsf(exposure.getPsf())
835 subtractedExposure.setPsf(templateExposure.getPsf())
842 if self.config.doDecorrelation
and self.config.doSubtract:
844 if preConvPsf
is not None:
845 preConvKernel = preConvPsf.getLocalKernel()
846 if self.config.convolveTemplate:
847 self.log.info(
"Decorrelation after template image convolution")
848 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
850 subtractRes.psfMatchingKernel,
851 spatiallyVarying=self.config.doSpatiallyVarying,
852 preConvKernel=preConvKernel)
854 self.log.info(
"Decorrelation after science image convolution")
855 decorrResult = self.decorrelate.
run(subtractRes.warpedExposure, exposureOrig,
857 subtractRes.psfMatchingKernel,
858 spatiallyVarying=self.config.doSpatiallyVarying,
859 preConvKernel=preConvKernel)
860 subtractedExposure = decorrResult.correctedExposure
864 if self.config.doDetection:
865 self.log.info(
"Running diaSource detection")
867 mask = subtractedExposure.getMaskedImage().getMask()
868 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
870 table = afwTable.SourceTable.make(self.schema, idFactory)
871 table.setMetadata(self.algMetadata)
872 results = self.detection.
run(
874 exposure=subtractedExposure,
875 doSmooth=
not self.config.doPreConvolve
878 if self.config.doMerge:
879 fpSet = results.fpSets.positive
880 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
881 self.config.growFootprint,
False)
882 diaSources = afwTable.SourceCatalog(table)
883 fpSet.makeSources(diaSources)
884 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
886 diaSources = results.sources
888 if self.config.doMeasurement:
889 newDipoleFitting = self.config.doDipoleFitting
890 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
891 if not newDipoleFitting:
893 self.measurement.
run(diaSources, subtractedExposure)
896 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
897 self.measurement.
run(diaSources, subtractedExposure, exposure,
898 subtractRes.matchedExposure)
900 self.measurement.
run(diaSources, subtractedExposure, exposure)
902 if self.config.doForcedMeasurement:
905 forcedSources = self.forcedMeasurement.generateMeasCat(
906 exposure, diaSources, subtractedExposure.getWcs())
907 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
908 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
909 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
910 "ip_diffim_forced_PsfFlux_instFlux",
True)
911 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
912 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
913 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
914 "ip_diffim_forced_PsfFlux_area",
True)
915 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
916 "ip_diffim_forced_PsfFlux_flag",
True)
917 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
918 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
919 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
920 "ip_diffim_forced_PsfFlux_flag_edge",
True)
921 for diaSource, forcedSource
in zip(diaSources, forcedSources):
922 diaSource.assign(forcedSource, mapper)
925 if self.config.doMatchSources:
926 if selectSources
is not None:
928 matchRadAsec = self.config.diaSourceMatchRadius
929 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
931 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
932 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
933 srcMatch
in srcMatches])
934 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
937 self.log.warn(
"Src product does not exist; cannot match with diaSources")
941 refAstromConfig = AstrometryConfig()
942 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
943 refAstrometer = AstrometryTask(refAstromConfig)
944 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
945 refMatches = astromRet.matches
946 if refMatches
is None:
947 self.log.warn(
"No diaSource matches with reference catalog")
950 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
952 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
953 refMatch
in refMatches])
956 for diaSource
in diaSources:
957 sid = diaSource.getId()
958 if sid
in srcMatchDict:
959 diaSource.set(
"srcMatchId", srcMatchDict[sid])
960 if sid
in refMatchDict:
961 diaSource.set(
"refMatchId", refMatchDict[sid])
963 if self.config.doAddMetrics
and self.config.doSelectSources:
964 self.log.info(
"Evaluating metrics and control sample")
967 for cell
in subtractRes.kernelCellSet.getCellList():
968 for cand
in cell.begin(
False):
969 kernelCandList.append(cand)
972 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
973 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
976 diffimTools.sourceTableToCandidateList(controlSources,
977 subtractRes.warpedExposure, exposure,
978 self.config.subtract.kernel.active,
979 self.config.subtract.kernel.active.detectionConfig,
980 self.log, doBuild=
True, basisList=basisList))
982 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
983 subtractRes.backgroundModel, dof=nparam)
984 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
985 subtractRes.backgroundModel)
987 if self.config.doDetection:
988 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
990 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
992 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
993 return pipeBase.Struct(
994 subtractedExposure=subtractedExposure,
995 warpedExposure=subtractRes.warpedExposure,
996 matchedExposure=subtractRes.matchedExposure,
997 subtractRes=subtractRes,
998 diaSources=diaSources,
999 selectSources=selectSources
1002 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1003 """Fit the relative astrometry between templateSources and selectSources
1008 Remove this method. It originally fit a new WCS to the template before calling register.run
1009 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1010 It remains because a subtask overrides it.
1012 results = self.register.
run(templateSources, templateExposure.getWcs(),
1013 templateExposure.getBBox(), selectSources)
1016 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1017 """Make debug plots and displays.
1021 Test and update for current debug display and slot names
1031 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1032 if not maskTransparency:
1033 maskTransparency = 0
1034 disp.setMaskTransparency(maskTransparency)
1036 if display
and showSubtracted:
1037 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1038 mi = subtractRes.subtractedExposure.getMaskedImage()
1039 x0, y0 = mi.getX0(), mi.getY0()
1040 with disp.Buffering():
1041 for s
in diaSources:
1042 x, y = s.getX() - x0, s.getY() - y0
1043 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1044 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1045 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1046 or s.get(
"base_PixelFlags_flag_crCenter")):
1048 elif (s.get(
"base_PixelFlags_flag_interpolated")
1049 or s.get(
"base_PixelFlags_flag_saturated")
1050 or s.get(
"base_PixelFlags_flag_cr")):
1054 disp.dot(ptype, x, y, size=4, ctype=ctype)
1055 lsstDebug.frame += 1
1057 if display
and showPixelResiduals
and selectSources:
1058 nonKernelSources = []
1059 for source
in selectSources:
1060 if source
not in kernelSources:
1061 nonKernelSources.append(source)
1063 diUtils.plotPixelResiduals(exposure,
1064 subtractRes.warpedExposure,
1065 subtractRes.subtractedExposure,
1066 subtractRes.kernelCellSet,
1067 subtractRes.psfMatchingKernel,
1068 subtractRes.backgroundModel,
1070 self.subtract.config.kernel.active.detectionConfig,
1072 diUtils.plotPixelResiduals(exposure,
1073 subtractRes.warpedExposure,
1074 subtractRes.subtractedExposure,
1075 subtractRes.kernelCellSet,
1076 subtractRes.psfMatchingKernel,
1077 subtractRes.backgroundModel,
1079 self.subtract.config.kernel.active.detectionConfig,
1081 if display
and showDiaSources:
1082 flagChecker = SourceFlagChecker(diaSources)
1083 isFlagged = [flagChecker(x)
for x
in diaSources]
1084 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1085 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1086 frame=lsstDebug.frame)
1087 lsstDebug.frame += 1
1089 if display
and showDipoles:
1090 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1091 frame=lsstDebug.frame)
1092 lsstDebug.frame += 1
1094 def _getConfigName(self):
1095 """Return the name of the config dataset
1097 return "%sDiff_config" % (self.config.coaddName,)
1099 def _getMetadataName(self):
1100 """Return the name of the metadata dataset
1102 return "%sDiff_metadata" % (self.config.coaddName,)
1104 def getSchemaCatalogs(self):
1105 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1106 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1109 def _makeArgumentParser(cls):
1110 """Create an argument parser
1112 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1113 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1114 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1115 help=
"Template data ID in case of calexp template,"
1116 " e.g. --templateId visit=6789")
1120 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1121 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1122 doc=
"Shift stars going into RegisterTask by this amount")
1123 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1124 doc=
"Perturb stars going into RegisterTask by this amount")
1126 def setDefaults(self):
1127 ImageDifferenceConfig.setDefaults(self)
1128 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1131 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1132 """!Image difference Task used in the Winter 2013 data challege.
1133 Enables testing the effects of registration shifts and scatter.
1135 For use with winter 2013 simulated images:
1136 Use --templateId visit=88868666 for sparse data
1137 --templateId visit=22222200 for dense data (g)
1138 --templateId visit=11111100 for dense data (i)
1140 ConfigClass = Winter2013ImageDifferenceConfig
1141 _DefaultName =
"winter2013ImageDifference"
1143 def __init__(self, **kwargs):
1144 ImageDifferenceTask.__init__(self, **kwargs)
1146 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1147 """Fit the relative astrometry between templateSources and selectSources"""
1148 if self.config.winter2013WcsShift > 0.0:
1150 self.config.winter2013WcsShift)
1151 cKey = templateSources[0].getTable().getCentroidKey()
1152 for source
in templateSources:
1153 centroid = source.get(cKey)
1154 source.set(cKey, centroid + offset)
1155 elif self.config.winter2013WcsRms > 0.0:
1156 cKey = templateSources[0].getTable().getCentroidKey()
1157 for source
in templateSources:
1158 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1159 self.config.winter2013WcsRms*numpy.random.normal())
1160 centroid = source.get(cKey)
1161 source.set(cKey, centroid + offset)
1163 results = self.register.
run(templateSources, templateExposure.getWcs(),
1164 templateExposure.getBBox(), selectSources)