27 import lsst.pex.config
as pexConfig
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
53 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
54 defaultTemplates={
"coaddName":
"deep",
59 exposure = pipeBase.connectionTypes.Input(
60 doc=
"Input science exposure to subtract from.",
61 dimensions=(
"instrument",
"visit",
"detector"),
62 storageClass=
"ExposureF",
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
76 name=
"{skyMapName}Coadd_skyMap",
77 dimensions=(
"skymap", ),
78 storageClass=
"SkyMap",
80 coaddExposures = pipeBase.connectionTypes.Input(
81 doc=
"Input template to match and subtract from the exposure",
82 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
83 storageClass=
"ExposureF",
84 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
88 dcrCoadds = pipeBase.connectionTypes.Input(
89 doc=
"Input DCR template to match and subtract from the exposure",
90 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
91 storageClass=
"ExposureF",
92 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter",
"subfilter"),
96 outputSchema = pipeBase.connectionTypes.InitOutput(
97 doc=
"Schema (as an example catalog) for output DIASource catalog.",
98 storageClass=
"SourceCatalog",
99 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
101 subtractedExposure = pipeBase.connectionTypes.Output(
102 doc=
"Output difference image",
103 dimensions=(
"instrument",
"visit",
"detector"),
104 storageClass=
"ExposureF",
105 name=
"{fakesType}{coaddName}Diff_differenceExp",
107 warpedExposure = pipeBase.connectionTypes.Output(
108 doc=
"Warped template used to create `subtractedExposure`.",
109 dimensions=(
"instrument",
"visit",
"detector"),
110 storageClass=
"ExposureF",
111 name=
"{fakesType}{coaddName}Diff_warpedExp",
113 diaSources = pipeBase.connectionTypes.Output(
114 doc=
"Output detected diaSources on the difference image",
115 dimensions=(
"instrument",
"visit",
"detector"),
116 storageClass=
"SourceCatalog",
117 name=
"{fakesType}{coaddName}Diff_diaSrc",
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
122 if config.coaddName ==
'dcr':
123 self.inputs.remove(
"coaddExposures")
125 self.inputs.remove(
"dcrCoadds")
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132 pipelineConnections=ImageDifferenceTaskConnections):
133 """Config for ImageDifferenceTask
135 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
136 doc=
"Add background to calexp before processing it. "
137 "Useful as ipDiffim does background matching.")
138 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
139 doc=
"Use image-to-image registration to align template with "
141 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Writing debugging data for doUseRegister")
143 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
144 doc=
"Select stars to use for kernel fitting")
145 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
146 doc=
"Select stars of extreme color as part of the control sample")
147 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Select stars that are variable to be part "
149 "of the control sample")
150 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
151 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
152 doc=
"Convolve science image by its PSF before PSF-matching?")
153 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
154 doc=
"Scale variance of the template before PSF matching")
155 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
156 doc=
"Use a simple gaussian PSF model for pre-convolution "
157 "(else use fit PSF)? Ignored if doPreConvolve false.")
158 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
159 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
160 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
161 "kernel convolution? If True, also update the diffim PSF.")
162 doMerge = pexConfig.Field(dtype=bool, default=
True,
163 doc=
"Merge positive and negative diaSources with grow radius "
164 "set by growFootprint")
165 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
166 doc=
"Match diaSources with input calexp sources and ref catalog sources")
167 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
168 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
169 doForcedMeasurement = pexConfig.Field(
172 doc=
"Force photometer diaSource locations on PVI?")
173 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
174 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
175 doc=
"Write WCS, warped template coadd exposure?")
176 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
177 doc=
"Write warped and PSF-matched template coadd exposure?")
178 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
179 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
180 doc=
"Add columns to the source table to hold analysis metrics?")
182 coaddName = pexConfig.Field(
183 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
187 convolveTemplate = pexConfig.Field(
188 doc=
"Which image gets convolved (default = template)",
192 refObjLoader = pexConfig.ConfigurableField(
193 target=LoadIndexedReferenceObjectsTask,
194 doc=
"reference object loader",
196 astrometer = pexConfig.ConfigurableField(
197 target=AstrometryTask,
198 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
200 sourceSelector = pexConfig.ConfigurableField(
201 target=ObjectSizeStarSelectorTask,
202 doc=
"Source selection algorithm",
204 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
205 decorrelate = pexConfig.ConfigurableField(
206 target=DecorrelateALKernelSpatialTask,
207 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
211 doSpatiallyVarying = pexConfig.Field(
214 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
215 "image in order to allow for spatial variations"
217 detection = pexConfig.ConfigurableField(
218 target=SourceDetectionTask,
219 doc=
"Low-threshold detection for final measurement",
221 measurement = pexConfig.ConfigurableField(
222 target=DipoleFitTask,
223 doc=
"Enable updated dipole fitting method",
225 forcedMeasurement = pexConfig.ConfigurableField(
226 target=ForcedMeasurementTask,
227 doc=
"Subtask to force photometer PVI at diaSource location.",
229 getTemplate = pexConfig.ConfigurableField(
230 target=GetCoaddAsTemplateTask,
231 doc=
"Subtask to retrieve template exposure and sources",
233 scaleVariance = pexConfig.ConfigurableField(
234 target=ScaleVarianceTask,
235 doc=
"Subtask to rescale the variance of the template "
236 "to the statistically expected level"
238 controlStepSize = pexConfig.Field(
239 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
243 controlRandomSeed = pexConfig.Field(
244 doc=
"Random seed for shuffing the control sample",
248 register = pexConfig.ConfigurableField(
250 doc=
"Task to enable image-to-image image registration (warping)",
252 kernelSourcesFromRef = pexConfig.Field(
253 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
257 templateSipOrder = pexConfig.Field(
258 dtype=int, default=2,
259 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
261 growFootprint = pexConfig.Field(
262 dtype=int, default=2,
263 doc=
"Grow positive and negative footprints by this amount before merging"
265 diaSourceMatchRadius = pexConfig.Field(
266 dtype=float, default=0.5,
267 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
270 def setDefaults(self):
273 self.subtract[
'al'].kernel.name =
"AL"
274 self.subtract[
'al'].kernel.active.fitForBackground =
True
275 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
276 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
277 self.doPreConvolve =
False
278 self.doMatchSources =
False
279 self.doAddMetrics =
False
280 self.doUseRegister =
False
283 self.detection.thresholdPolarity =
"both"
284 self.detection.thresholdValue = 5.5
285 self.detection.reEstimateBackground =
False
286 self.detection.thresholdType =
"pixel_stdev"
292 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
293 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
296 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
297 self.forcedMeasurement.copyColumns = {
298 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
299 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
300 self.forcedMeasurement.slots.shape =
None
303 random.seed(self.controlRandomSeed)
306 pexConfig.Config.validate(self)
307 if self.doAddMetrics
and not self.doSubtract:
308 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
309 if not self.doSubtract
and not self.doDetection:
310 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
311 if self.subtract.name ==
'zogy' and self.doAddMetrics:
312 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
313 if self.doMeasurement
and not self.doDetection:
314 raise ValueError(
"Cannot run source measurement without source detection.")
315 if self.doMerge
and not self.doDetection:
316 raise ValueError(
"Cannot run source merging without source detection.")
317 if self.doUseRegister
and not self.doSelectSources:
318 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
319 "Cannot run RegisterTask without selecting sources.")
320 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
321 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
322 "convolveTemplate=False is not supported.")
323 if hasattr(self.getTemplate,
"coaddName"):
324 if self.getTemplate.coaddName != self.coaddName:
325 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
331 def getTargetList(parsedCmd, **kwargs):
332 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337 """Subtract an image from a template and measure the result
339 ConfigClass = ImageDifferenceConfig
340 RunnerClass = ImageDifferenceTaskRunner
341 _DefaultName =
"imageDifference"
343 def __init__(self, butler=None, **kwargs):
344 """!Construct an ImageDifference Task
346 @param[in] butler Butler object to use in constructing reference object loaders
348 super().__init__(**kwargs)
349 self.makeSubtask(
"getTemplate")
351 self.makeSubtask(
"subtract")
353 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
354 self.makeSubtask(
"decorrelate")
356 if self.config.doScaleTemplateVariance:
357 self.makeSubtask(
"scaleVariance")
359 if self.config.doUseRegister:
360 self.makeSubtask(
"register")
361 self.schema = afwTable.SourceTable.makeMinimalSchema()
363 if self.config.doSelectSources:
364 self.makeSubtask(
"sourceSelector")
365 if self.config.kernelSourcesFromRef:
366 self.makeSubtask(
'refObjLoader', butler=butler)
367 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
369 self.algMetadata = dafBase.PropertyList()
370 if self.config.doDetection:
371 self.makeSubtask(
"detection", schema=self.schema)
372 if self.config.doMeasurement:
373 self.makeSubtask(
"measurement", schema=self.schema,
374 algMetadata=self.algMetadata)
375 if self.config.doForcedMeasurement:
376 self.schema.addField(
377 "ip_diffim_forced_PsfFlux_instFlux",
"D",
378 "Forced PSF flux measured on the direct image.")
379 self.schema.addField(
380 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
381 "Forced PSF flux error measured on the direct image.")
382 self.schema.addField(
383 "ip_diffim_forced_PsfFlux_area",
"F",
384 "Forced PSF flux effective area of PSF.",
386 self.schema.addField(
387 "ip_diffim_forced_PsfFlux_flag",
"Flag",
388 "Forced PSF flux general failure flag.")
389 self.schema.addField(
390 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
391 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392 self.schema.addField(
393 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
394 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
396 if self.config.doMatchSources:
397 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
398 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
401 self.outputSchema = afwTable.SourceCatalog(self.schema)
402 self.outputSchema.getTable().setMetadata(self.algMetadata)
405 def makeIdFactory(expId, expBits):
406 """Create IdFactory instance for unique 64 bit diaSource id-s.
414 Number of used bits in ``expId``.
418 The diasource id-s consists of the ``expId`` stored fixed in the highest value
419 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420 low value end of the integer.
424 idFactory: `lsst.afw.table.IdFactory`
426 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
429 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430 inputRefs: pipeBase.InputQuantizedConnection,
431 outputRefs: pipeBase.OutputQuantizedConnection):
432 inputs = butlerQC.get(inputRefs)
433 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
434 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
436 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437 if self.config.coaddName ==
'dcr':
438 templateExposures = inputRefs.dcrCoadds
440 templateExposures = inputRefs.coaddExposures
441 templateStruct = self.getTemplate.runQuantum(
442 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
445 outputs = self.run(exposure=inputs[
'exposure'],
446 templateExposure=templateStruct.exposure,
448 butlerQC.put(outputs, outputRefs)
451 def runDataRef(self, sensorRef, templateIdList=None):
452 """Subtract an image from a template coadd and measure the result.
454 Data I/O wrapper around `run` using the butler in Gen2.
458 sensorRef : `lsst.daf.persistence.ButlerDataRef`
459 Sensor-level butler data reference, used for the following data products:
466 - self.config.coaddName + "Coadd_skyMap"
467 - self.config.coaddName + "Coadd"
468 Input or output, depending on config:
469 - self.config.coaddName + "Diff_subtractedExp"
470 Output, depending on config:
471 - self.config.coaddName + "Diff_matchedExp"
472 - self.config.coaddName + "Diff_src"
476 results : `lsst.pipe.base.Struct`
477 Returns the Struct by `run`.
479 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
480 subtractedExposure =
None
482 calexpBackgroundExposure =
None
483 self.log.info(
"Processing %s" % (sensorRef.dataId))
488 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
489 expBits=sensorRef.get(
"ccdExposureId_bits"))
490 if self.config.doAddCalexpBackground:
491 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
494 exposure = sensorRef.get(
"calexp", immediate=
True)
497 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
499 if sensorRef.datasetExists(
"src"):
500 self.log.info(
"Source selection via src product")
502 selectSources = sensorRef.get(
"src")
504 if not self.config.doSubtract
and self.config.doDetection:
506 subtractedExposure = sensorRef.get(subtractedExposureName)
509 results = self.run(exposure=exposure,
510 selectSources=selectSources,
511 templateExposure=template.exposure,
512 templateSources=template.sources,
514 calexpBackgroundExposure=calexpBackgroundExposure,
515 subtractedExposure=subtractedExposure)
517 if self.config.doWriteSources
and results.diaSources
is not None:
518 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
519 if self.config.doWriteWarpedExp:
520 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
521 if self.config.doWriteMatchedExp:
522 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
523 if self.config.doAddMetrics
and self.config.doSelectSources:
524 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
525 if self.config.doWriteSubtractedExp:
526 sensorRef.put(results.subtractedExposure, subtractedExposureName)
529 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
530 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
531 """PSF matches, subtract two images and perform detection on the difference image.
535 exposure : `lsst.afw.image.ExposureF`, optional
536 The science exposure, the minuend in the image subtraction.
537 Can be None only if ``config.doSubtract==False``.
538 selectSources : `lsst.afw.table.SourceCatalog`, optional
539 Identified sources on the science exposure. This catalog is used to
540 select sources in order to perform the AL PSF matching on stamp images
541 around them. The selection steps depend on config options and whether
542 ``templateSources`` and ``matchingSources`` specified.
543 templateExposure : `lsst.afw.image.ExposureF`, optional
544 The template to be subtracted from ``exposure`` in the image subtraction.
545 The template exposure should cover the same sky area as the science exposure.
546 It is either a stich of patches of a coadd skymap image or a calexp
547 of the same pointing as the science exposure. Can be None only
548 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
549 templateSources : `lsst.afw.table.SourceCatalog`, optional
550 Identified sources on the template exposure.
551 idFactory : `lsst.afw.table.IdFactory`
552 Generator object to assign ids to detected sources in the difference image.
553 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
554 Background exposure to be added back to the science exposure
555 if ``config.doAddCalexpBackground==True``
556 subtractedExposure : `lsst.afw.image.ExposureF`, optional
557 If ``config.doSubtract==False`` and ``config.doDetection==True``,
558 performs the post subtraction source detection only on this exposure.
559 Otherwise should be None.
563 results : `lsst.pipe.base.Struct`
564 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
566 ``matchedExposure`` : `lsst.afw.image.ExposureF`
567 The matched PSF exposure.
568 ``subtractRes`` : `lsst.pipe.base.Struct`
569 The returned result structure of the ImagePsfMatchTask subtask.
570 ``diaSources`` : `lsst.afw.table.SourceCatalog`
571 The catalog of detected sources.
572 ``selectSources`` : `lsst.afw.table.SourceCatalog`
573 The input source catalog with optionally added Qa information.
577 The following major steps are included:
579 - warp template coadd to match WCS of image
580 - PSF match image to warped template
581 - subtract image from PSF-matched, warped template
585 For details about the image subtraction configuration modes
586 see `lsst.ip.diffim`.
589 controlSources =
None
593 if self.config.doAddCalexpBackground:
594 mi = exposure.getMaskedImage()
595 mi += calexpBackgroundExposure.getImage()
597 if not exposure.hasPsf():
598 raise pipeBase.TaskError(
"Exposure has no psf")
599 sciencePsf = exposure.getPsf()
601 if self.config.doSubtract:
602 if self.config.doScaleTemplateVariance:
603 templateVarFactor = self.scaleVariance.
run(
604 templateExposure.getMaskedImage())
605 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
607 if self.config.subtract.name ==
'zogy':
608 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
610 spatiallyVarying=self.config.doSpatiallyVarying,
611 doPreConvolve=self.config.doPreConvolve)
612 subtractedExposure = subtractRes.subtractedExposure
614 elif self.config.subtract.name ==
'al':
616 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
617 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
625 if self.config.doPreConvolve:
626 convControl = afwMath.ConvolutionControl()
628 srcMI = exposure.getMaskedImage()
629 exposureOrig = exposure.clone()
630 destMI = srcMI.Factory(srcMI.getDimensions())
632 if self.config.useGaussianForPreConvolution:
634 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
639 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
640 exposure.setMaskedImage(destMI)
641 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
643 scienceSigmaPost = scienceSigmaOrig
644 exposureOrig = exposure
649 if self.config.doSelectSources:
650 if selectSources
is None:
651 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
653 selectSources = self.subtract.getSelectSources(
655 sigma=scienceSigmaPost,
656 doSmooth=
not self.config.doPreConvolve,
660 if self.config.doAddMetrics:
663 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
664 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
665 targetFwhmPix=templateSigma*FwhmPerSigma))
672 kcQa = KernelCandidateQa(nparam)
673 selectSources = kcQa.addToSchema(selectSources)
674 if self.config.kernelSourcesFromRef:
676 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
677 matches = astromRet.matches
678 elif templateSources:
680 mc = afwTable.MatchControl()
681 mc.findOnlyClosest =
False
682 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
685 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
686 "but template sources not available. Cannot match science "
687 "sources with template sources. Run process* on data from "
688 "which templates are built.")
690 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
691 matches=matches).sourceCat
692 random.shuffle(kernelSources, random.random)
693 controlSources = kernelSources[::self.config.controlStepSize]
694 kernelSources = [k
for i, k
in enumerate(kernelSources)
695 if i % self.config.controlStepSize]
697 if self.config.doSelectDcrCatalog:
698 redSelector = DiaCatalogSourceSelectorTask(
699 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
701 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
702 controlSources.extend(redSources)
704 blueSelector = DiaCatalogSourceSelectorTask(
705 DiaCatalogSourceSelectorConfig(grMin=-99.999,
706 grMax=self.sourceSelector.config.grMin))
707 blueSources = blueSelector.selectStars(exposure, selectSources,
708 matches=matches).starCat
709 controlSources.extend(blueSources)
711 if self.config.doSelectVariableCatalog:
712 varSelector = DiaCatalogSourceSelectorTask(
713 DiaCatalogSourceSelectorConfig(includeVariable=
True))
714 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
715 controlSources.extend(varSources)
717 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
718 % (len(kernelSources), len(selectSources), len(controlSources)))
722 if self.config.doUseRegister:
723 self.log.info(
"Registering images")
725 if templateSources
is None:
729 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
730 templateSources = self.subtract.getSelectSources(
739 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
740 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
741 exposure.getWcs(), exposure.getBBox())
742 templateExposure = warpedExp
747 if self.config.doDebugRegister:
749 srcToMatch = {x.second.getId(): x.first
for x
in matches}
751 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
752 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
753 sids = [m.first.getId()
for m
in wcsResults.matches]
754 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
755 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
756 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
757 allresids = dict(zip(sids, zip(positions, residuals)))
759 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
760 wcsResults.wcs.pixelToSky(
761 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
762 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
763 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
764 for x
in sids
if x
in srcToMatch.keys()])
765 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
766 if s
in srcToMatch.keys()])
767 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
768 if s
in srcToMatch.keys()])
769 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
770 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
771 & (colors <= self.sourceSelector.config.grMax))
772 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
773 rms1Long = IqrToSigma*(
774 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
775 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
776 - numpy.percentile(dlat[idx1], 25))
777 rms2Long = IqrToSigma*(
778 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
779 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
780 - numpy.percentile(dlat[idx2], 25))
781 rms3Long = IqrToSigma*(
782 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
783 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
784 - numpy.percentile(dlat[idx3], 25))
785 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
786 (numpy.median(dlong[idx1]), rms1Long,
787 numpy.median(dlat[idx1]), rms1Lat))
788 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
789 (numpy.median(dlong[idx2]), rms2Long,
790 numpy.median(dlat[idx2]), rms2Lat))
791 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
792 (numpy.median(dlong[idx3]), rms3Long,
793 numpy.median(dlat[idx3]), rms3Lat))
795 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
796 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
797 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
798 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
799 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
800 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
802 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
803 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
804 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
805 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
806 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
807 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
814 self.log.info(
"Subtracting images")
815 subtractRes = self.subtract.subtractExposures(
816 templateExposure=templateExposure,
817 scienceExposure=exposure,
818 candidateList=kernelSources,
819 convolveTemplate=self.config.convolveTemplate,
820 doWarping=
not self.config.doUseRegister
822 subtractedExposure = subtractRes.subtractedExposure
824 if self.config.doDetection:
825 self.log.info(
"Computing diffim PSF")
828 if not subtractedExposure.hasPsf():
829 if self.config.convolveTemplate:
830 subtractedExposure.setPsf(exposure.getPsf())
832 subtractedExposure.setPsf(templateExposure.getPsf())
839 if self.config.doDecorrelation
and self.config.doSubtract:
841 if preConvPsf
is not None:
842 preConvKernel = preConvPsf.getLocalKernel()
843 if self.config.convolveTemplate:
844 self.log.info(
"Decorrelation after template image convolution")
845 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
847 subtractRes.psfMatchingKernel,
848 spatiallyVarying=self.config.doSpatiallyVarying,
849 preConvKernel=preConvKernel)
851 self.log.info(
"Decorrelation after science image convolution")
852 decorrResult = self.decorrelate.
run(subtractRes.warpedExposure, exposureOrig,
854 subtractRes.psfMatchingKernel,
855 spatiallyVarying=self.config.doSpatiallyVarying,
856 preConvKernel=preConvKernel)
857 subtractedExposure = decorrResult.correctedExposure
861 if self.config.doDetection:
862 self.log.info(
"Running diaSource detection")
864 mask = subtractedExposure.getMaskedImage().getMask()
865 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
867 table = afwTable.SourceTable.make(self.schema, idFactory)
868 table.setMetadata(self.algMetadata)
869 results = self.detection.
run(
871 exposure=subtractedExposure,
872 doSmooth=
not self.config.doPreConvolve
875 if self.config.doMerge:
876 fpSet = results.fpSets.positive
877 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
878 self.config.growFootprint,
False)
879 diaSources = afwTable.SourceCatalog(table)
880 fpSet.makeSources(diaSources)
881 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
883 diaSources = results.sources
885 if self.config.doMeasurement:
886 newDipoleFitting = self.config.doDipoleFitting
887 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
888 if not newDipoleFitting:
890 self.measurement.
run(diaSources, subtractedExposure)
893 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
894 self.measurement.
run(diaSources, subtractedExposure, exposure,
895 subtractRes.matchedExposure)
897 self.measurement.
run(diaSources, subtractedExposure, exposure)
899 if self.config.doForcedMeasurement:
902 forcedSources = self.forcedMeasurement.generateMeasCat(
903 exposure, diaSources, subtractedExposure.getWcs())
904 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
905 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
906 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
907 "ip_diffim_forced_PsfFlux_instFlux",
True)
908 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
909 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
910 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
911 "ip_diffim_forced_PsfFlux_area",
True)
912 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
913 "ip_diffim_forced_PsfFlux_flag",
True)
914 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
915 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
916 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
917 "ip_diffim_forced_PsfFlux_flag_edge",
True)
918 for diaSource, forcedSource
in zip(diaSources, forcedSources):
919 diaSource.assign(forcedSource, mapper)
922 if self.config.doMatchSources:
923 if selectSources
is not None:
925 matchRadAsec = self.config.diaSourceMatchRadius
926 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
928 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
929 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
930 srcMatch
in srcMatches])
931 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
934 self.log.warn(
"Src product does not exist; cannot match with diaSources")
938 refAstromConfig = AstrometryConfig()
939 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
940 refAstrometer = AstrometryTask(refAstromConfig)
941 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
942 refMatches = astromRet.matches
943 if refMatches
is None:
944 self.log.warn(
"No diaSource matches with reference catalog")
947 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
949 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
950 refMatch
in refMatches])
953 for diaSource
in diaSources:
954 sid = diaSource.getId()
955 if sid
in srcMatchDict:
956 diaSource.set(
"srcMatchId", srcMatchDict[sid])
957 if sid
in refMatchDict:
958 diaSource.set(
"refMatchId", refMatchDict[sid])
960 if self.config.doAddMetrics
and self.config.doSelectSources:
961 self.log.info(
"Evaluating metrics and control sample")
964 for cell
in subtractRes.kernelCellSet.getCellList():
965 for cand
in cell.begin(
False):
966 kernelCandList.append(cand)
969 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
970 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
973 diffimTools.sourceTableToCandidateList(controlSources,
974 subtractRes.warpedExposure, exposure,
975 self.config.subtract.kernel.active,
976 self.config.subtract.kernel.active.detectionConfig,
977 self.log, doBuild=
True, basisList=basisList))
979 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
980 subtractRes.backgroundModel, dof=nparam)
981 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
982 subtractRes.backgroundModel)
984 if self.config.doDetection:
985 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
987 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
989 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
990 return pipeBase.Struct(
991 subtractedExposure=subtractedExposure,
992 warpedExposure=subtractRes.warpedExposure,
993 matchedExposure=subtractRes.matchedExposure,
994 subtractRes=subtractRes,
995 diaSources=diaSources,
996 selectSources=selectSources
999 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1000 """Fit the relative astrometry between templateSources and selectSources
1005 Remove this method. It originally fit a new WCS to the template before calling register.run
1006 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1007 It remains because a subtask overrides it.
1009 results = self.register.
run(templateSources, templateExposure.getWcs(),
1010 templateExposure.getBBox(), selectSources)
1013 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1014 """Make debug plots and displays.
1018 Test and update for current debug display and slot names
1028 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1029 if not maskTransparency:
1030 maskTransparency = 0
1031 disp.setMaskTransparency(maskTransparency)
1033 if display
and showSubtracted:
1034 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1035 mi = subtractRes.subtractedExposure.getMaskedImage()
1036 x0, y0 = mi.getX0(), mi.getY0()
1037 with disp.Buffering():
1038 for s
in diaSources:
1039 x, y = s.getX() - x0, s.getY() - y0
1040 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1041 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1042 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1043 or s.get(
"base_PixelFlags_flag_crCenter")):
1045 elif (s.get(
"base_PixelFlags_flag_interpolated")
1046 or s.get(
"base_PixelFlags_flag_saturated")
1047 or s.get(
"base_PixelFlags_flag_cr")):
1051 disp.dot(ptype, x, y, size=4, ctype=ctype)
1052 lsstDebug.frame += 1
1054 if display
and showPixelResiduals
and selectSources:
1055 nonKernelSources = []
1056 for source
in selectSources:
1057 if source
not in kernelSources:
1058 nonKernelSources.append(source)
1060 diUtils.plotPixelResiduals(exposure,
1061 subtractRes.warpedExposure,
1062 subtractRes.subtractedExposure,
1063 subtractRes.kernelCellSet,
1064 subtractRes.psfMatchingKernel,
1065 subtractRes.backgroundModel,
1067 self.subtract.config.kernel.active.detectionConfig,
1069 diUtils.plotPixelResiduals(exposure,
1070 subtractRes.warpedExposure,
1071 subtractRes.subtractedExposure,
1072 subtractRes.kernelCellSet,
1073 subtractRes.psfMatchingKernel,
1074 subtractRes.backgroundModel,
1076 self.subtract.config.kernel.active.detectionConfig,
1078 if display
and showDiaSources:
1079 flagChecker = SourceFlagChecker(diaSources)
1080 isFlagged = [flagChecker(x)
for x
in diaSources]
1081 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1082 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1083 frame=lsstDebug.frame)
1084 lsstDebug.frame += 1
1086 if display
and showDipoles:
1087 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1088 frame=lsstDebug.frame)
1089 lsstDebug.frame += 1
1091 def _getConfigName(self):
1092 """Return the name of the config dataset
1094 return "%sDiff_config" % (self.config.coaddName,)
1096 def _getMetadataName(self):
1097 """Return the name of the metadata dataset
1099 return "%sDiff_metadata" % (self.config.coaddName,)
1101 def getSchemaCatalogs(self):
1102 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1103 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1106 def _makeArgumentParser(cls):
1107 """Create an argument parser
1109 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1110 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1111 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1112 help=
"Template data ID in case of calexp template,"
1113 " e.g. --templateId visit=6789")
1117 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1118 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1119 doc=
"Shift stars going into RegisterTask by this amount")
1120 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1121 doc=
"Perturb stars going into RegisterTask by this amount")
1123 def setDefaults(self):
1124 ImageDifferenceConfig.setDefaults(self)
1125 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1128 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1129 """!Image difference Task used in the Winter 2013 data challege.
1130 Enables testing the effects of registration shifts and scatter.
1132 For use with winter 2013 simulated images:
1133 Use --templateId visit=88868666 for sparse data
1134 --templateId visit=22222200 for dense data (g)
1135 --templateId visit=11111100 for dense data (i)
1137 ConfigClass = Winter2013ImageDifferenceConfig
1138 _DefaultName =
"winter2013ImageDifference"
1140 def __init__(self, **kwargs):
1141 ImageDifferenceTask.__init__(self, **kwargs)
1143 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1144 """Fit the relative astrometry between templateSources and selectSources"""
1145 if self.config.winter2013WcsShift > 0.0:
1147 self.config.winter2013WcsShift)
1148 cKey = templateSources[0].getTable().getCentroidKey()
1149 for source
in templateSources:
1150 centroid = source.get(cKey)
1151 source.set(cKey, centroid + offset)
1152 elif self.config.winter2013WcsRms > 0.0:
1153 cKey = templateSources[0].getTable().getCentroidKey()
1154 for source
in templateSources:
1155 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1156 self.config.winter2013WcsRms*numpy.random.normal())
1157 centroid = source.get(cKey)
1158 source.set(cKey, centroid + offset)
1160 results = self.register.
run(templateSources, templateExposure.getWcs(),
1161 templateExposure.getBBox(), selectSources)