38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
48 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
54 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
55 defaultTemplates={
"coaddName":
"deep",
60 exposure = pipeBase.connectionTypes.Input(
61 doc=
"Input science exposure to subtract from.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}calexp"
75 skyMap = pipeBase.connectionTypes.Input(
76 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 dimensions=(
"skymap", ),
79 storageClass=
"SkyMap",
81 coaddExposures = pipeBase.connectionTypes.Input(
82 doc=
"Input template to match and subtract from the exposure",
83 dimensions=(
"tract",
"patch",
"skymap",
"band"),
84 storageClass=
"ExposureF",
85 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
89 dcrCoadds = pipeBase.connectionTypes.Input(
90 doc=
"Input DCR template to match and subtract from the exposure",
91 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
92 storageClass=
"ExposureF",
93 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
97 outputSchema = pipeBase.connectionTypes.InitOutput(
98 doc=
"Schema (as an example catalog) for output DIASource catalog.",
99 storageClass=
"SourceCatalog",
100 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
102 subtractedExposure = pipeBase.connectionTypes.Output(
103 doc=
"Output difference image",
104 dimensions=(
"instrument",
"visit",
"detector"),
105 storageClass=
"ExposureF",
106 name=
"{fakesType}{coaddName}Diff_differenceExp",
108 warpedExposure = pipeBase.connectionTypes.Output(
109 doc=
"Warped template used to create `subtractedExposure`.",
110 dimensions=(
"instrument",
"visit",
"detector"),
111 storageClass=
"ExposureF",
112 name=
"{fakesType}{coaddName}Diff_warpedExp",
114 diaSources = pipeBase.connectionTypes.Output(
115 doc=
"Output detected diaSources on the difference image",
116 dimensions=(
"instrument",
"visit",
"detector"),
117 storageClass=
"SourceCatalog",
118 name=
"{fakesType}{coaddName}Diff_diaSrc",
121 def __init__(self, *, config=None):
122 super().__init__(config=config)
123 if config.coaddName ==
'dcr':
124 self.inputs.remove(
"coaddExposures")
126 self.inputs.remove(
"dcrCoadds")
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133 pipelineConnections=ImageDifferenceTaskConnections):
134 """Config for ImageDifferenceTask
136 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
137 doc=
"Add background to calexp before processing it. "
138 "Useful as ipDiffim does background matching.")
139 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
140 doc=
"Use image-to-image registration to align template with "
142 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
143 doc=
"Writing debugging data for doUseRegister")
144 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
145 doc=
"Select stars to use for kernel fitting")
146 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
147 doc=
"Select stars of extreme color as part of the control sample")
148 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
149 doc=
"Select stars that are variable to be part "
150 "of the control sample")
151 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
152 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
153 doc=
"Convolve science image by its PSF before PSF-matching?")
154 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
155 doc=
"Scale variance of the template before PSF matching")
156 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=
False,
157 doc=
"Scale variance of the diffim before PSF matching. "
158 "You may do either this or template variance scaling, "
159 "or neither. (Doing both is a waste of CPU.)")
160 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
161 doc=
"Use a simple gaussian PSF model for pre-convolution "
162 "(else use fit PSF)? Ignored if doPreConvolve false.")
163 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
164 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
165 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
166 "kernel convolution? If True, also update the diffim PSF.")
167 doMerge = pexConfig.Field(dtype=bool, default=
True,
168 doc=
"Merge positive and negative diaSources with grow radius "
169 "set by growFootprint")
170 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
171 doc=
"Match diaSources with input calexp sources and ref catalog sources")
172 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
173 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
174 doForcedMeasurement = pexConfig.Field(
177 doc=
"Force photometer diaSource locations on PVI?")
178 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
179 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
180 doc=
"Write WCS, warped template coadd exposure?")
181 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
182 doc=
"Write warped and PSF-matched template coadd exposure?")
183 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
184 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
185 doc=
"Add columns to the source table to hold analysis metrics?")
187 coaddName = pexConfig.Field(
188 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
192 convolveTemplate = pexConfig.Field(
193 doc=
"Which image gets convolved (default = template)",
197 refObjLoader = pexConfig.ConfigurableField(
198 target=LoadIndexedReferenceObjectsTask,
199 doc=
"reference object loader",
201 astrometer = pexConfig.ConfigurableField(
202 target=AstrometryTask,
203 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
205 sourceSelector = pexConfig.ConfigurableField(
206 target=ObjectSizeStarSelectorTask,
207 doc=
"Source selection algorithm",
209 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
210 decorrelate = pexConfig.ConfigurableField(
211 target=DecorrelateALKernelSpatialTask,
212 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
216 doSpatiallyVarying = pexConfig.Field(
219 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
220 "image in order to allow for spatial variations"
222 detection = pexConfig.ConfigurableField(
223 target=SourceDetectionTask,
224 doc=
"Low-threshold detection for final measurement",
226 measurement = pexConfig.ConfigurableField(
227 target=DipoleFitTask,
228 doc=
"Enable updated dipole fitting method",
230 doApCorr = lsst.pex.config.Field(
233 doc=
"Run subtask to apply aperture corrections"
235 applyApCorr = lsst.pex.config.ConfigurableField(
236 target=ApplyApCorrTask,
237 doc=
"Subtask to apply aperture corrections"
239 forcedMeasurement = pexConfig.ConfigurableField(
240 target=ForcedMeasurementTask,
241 doc=
"Subtask to force photometer PVI at diaSource location.",
243 getTemplate = pexConfig.ConfigurableField(
244 target=GetCoaddAsTemplateTask,
245 doc=
"Subtask to retrieve template exposure and sources",
247 scaleVariance = pexConfig.ConfigurableField(
248 target=ScaleVarianceTask,
249 doc=
"Subtask to rescale the variance of the template "
250 "to the statistically expected level"
252 controlStepSize = pexConfig.Field(
253 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
257 controlRandomSeed = pexConfig.Field(
258 doc=
"Random seed for shuffing the control sample",
262 register = pexConfig.ConfigurableField(
264 doc=
"Task to enable image-to-image image registration (warping)",
266 kernelSourcesFromRef = pexConfig.Field(
267 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
271 templateSipOrder = pexConfig.Field(
272 dtype=int, default=2,
273 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
275 growFootprint = pexConfig.Field(
276 dtype=int, default=2,
277 doc=
"Grow positive and negative footprints by this amount before merging"
279 diaSourceMatchRadius = pexConfig.Field(
280 dtype=float, default=0.5,
281 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
284 def setDefaults(self):
287 self.subtract[
'al'].kernel.name =
"AL"
288 self.subtract[
'al'].kernel.active.fitForBackground =
True
289 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
290 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
291 self.doPreConvolve =
False
292 self.doMatchSources =
False
293 self.doAddMetrics =
False
294 self.doUseRegister =
False
297 self.detection.thresholdPolarity =
"both"
298 self.detection.thresholdValue = 5.5
299 self.detection.reEstimateBackground =
False
300 self.detection.thresholdType =
"pixel_stdev"
306 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
307 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
310 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
311 self.forcedMeasurement.copyColumns = {
312 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
313 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
314 self.forcedMeasurement.slots.shape =
None
317 random.seed(self.controlRandomSeed)
320 pexConfig.Config.validate(self)
321 if self.doAddMetrics
and not self.doSubtract:
322 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
323 if not self.doSubtract
and not self.doDetection:
324 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
325 if self.subtract.name ==
'zogy' and self.doAddMetrics:
326 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
327 if self.doMeasurement
and not self.doDetection:
328 raise ValueError(
"Cannot run source measurement without source detection.")
329 if self.doMerge
and not self.doDetection:
330 raise ValueError(
"Cannot run source merging without source detection.")
331 if self.doUseRegister
and not self.doSelectSources:
332 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
333 "Cannot run RegisterTask without selecting sources.")
334 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
335 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
336 "convolveTemplate=False is not supported.")
337 if hasattr(self.getTemplate,
"coaddName"):
338 if self.getTemplate.coaddName != self.coaddName:
339 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
340 if self.doScaleDiffimVariance
and self.doScaleTemplateVariance:
341 raise ValueError(
"Scaling the diffim variance and scaling the template variance "
342 "are both set. Please choose one or the other.")
345 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
348 def getTargetList(parsedCmd, **kwargs):
349 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
353 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
354 """Subtract an image from a template and measure the result
356 ConfigClass = ImageDifferenceConfig
357 RunnerClass = ImageDifferenceTaskRunner
358 _DefaultName =
"imageDifference"
360 def __init__(self, butler=None, **kwargs):
361 """!Construct an ImageDifference Task
363 @param[in] butler Butler object to use in constructing reference object loaders
365 super().__init__(**kwargs)
366 self.makeSubtask(
"getTemplate")
368 self.makeSubtask(
"subtract")
370 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
371 self.makeSubtask(
"decorrelate")
373 if self.config.doScaleTemplateVariance
or self.config.doScaleDiffimVariance:
374 self.makeSubtask(
"scaleVariance")
376 if self.config.doUseRegister:
377 self.makeSubtask(
"register")
378 self.schema = afwTable.SourceTable.makeMinimalSchema()
380 if self.config.doSelectSources:
381 self.makeSubtask(
"sourceSelector")
382 if self.config.kernelSourcesFromRef:
383 self.makeSubtask(
'refObjLoader', butler=butler)
384 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
386 self.algMetadata = dafBase.PropertyList()
387 if self.config.doDetection:
388 self.makeSubtask(
"detection", schema=self.schema)
389 if self.config.doMeasurement:
390 self.makeSubtask(
"measurement", schema=self.schema,
391 algMetadata=self.algMetadata)
392 if self.config.doApCorr:
393 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
394 if self.config.doForcedMeasurement:
395 self.schema.addField(
396 "ip_diffim_forced_PsfFlux_instFlux",
"D",
397 "Forced PSF flux measured on the direct image.",
399 self.schema.addField(
400 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
401 "Forced PSF flux error measured on the direct image.",
403 self.schema.addField(
404 "ip_diffim_forced_PsfFlux_area",
"F",
405 "Forced PSF flux effective area of PSF.",
407 self.schema.addField(
408 "ip_diffim_forced_PsfFlux_flag",
"Flag",
409 "Forced PSF flux general failure flag.")
410 self.schema.addField(
411 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
412 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
413 self.schema.addField(
414 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
415 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
416 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
417 if self.config.doMatchSources:
418 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
419 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
422 self.outputSchema = afwTable.SourceCatalog(self.schema)
423 self.outputSchema.getTable().setMetadata(self.algMetadata)
426 def makeIdFactory(expId, expBits):
427 """Create IdFactory instance for unique 64 bit diaSource id-s.
435 Number of used bits in ``expId``.
439 The diasource id-s consists of the ``expId`` stored fixed in the highest value
440 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
441 low value end of the integer.
445 idFactory: `lsst.afw.table.IdFactory`
447 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
449 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
450 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
451 inputRefs: pipeBase.InputQuantizedConnection,
452 outputRefs: pipeBase.OutputQuantizedConnection):
453 inputs = butlerQC.get(inputRefs)
454 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
455 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
457 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
458 if self.config.coaddName ==
'dcr':
459 templateExposures = inputRefs.dcrCoadds
461 templateExposures = inputRefs.coaddExposures
462 templateStruct = self.getTemplate.runQuantum(
463 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
466 outputs = self.run(exposure=inputs[
'exposure'],
467 templateExposure=templateStruct.exposure,
469 butlerQC.put(outputs, outputRefs)
472 def runDataRef(self, sensorRef, templateIdList=None):
473 """Subtract an image from a template coadd and measure the result.
475 Data I/O wrapper around `run` using the butler in Gen2.
479 sensorRef : `lsst.daf.persistence.ButlerDataRef`
480 Sensor-level butler data reference, used for the following data products:
487 - self.config.coaddName + "Coadd_skyMap"
488 - self.config.coaddName + "Coadd"
489 Input or output, depending on config:
490 - self.config.coaddName + "Diff_subtractedExp"
491 Output, depending on config:
492 - self.config.coaddName + "Diff_matchedExp"
493 - self.config.coaddName + "Diff_src"
497 results : `lsst.pipe.base.Struct`
498 Returns the Struct by `run`.
500 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
501 subtractedExposure =
None
503 calexpBackgroundExposure =
None
504 self.log.info(
"Processing %s" % (sensorRef.dataId))
509 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
510 expBits=sensorRef.get(
"ccdExposureId_bits"))
511 if self.config.doAddCalexpBackground:
512 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
515 exposure = sensorRef.get(
"calexp", immediate=
True)
518 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
520 if sensorRef.datasetExists(
"src"):
521 self.log.info(
"Source selection via src product")
523 selectSources = sensorRef.get(
"src")
525 if not self.config.doSubtract
and self.config.doDetection:
527 subtractedExposure = sensorRef.get(subtractedExposureName)
530 results = self.run(exposure=exposure,
531 selectSources=selectSources,
532 templateExposure=template.exposure,
533 templateSources=template.sources,
535 calexpBackgroundExposure=calexpBackgroundExposure,
536 subtractedExposure=subtractedExposure)
538 if self.config.doWriteSources
and results.diaSources
is not None:
539 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
540 if self.config.doWriteWarpedExp:
541 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
542 if self.config.doWriteMatchedExp:
543 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
544 if self.config.doAddMetrics
and self.config.doSelectSources:
545 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
546 if self.config.doWriteSubtractedExp:
547 sensorRef.put(results.subtractedExposure, subtractedExposureName)
551 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
552 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
553 """PSF matches, subtract two images and perform detection on the difference image.
557 exposure : `lsst.afw.image.ExposureF`, optional
558 The science exposure, the minuend in the image subtraction.
559 Can be None only if ``config.doSubtract==False``.
560 selectSources : `lsst.afw.table.SourceCatalog`, optional
561 Identified sources on the science exposure. This catalog is used to
562 select sources in order to perform the AL PSF matching on stamp images
563 around them. The selection steps depend on config options and whether
564 ``templateSources`` and ``matchingSources`` specified.
565 templateExposure : `lsst.afw.image.ExposureF`, optional
566 The template to be subtracted from ``exposure`` in the image subtraction.
567 The template exposure should cover the same sky area as the science exposure.
568 It is either a stich of patches of a coadd skymap image or a calexp
569 of the same pointing as the science exposure. Can be None only
570 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
571 templateSources : `lsst.afw.table.SourceCatalog`, optional
572 Identified sources on the template exposure.
573 idFactory : `lsst.afw.table.IdFactory`
574 Generator object to assign ids to detected sources in the difference image.
575 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
576 Background exposure to be added back to the science exposure
577 if ``config.doAddCalexpBackground==True``
578 subtractedExposure : `lsst.afw.image.ExposureF`, optional
579 If ``config.doSubtract==False`` and ``config.doDetection==True``,
580 performs the post subtraction source detection only on this exposure.
581 Otherwise should be None.
585 results : `lsst.pipe.base.Struct`
586 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
588 ``matchedExposure`` : `lsst.afw.image.ExposureF`
589 The matched PSF exposure.
590 ``subtractRes`` : `lsst.pipe.base.Struct`
591 The returned result structure of the ImagePsfMatchTask subtask.
592 ``diaSources`` : `lsst.afw.table.SourceCatalog`
593 The catalog of detected sources.
594 ``selectSources`` : `lsst.afw.table.SourceCatalog`
595 The input source catalog with optionally added Qa information.
599 The following major steps are included:
601 - warp template coadd to match WCS of image
602 - PSF match image to warped template
603 - subtract image from PSF-matched, warped template
607 For details about the image subtraction configuration modes
608 see `lsst.ip.diffim`.
611 controlSources =
None
615 if self.config.doAddCalexpBackground:
616 mi = exposure.getMaskedImage()
617 mi += calexpBackgroundExposure.getImage()
619 if not exposure.hasPsf():
620 raise pipeBase.TaskError(
"Exposure has no psf")
621 sciencePsf = exposure.getPsf()
623 if self.config.doSubtract:
624 if self.config.doScaleTemplateVariance:
625 self.log.info(
"Rescaling template variance")
626 templateVarFactor = self.scaleVariance.
run(
627 templateExposure.getMaskedImage())
628 self.log.info(
"Template variance scaling factor: %.2f" % templateVarFactor)
629 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
631 if self.config.subtract.name ==
'zogy':
632 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
633 if self.config.doPreConvolve:
634 subtractedExposure = subtractRes.scoreExp
636 subtractedExposure = subtractRes.diffExp
637 subtractRes.subtractedExposure = subtractedExposure
638 subtractRes.matchedExposure =
None
640 elif self.config.subtract.name ==
'al':
642 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
643 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
651 if self.config.doPreConvolve:
652 convControl = afwMath.ConvolutionControl()
654 srcMI = exposure.getMaskedImage()
655 exposureOrig = exposure.clone()
656 destMI = srcMI.Factory(srcMI.getDimensions())
658 if self.config.useGaussianForPreConvolution:
660 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
665 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
666 exposure.setMaskedImage(destMI)
667 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
669 scienceSigmaPost = scienceSigmaOrig
670 exposureOrig = exposure
675 if self.config.doSelectSources:
676 if selectSources
is None:
677 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
679 selectSources = self.subtract.getSelectSources(
681 sigma=scienceSigmaPost,
682 doSmooth=
not self.config.doPreConvolve,
686 if self.config.doAddMetrics:
689 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
690 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
691 targetFwhmPix=templateSigma*FwhmPerSigma))
698 kcQa = KernelCandidateQa(nparam)
699 selectSources = kcQa.addToSchema(selectSources)
700 if self.config.kernelSourcesFromRef:
702 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
703 matches = astromRet.matches
704 elif templateSources:
706 mc = afwTable.MatchControl()
707 mc.findOnlyClosest =
False
708 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
711 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
712 "but template sources not available. Cannot match science "
713 "sources with template sources. Run process* on data from "
714 "which templates are built.")
716 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
717 matches=matches).sourceCat
718 random.shuffle(kernelSources, random.random)
719 controlSources = kernelSources[::self.config.controlStepSize]
720 kernelSources = [k
for i, k
in enumerate(kernelSources)
721 if i % self.config.controlStepSize]
723 if self.config.doSelectDcrCatalog:
724 redSelector = DiaCatalogSourceSelectorTask(
725 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
727 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
728 controlSources.extend(redSources)
730 blueSelector = DiaCatalogSourceSelectorTask(
731 DiaCatalogSourceSelectorConfig(grMin=-99.999,
732 grMax=self.sourceSelector.config.grMin))
733 blueSources = blueSelector.selectStars(exposure, selectSources,
734 matches=matches).starCat
735 controlSources.extend(blueSources)
737 if self.config.doSelectVariableCatalog:
738 varSelector = DiaCatalogSourceSelectorTask(
739 DiaCatalogSourceSelectorConfig(includeVariable=
True))
740 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
741 controlSources.extend(varSources)
743 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
744 % (len(kernelSources), len(selectSources), len(controlSources)))
748 if self.config.doUseRegister:
749 self.log.info(
"Registering images")
751 if templateSources
is None:
755 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
756 templateSources = self.subtract.getSelectSources(
765 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
766 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
767 exposure.getWcs(), exposure.getBBox())
768 templateExposure = warpedExp
773 if self.config.doDebugRegister:
775 srcToMatch = {x.second.getId(): x.first
for x
in matches}
777 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
778 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
779 sids = [m.first.getId()
for m
in wcsResults.matches]
780 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
781 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
782 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
783 allresids = dict(zip(sids, zip(positions, residuals)))
785 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
786 wcsResults.wcs.pixelToSky(
787 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
788 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
789 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
790 for x
in sids
if x
in srcToMatch.keys()])
791 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
792 if s
in srcToMatch.keys()])
793 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
794 if s
in srcToMatch.keys()])
795 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
796 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
797 & (colors <= self.sourceSelector.config.grMax))
798 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
799 rms1Long = IqrToSigma*(
800 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
801 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
802 - numpy.percentile(dlat[idx1], 25))
803 rms2Long = IqrToSigma*(
804 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
805 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
806 - numpy.percentile(dlat[idx2], 25))
807 rms3Long = IqrToSigma*(
808 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
809 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
810 - numpy.percentile(dlat[idx3], 25))
811 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
812 (numpy.median(dlong[idx1]), rms1Long,
813 numpy.median(dlat[idx1]), rms1Lat))
814 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
815 (numpy.median(dlong[idx2]), rms2Long,
816 numpy.median(dlat[idx2]), rms2Lat))
817 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
818 (numpy.median(dlong[idx3]), rms3Long,
819 numpy.median(dlat[idx3]), rms3Lat))
821 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
822 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
823 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
824 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
825 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
826 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
828 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
829 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
830 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
831 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
832 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
833 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
840 self.log.info(
"Subtracting images")
841 subtractRes = self.subtract.subtractExposures(
842 templateExposure=templateExposure,
843 scienceExposure=exposure,
844 candidateList=kernelSources,
845 convolveTemplate=self.config.convolveTemplate,
846 doWarping=
not self.config.doUseRegister
848 subtractedExposure = subtractRes.subtractedExposure
850 if self.config.doDetection:
851 self.log.info(
"Computing diffim PSF")
854 if not subtractedExposure.hasPsf():
855 if self.config.convolveTemplate:
856 subtractedExposure.setPsf(exposure.getPsf())
858 subtractedExposure.setPsf(templateExposure.getPsf())
865 if self.config.doDecorrelation
and self.config.doSubtract:
867 if preConvPsf
is not None:
868 preConvKernel = preConvPsf.getLocalKernel()
869 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
871 subtractRes.psfMatchingKernel,
872 spatiallyVarying=self.config.doSpatiallyVarying,
873 preConvKernel=preConvKernel,
874 templateMatched=self.config.convolveTemplate)
875 subtractedExposure = decorrResult.correctedExposure
879 if self.config.doDetection:
880 self.log.info(
"Running diaSource detection")
883 if self.config.doScaleDiffimVariance:
884 self.log.info(
"Rescaling diffim variance")
885 diffimVarFactor = self.scaleVariance.
run(subtractedExposure.getMaskedImage())
886 self.log.info(
"Diffim variance scaling factor: %.2f" % diffimVarFactor)
887 self.metadata.add(
"scaleDiffimVarianceFactor", diffimVarFactor)
890 mask = subtractedExposure.getMaskedImage().getMask()
891 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
893 table = afwTable.SourceTable.make(self.schema, idFactory)
894 table.setMetadata(self.algMetadata)
895 results = self.detection.
run(
897 exposure=subtractedExposure,
898 doSmooth=
not self.config.doPreConvolve
901 if self.config.doMerge:
902 fpSet = results.fpSets.positive
903 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
904 self.config.growFootprint,
False)
905 diaSources = afwTable.SourceCatalog(table)
906 fpSet.makeSources(diaSources)
907 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
909 diaSources = results.sources
911 if self.config.doMeasurement:
912 newDipoleFitting = self.config.doDipoleFitting
913 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
914 if not newDipoleFitting:
916 self.measurement.
run(diaSources, subtractedExposure)
919 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
920 self.measurement.
run(diaSources, subtractedExposure, exposure,
921 subtractRes.matchedExposure)
923 self.measurement.
run(diaSources, subtractedExposure, exposure)
924 if self.config.doApCorr:
925 self.applyApCorr.
run(
927 apCorrMap=subtractedExposure.getInfo().getApCorrMap()
930 if self.config.doForcedMeasurement:
933 forcedSources = self.forcedMeasurement.generateMeasCat(
934 exposure, diaSources, subtractedExposure.getWcs())
935 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
936 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
937 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
938 "ip_diffim_forced_PsfFlux_instFlux",
True)
939 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
940 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
941 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
942 "ip_diffim_forced_PsfFlux_area",
True)
943 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
944 "ip_diffim_forced_PsfFlux_flag",
True)
945 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
946 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
947 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
948 "ip_diffim_forced_PsfFlux_flag_edge",
True)
949 for diaSource, forcedSource
in zip(diaSources, forcedSources):
950 diaSource.assign(forcedSource, mapper)
953 if self.config.doMatchSources:
954 if selectSources
is not None:
956 matchRadAsec = self.config.diaSourceMatchRadius
957 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
959 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
960 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
961 srcMatch
in srcMatches])
962 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
965 self.log.warn(
"Src product does not exist; cannot match with diaSources")
969 refAstromConfig = AstrometryConfig()
970 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
971 refAstrometer = AstrometryTask(refAstromConfig)
972 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
973 refMatches = astromRet.matches
974 if refMatches
is None:
975 self.log.warn(
"No diaSource matches with reference catalog")
978 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
980 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
981 refMatch
in refMatches])
984 for diaSource
in diaSources:
985 sid = diaSource.getId()
986 if sid
in srcMatchDict:
987 diaSource.set(
"srcMatchId", srcMatchDict[sid])
988 if sid
in refMatchDict:
989 diaSource.set(
"refMatchId", refMatchDict[sid])
991 if self.config.doAddMetrics
and self.config.doSelectSources:
992 self.log.info(
"Evaluating metrics and control sample")
995 for cell
in subtractRes.kernelCellSet.getCellList():
996 for cand
in cell.begin(
False):
997 kernelCandList.append(cand)
1000 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1001 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1004 diffimTools.sourceTableToCandidateList(controlSources,
1005 subtractRes.warpedExposure, exposure,
1006 self.config.subtract.kernel.active,
1007 self.config.subtract.kernel.active.detectionConfig,
1008 self.log, doBuild=
True, basisList=basisList))
1010 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1011 subtractRes.backgroundModel, dof=nparam)
1012 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1013 subtractRes.backgroundModel)
1015 if self.config.doDetection:
1016 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1018 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1020 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1021 return pipeBase.Struct(
1022 subtractedExposure=subtractedExposure,
1023 warpedExposure=subtractRes.warpedExposure,
1024 matchedExposure=subtractRes.matchedExposure,
1025 subtractRes=subtractRes,
1026 diaSources=diaSources,
1027 selectSources=selectSources
1030 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1031 """Fit the relative astrometry between templateSources and selectSources
1036 Remove this method. It originally fit a new WCS to the template before calling register.run
1037 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1038 It remains because a subtask overrides it.
1040 results = self.register.
run(templateSources, templateExposure.getWcs(),
1041 templateExposure.getBBox(), selectSources)
1044 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1045 """Make debug plots and displays.
1049 Test and update for current debug display and slot names
1059 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1060 if not maskTransparency:
1061 maskTransparency = 0
1062 disp.setMaskTransparency(maskTransparency)
1064 if display
and showSubtracted:
1065 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1066 mi = subtractRes.subtractedExposure.getMaskedImage()
1067 x0, y0 = mi.getX0(), mi.getY0()
1068 with disp.Buffering():
1069 for s
in diaSources:
1070 x, y = s.getX() - x0, s.getY() - y0
1071 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1072 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1073 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1074 or s.get(
"base_PixelFlags_flag_crCenter")):
1076 elif (s.get(
"base_PixelFlags_flag_interpolated")
1077 or s.get(
"base_PixelFlags_flag_saturated")
1078 or s.get(
"base_PixelFlags_flag_cr")):
1082 disp.dot(ptype, x, y, size=4, ctype=ctype)
1083 lsstDebug.frame += 1
1085 if display
and showPixelResiduals
and selectSources:
1086 nonKernelSources = []
1087 for source
in selectSources:
1088 if source
not in kernelSources:
1089 nonKernelSources.append(source)
1091 diUtils.plotPixelResiduals(exposure,
1092 subtractRes.warpedExposure,
1093 subtractRes.subtractedExposure,
1094 subtractRes.kernelCellSet,
1095 subtractRes.psfMatchingKernel,
1096 subtractRes.backgroundModel,
1098 self.subtract.config.kernel.active.detectionConfig,
1100 diUtils.plotPixelResiduals(exposure,
1101 subtractRes.warpedExposure,
1102 subtractRes.subtractedExposure,
1103 subtractRes.kernelCellSet,
1104 subtractRes.psfMatchingKernel,
1105 subtractRes.backgroundModel,
1107 self.subtract.config.kernel.active.detectionConfig,
1109 if display
and showDiaSources:
1110 flagChecker = SourceFlagChecker(diaSources)
1111 isFlagged = [flagChecker(x)
for x
in diaSources]
1112 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1113 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1114 frame=lsstDebug.frame)
1115 lsstDebug.frame += 1
1117 if display
and showDipoles:
1118 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1119 frame=lsstDebug.frame)
1120 lsstDebug.frame += 1
1122 def _getConfigName(self):
1123 """Return the name of the config dataset
1125 return "%sDiff_config" % (self.config.coaddName,)
1127 def _getMetadataName(self):
1128 """Return the name of the metadata dataset
1130 return "%sDiff_metadata" % (self.config.coaddName,)
1132 def getSchemaCatalogs(self):
1133 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1134 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1137 def _makeArgumentParser(cls):
1138 """Create an argument parser
1140 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1141 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1142 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1143 help=
"Template data ID in case of calexp template,"
1144 " e.g. --templateId visit=6789")
1148 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1149 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1150 doc=
"Shift stars going into RegisterTask by this amount")
1151 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1152 doc=
"Perturb stars going into RegisterTask by this amount")
1154 def setDefaults(self):
1155 ImageDifferenceConfig.setDefaults(self)
1156 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1159 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1160 """!Image difference Task used in the Winter 2013 data challege.
1161 Enables testing the effects of registration shifts and scatter.
1163 For use with winter 2013 simulated images:
1164 Use --templateId visit=88868666 for sparse data
1165 --templateId visit=22222200 for dense data (g)
1166 --templateId visit=11111100 for dense data (i)
1168 ConfigClass = Winter2013ImageDifferenceConfig
1169 _DefaultName =
"winter2013ImageDifference"
1171 def __init__(self, **kwargs):
1172 ImageDifferenceTask.__init__(self, **kwargs)
1174 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1175 """Fit the relative astrometry between templateSources and selectSources"""
1176 if self.config.winter2013WcsShift > 0.0:
1178 self.config.winter2013WcsShift)
1179 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1180 for source
in templateSources:
1181 centroid = source.get(cKey)
1182 source.set(cKey, centroid + offset)
1183 elif self.config.winter2013WcsRms > 0.0:
1184 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1185 for source
in templateSources:
1186 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1187 self.config.winter2013WcsRms*numpy.random.normal())
1188 centroid = source.get(cKey)
1189 source.set(cKey, centroid + offset)
1191 results = self.register.
run(templateSources, templateExposure.getWcs(),
1192 templateExposure.getBBox(), selectSources)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)