38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
48 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
54 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
55 defaultTemplates={
"coaddName":
"deep",
60 exposure = pipeBase.connectionTypes.Input(
61 doc=
"Input science exposure to subtract from.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}calexp"
75 skyMap = pipeBase.connectionTypes.Input(
76 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 dimensions=(
"skymap", ),
79 storageClass=
"SkyMap",
81 coaddExposures = pipeBase.connectionTypes.Input(
82 doc=
"Input template to match and subtract from the exposure",
83 dimensions=(
"tract",
"patch",
"skymap",
"band"),
84 storageClass=
"ExposureF",
85 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
89 dcrCoadds = pipeBase.connectionTypes.Input(
90 doc=
"Input DCR template to match and subtract from the exposure",
91 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
92 storageClass=
"ExposureF",
93 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
97 outputSchema = pipeBase.connectionTypes.InitOutput(
98 doc=
"Schema (as an example catalog) for output DIASource catalog.",
99 storageClass=
"SourceCatalog",
100 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
102 subtractedExposure = pipeBase.connectionTypes.Output(
103 doc=
"Output difference image",
104 dimensions=(
"instrument",
"visit",
"detector"),
105 storageClass=
"ExposureF",
106 name=
"{fakesType}{coaddName}Diff_differenceExp",
108 warpedExposure = pipeBase.connectionTypes.Output(
109 doc=
"Warped template used to create `subtractedExposure`.",
110 dimensions=(
"instrument",
"visit",
"detector"),
111 storageClass=
"ExposureF",
112 name=
"{fakesType}{coaddName}Diff_warpedExp",
114 diaSources = pipeBase.connectionTypes.Output(
115 doc=
"Output detected diaSources on the difference image",
116 dimensions=(
"instrument",
"visit",
"detector"),
117 storageClass=
"SourceCatalog",
118 name=
"{fakesType}{coaddName}Diff_diaSrc",
121 def __init__(self, *, config=None):
122 super().__init__(config=config)
123 if config.coaddName ==
'dcr':
124 self.inputs.remove(
"coaddExposures")
126 self.inputs.remove(
"dcrCoadds")
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133 pipelineConnections=ImageDifferenceTaskConnections):
134 """Config for ImageDifferenceTask
136 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
137 doc=
"Add background to calexp before processing it. "
138 "Useful as ipDiffim does background matching.")
139 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
140 doc=
"Use image-to-image registration to align template with "
142 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
143 doc=
"Writing debugging data for doUseRegister")
144 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
145 doc=
"Select stars to use for kernel fitting")
146 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
147 doc=
"Select stars of extreme color as part of the control sample")
148 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
149 doc=
"Select stars that are variable to be part "
150 "of the control sample")
151 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
152 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
153 doc=
"Convolve science image by its PSF before PSF-matching?")
154 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
155 doc=
"Scale variance of the template before PSF matching")
156 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=
False,
157 doc=
"Scale variance of the diffim before PSF matching. "
158 "You may do either this or template variance scaling, "
159 "or neither. (Doing both is a waste of CPU.)")
160 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
161 doc=
"Use a simple gaussian PSF model for pre-convolution "
162 "(else use fit PSF)? Ignored if doPreConvolve false.")
163 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
164 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
165 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
166 "kernel convolution? If True, also update the diffim PSF.")
167 doMerge = pexConfig.Field(dtype=bool, default=
True,
168 doc=
"Merge positive and negative diaSources with grow radius "
169 "set by growFootprint")
170 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
171 doc=
"Match diaSources with input calexp sources and ref catalog sources")
172 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
173 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
174 doForcedMeasurement = pexConfig.Field(
177 doc=
"Force photometer diaSource locations on PVI?")
178 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
179 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
180 doc=
"Write WCS, warped template coadd exposure?")
181 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
182 doc=
"Write warped and PSF-matched template coadd exposure?")
183 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
184 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
185 doc=
"Add columns to the source table to hold analysis metrics?")
187 coaddName = pexConfig.Field(
188 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
192 convolveTemplate = pexConfig.Field(
193 doc=
"Which image gets convolved (default = template)",
197 refObjLoader = pexConfig.ConfigurableField(
198 target=LoadIndexedReferenceObjectsTask,
199 doc=
"reference object loader",
201 astrometer = pexConfig.ConfigurableField(
202 target=AstrometryTask,
203 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
205 sourceSelector = pexConfig.ConfigurableField(
206 target=ObjectSizeStarSelectorTask,
207 doc=
"Source selection algorithm",
209 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
210 decorrelate = pexConfig.ConfigurableField(
211 target=DecorrelateALKernelSpatialTask,
212 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
216 doSpatiallyVarying = pexConfig.Field(
219 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
220 "image in order to allow for spatial variations"
222 detection = pexConfig.ConfigurableField(
223 target=SourceDetectionTask,
224 doc=
"Low-threshold detection for final measurement",
226 measurement = pexConfig.ConfigurableField(
227 target=DipoleFitTask,
228 doc=
"Enable updated dipole fitting method",
230 forcedMeasurement = pexConfig.ConfigurableField(
231 target=ForcedMeasurementTask,
232 doc=
"Subtask to force photometer PVI at diaSource location.",
234 getTemplate = pexConfig.ConfigurableField(
235 target=GetCoaddAsTemplateTask,
236 doc=
"Subtask to retrieve template exposure and sources",
238 scaleVariance = pexConfig.ConfigurableField(
239 target=ScaleVarianceTask,
240 doc=
"Subtask to rescale the variance of the template "
241 "to the statistically expected level"
243 controlStepSize = pexConfig.Field(
244 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
248 controlRandomSeed = pexConfig.Field(
249 doc=
"Random seed for shuffing the control sample",
253 register = pexConfig.ConfigurableField(
255 doc=
"Task to enable image-to-image image registration (warping)",
257 kernelSourcesFromRef = pexConfig.Field(
258 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
262 templateSipOrder = pexConfig.Field(
263 dtype=int, default=2,
264 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
266 growFootprint = pexConfig.Field(
267 dtype=int, default=2,
268 doc=
"Grow positive and negative footprints by this amount before merging"
270 diaSourceMatchRadius = pexConfig.Field(
271 dtype=float, default=0.5,
272 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
275 def setDefaults(self):
278 self.subtract[
'al'].kernel.name =
"AL"
279 self.subtract[
'al'].kernel.active.fitForBackground =
True
280 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
281 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
282 self.doPreConvolve =
False
283 self.doMatchSources =
False
284 self.doAddMetrics =
False
285 self.doUseRegister =
False
288 self.detection.thresholdPolarity =
"both"
289 self.detection.thresholdValue = 5.5
290 self.detection.reEstimateBackground =
False
291 self.detection.thresholdType =
"pixel_stdev"
297 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
298 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
301 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
302 self.forcedMeasurement.copyColumns = {
303 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
304 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
305 self.forcedMeasurement.slots.shape =
None
308 random.seed(self.controlRandomSeed)
311 pexConfig.Config.validate(self)
312 if self.doAddMetrics
and not self.doSubtract:
313 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
314 if not self.doSubtract
and not self.doDetection:
315 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
316 if self.subtract.name ==
'zogy' and self.doAddMetrics:
317 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
318 if self.doMeasurement
and not self.doDetection:
319 raise ValueError(
"Cannot run source measurement without source detection.")
320 if self.doMerge
and not self.doDetection:
321 raise ValueError(
"Cannot run source merging without source detection.")
322 if self.doUseRegister
and not self.doSelectSources:
323 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
324 "Cannot run RegisterTask without selecting sources.")
325 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
326 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
327 "convolveTemplate=False is not supported.")
328 if hasattr(self.getTemplate,
"coaddName"):
329 if self.getTemplate.coaddName != self.coaddName:
330 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
331 if self.doScaleDiffimVariance
and self.doScaleTemplateVariance:
332 raise ValueError(
"Scaling the diffim variance and scaling the template variance "
333 "are both set. Please choose one or the other.")
336 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
339 def getTargetList(parsedCmd, **kwargs):
340 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
344 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
345 """Subtract an image from a template and measure the result
347 ConfigClass = ImageDifferenceConfig
348 RunnerClass = ImageDifferenceTaskRunner
349 _DefaultName =
"imageDifference"
351 def __init__(self, butler=None, **kwargs):
352 """!Construct an ImageDifference Task
354 @param[in] butler Butler object to use in constructing reference object loaders
356 super().__init__(**kwargs)
357 self.makeSubtask(
"getTemplate")
359 self.makeSubtask(
"subtract")
361 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
362 self.makeSubtask(
"decorrelate")
364 if self.config.doScaleTemplateVariance
or self.config.doScaleDiffimVariance:
365 self.makeSubtask(
"scaleVariance")
367 if self.config.doUseRegister:
368 self.makeSubtask(
"register")
369 self.schema = afwTable.SourceTable.makeMinimalSchema()
371 if self.config.doSelectSources:
372 self.makeSubtask(
"sourceSelector")
373 if self.config.kernelSourcesFromRef:
374 self.makeSubtask(
'refObjLoader', butler=butler)
375 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
377 self.algMetadata = dafBase.PropertyList()
378 if self.config.doDetection:
379 self.makeSubtask(
"detection", schema=self.schema)
380 if self.config.doMeasurement:
381 self.makeSubtask(
"measurement", schema=self.schema,
382 algMetadata=self.algMetadata)
383 if self.config.doForcedMeasurement:
384 self.schema.addField(
385 "ip_diffim_forced_PsfFlux_instFlux",
"D",
386 "Forced PSF flux measured on the direct image.")
387 self.schema.addField(
388 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
389 "Forced PSF flux error measured on the direct image.")
390 self.schema.addField(
391 "ip_diffim_forced_PsfFlux_area",
"F",
392 "Forced PSF flux effective area of PSF.",
394 self.schema.addField(
395 "ip_diffim_forced_PsfFlux_flag",
"Flag",
396 "Forced PSF flux general failure flag.")
397 self.schema.addField(
398 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
399 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
400 self.schema.addField(
401 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
402 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
403 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
404 if self.config.doMatchSources:
405 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
406 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
409 self.outputSchema = afwTable.SourceCatalog(self.schema)
410 self.outputSchema.getTable().setMetadata(self.algMetadata)
413 def makeIdFactory(expId, expBits):
414 """Create IdFactory instance for unique 64 bit diaSource id-s.
422 Number of used bits in ``expId``.
426 The diasource id-s consists of the ``expId`` stored fixed in the highest value
427 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
428 low value end of the integer.
432 idFactory: `lsst.afw.table.IdFactory`
434 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
437 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
438 inputRefs: pipeBase.InputQuantizedConnection,
439 outputRefs: pipeBase.OutputQuantizedConnection):
440 inputs = butlerQC.get(inputRefs)
441 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
442 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
444 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
445 if self.config.coaddName ==
'dcr':
446 templateExposures = inputRefs.dcrCoadds
448 templateExposures = inputRefs.coaddExposures
449 templateStruct = self.getTemplate.runQuantum(
450 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
453 outputs = self.run(exposure=inputs[
'exposure'],
454 templateExposure=templateStruct.exposure,
456 butlerQC.put(outputs, outputRefs)
459 def runDataRef(self, sensorRef, templateIdList=None):
460 """Subtract an image from a template coadd and measure the result.
462 Data I/O wrapper around `run` using the butler in Gen2.
466 sensorRef : `lsst.daf.persistence.ButlerDataRef`
467 Sensor-level butler data reference, used for the following data products:
474 - self.config.coaddName + "Coadd_skyMap"
475 - self.config.coaddName + "Coadd"
476 Input or output, depending on config:
477 - self.config.coaddName + "Diff_subtractedExp"
478 Output, depending on config:
479 - self.config.coaddName + "Diff_matchedExp"
480 - self.config.coaddName + "Diff_src"
484 results : `lsst.pipe.base.Struct`
485 Returns the Struct by `run`.
487 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
488 subtractedExposure =
None
490 calexpBackgroundExposure =
None
491 self.log.info(
"Processing %s" % (sensorRef.dataId))
496 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
497 expBits=sensorRef.get(
"ccdExposureId_bits"))
498 if self.config.doAddCalexpBackground:
499 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
502 exposure = sensorRef.get(
"calexp", immediate=
True)
505 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
507 if sensorRef.datasetExists(
"src"):
508 self.log.info(
"Source selection via src product")
510 selectSources = sensorRef.get(
"src")
512 if not self.config.doSubtract
and self.config.doDetection:
514 subtractedExposure = sensorRef.get(subtractedExposureName)
517 results = self.run(exposure=exposure,
518 selectSources=selectSources,
519 templateExposure=template.exposure,
520 templateSources=template.sources,
522 calexpBackgroundExposure=calexpBackgroundExposure,
523 subtractedExposure=subtractedExposure)
525 if self.config.doWriteSources
and results.diaSources
is not None:
526 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
527 if self.config.doWriteWarpedExp:
528 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
529 if self.config.doWriteMatchedExp:
530 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
531 if self.config.doAddMetrics
and self.config.doSelectSources:
532 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
533 if self.config.doWriteSubtractedExp:
534 sensorRef.put(results.subtractedExposure, subtractedExposureName)
538 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
539 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
540 """PSF matches, subtract two images and perform detection on the difference image.
544 exposure : `lsst.afw.image.ExposureF`, optional
545 The science exposure, the minuend in the image subtraction.
546 Can be None only if ``config.doSubtract==False``.
547 selectSources : `lsst.afw.table.SourceCatalog`, optional
548 Identified sources on the science exposure. This catalog is used to
549 select sources in order to perform the AL PSF matching on stamp images
550 around them. The selection steps depend on config options and whether
551 ``templateSources`` and ``matchingSources`` specified.
552 templateExposure : `lsst.afw.image.ExposureF`, optional
553 The template to be subtracted from ``exposure`` in the image subtraction.
554 The template exposure should cover the same sky area as the science exposure.
555 It is either a stich of patches of a coadd skymap image or a calexp
556 of the same pointing as the science exposure. Can be None only
557 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
558 templateSources : `lsst.afw.table.SourceCatalog`, optional
559 Identified sources on the template exposure.
560 idFactory : `lsst.afw.table.IdFactory`
561 Generator object to assign ids to detected sources in the difference image.
562 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
563 Background exposure to be added back to the science exposure
564 if ``config.doAddCalexpBackground==True``
565 subtractedExposure : `lsst.afw.image.ExposureF`, optional
566 If ``config.doSubtract==False`` and ``config.doDetection==True``,
567 performs the post subtraction source detection only on this exposure.
568 Otherwise should be None.
572 results : `lsst.pipe.base.Struct`
573 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
575 ``matchedExposure`` : `lsst.afw.image.ExposureF`
576 The matched PSF exposure.
577 ``subtractRes`` : `lsst.pipe.base.Struct`
578 The returned result structure of the ImagePsfMatchTask subtask.
579 ``diaSources`` : `lsst.afw.table.SourceCatalog`
580 The catalog of detected sources.
581 ``selectSources`` : `lsst.afw.table.SourceCatalog`
582 The input source catalog with optionally added Qa information.
586 The following major steps are included:
588 - warp template coadd to match WCS of image
589 - PSF match image to warped template
590 - subtract image from PSF-matched, warped template
594 For details about the image subtraction configuration modes
595 see `lsst.ip.diffim`.
598 controlSources =
None
602 if self.config.doAddCalexpBackground:
603 mi = exposure.getMaskedImage()
604 mi += calexpBackgroundExposure.getImage()
606 if not exposure.hasPsf():
607 raise pipeBase.TaskError(
"Exposure has no psf")
608 sciencePsf = exposure.getPsf()
610 if self.config.doSubtract:
611 if self.config.doScaleTemplateVariance:
612 self.log.info(
"Rescaling template variance")
613 templateVarFactor = self.scaleVariance.
run(
614 templateExposure.getMaskedImage())
615 self.log.info(
"Template variance scaling factor: %.2f" % templateVarFactor)
616 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
618 if self.config.subtract.name ==
'zogy':
619 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
620 if self.config.doPreConvolve:
621 subtractedExposure = subtractRes.scoreExp
623 subtractedExposure = subtractRes.diffExp
624 subtractRes.subtractedExposure = subtractedExposure
625 subtractRes.matchedExposure =
None
627 elif self.config.subtract.name ==
'al':
629 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
630 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
638 if self.config.doPreConvolve:
639 convControl = afwMath.ConvolutionControl()
641 srcMI = exposure.getMaskedImage()
642 exposureOrig = exposure.clone()
643 destMI = srcMI.Factory(srcMI.getDimensions())
645 if self.config.useGaussianForPreConvolution:
647 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
652 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
653 exposure.setMaskedImage(destMI)
654 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
656 scienceSigmaPost = scienceSigmaOrig
657 exposureOrig = exposure
662 if self.config.doSelectSources:
663 if selectSources
is None:
664 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
666 selectSources = self.subtract.getSelectSources(
668 sigma=scienceSigmaPost,
669 doSmooth=
not self.config.doPreConvolve,
673 if self.config.doAddMetrics:
676 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
677 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
678 targetFwhmPix=templateSigma*FwhmPerSigma))
685 kcQa = KernelCandidateQa(nparam)
686 selectSources = kcQa.addToSchema(selectSources)
687 if self.config.kernelSourcesFromRef:
689 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
690 matches = astromRet.matches
691 elif templateSources:
693 mc = afwTable.MatchControl()
694 mc.findOnlyClosest =
False
695 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
698 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
699 "but template sources not available. Cannot match science "
700 "sources with template sources. Run process* on data from "
701 "which templates are built.")
703 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
704 matches=matches).sourceCat
705 random.shuffle(kernelSources, random.random)
706 controlSources = kernelSources[::self.config.controlStepSize]
707 kernelSources = [k
for i, k
in enumerate(kernelSources)
708 if i % self.config.controlStepSize]
710 if self.config.doSelectDcrCatalog:
711 redSelector = DiaCatalogSourceSelectorTask(
712 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
714 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
715 controlSources.extend(redSources)
717 blueSelector = DiaCatalogSourceSelectorTask(
718 DiaCatalogSourceSelectorConfig(grMin=-99.999,
719 grMax=self.sourceSelector.config.grMin))
720 blueSources = blueSelector.selectStars(exposure, selectSources,
721 matches=matches).starCat
722 controlSources.extend(blueSources)
724 if self.config.doSelectVariableCatalog:
725 varSelector = DiaCatalogSourceSelectorTask(
726 DiaCatalogSourceSelectorConfig(includeVariable=
True))
727 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
728 controlSources.extend(varSources)
730 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
731 % (len(kernelSources), len(selectSources), len(controlSources)))
735 if self.config.doUseRegister:
736 self.log.info(
"Registering images")
738 if templateSources
is None:
742 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
743 templateSources = self.subtract.getSelectSources(
752 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
753 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
754 exposure.getWcs(), exposure.getBBox())
755 templateExposure = warpedExp
760 if self.config.doDebugRegister:
762 srcToMatch = {x.second.getId(): x.first
for x
in matches}
764 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
765 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
766 sids = [m.first.getId()
for m
in wcsResults.matches]
767 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
768 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
769 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
770 allresids = dict(zip(sids, zip(positions, residuals)))
772 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
773 wcsResults.wcs.pixelToSky(
774 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
775 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
776 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
777 for x
in sids
if x
in srcToMatch.keys()])
778 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
779 if s
in srcToMatch.keys()])
780 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
781 if s
in srcToMatch.keys()])
782 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
783 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
784 & (colors <= self.sourceSelector.config.grMax))
785 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
786 rms1Long = IqrToSigma*(
787 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
788 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
789 - numpy.percentile(dlat[idx1], 25))
790 rms2Long = IqrToSigma*(
791 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
792 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
793 - numpy.percentile(dlat[idx2], 25))
794 rms3Long = IqrToSigma*(
795 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
796 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
797 - numpy.percentile(dlat[idx3], 25))
798 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
799 (numpy.median(dlong[idx1]), rms1Long,
800 numpy.median(dlat[idx1]), rms1Lat))
801 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
802 (numpy.median(dlong[idx2]), rms2Long,
803 numpy.median(dlat[idx2]), rms2Lat))
804 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
805 (numpy.median(dlong[idx3]), rms3Long,
806 numpy.median(dlat[idx3]), rms3Lat))
808 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
809 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
810 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
811 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
812 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
813 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
815 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
816 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
817 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
818 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
819 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
820 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
827 self.log.info(
"Subtracting images")
828 subtractRes = self.subtract.subtractExposures(
829 templateExposure=templateExposure,
830 scienceExposure=exposure,
831 candidateList=kernelSources,
832 convolveTemplate=self.config.convolveTemplate,
833 doWarping=
not self.config.doUseRegister
835 subtractedExposure = subtractRes.subtractedExposure
837 if self.config.doDetection:
838 self.log.info(
"Computing diffim PSF")
841 if not subtractedExposure.hasPsf():
842 if self.config.convolveTemplate:
843 subtractedExposure.setPsf(exposure.getPsf())
845 subtractedExposure.setPsf(templateExposure.getPsf())
852 if self.config.doDecorrelation
and self.config.doSubtract:
854 if preConvPsf
is not None:
855 preConvKernel = preConvPsf.getLocalKernel()
856 if self.config.convolveTemplate:
857 self.log.info(
"Decorrelation after template image convolution")
858 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
860 subtractRes.psfMatchingKernel,
861 spatiallyVarying=self.config.doSpatiallyVarying,
862 preConvKernel=preConvKernel)
864 self.log.info(
"Decorrelation after science image convolution")
865 decorrResult = self.decorrelate.
run(subtractRes.warpedExposure, exposureOrig,
867 subtractRes.psfMatchingKernel,
868 spatiallyVarying=self.config.doSpatiallyVarying,
869 preConvKernel=preConvKernel)
870 subtractedExposure = decorrResult.correctedExposure
874 if self.config.doDetection:
875 self.log.info(
"Running diaSource detection")
878 if self.config.doScaleDiffimVariance:
879 self.log.info(
"Rescaling diffim variance")
880 diffimVarFactor = self.scaleVariance.
run(subtractedExposure.getMaskedImage())
881 self.log.info(
"Diffim variance scaling factor: %.2f" % diffimVarFactor)
882 self.metadata.add(
"scaleDiffimVarianceFactor", diffimVarFactor)
885 mask = subtractedExposure.getMaskedImage().getMask()
886 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
888 table = afwTable.SourceTable.make(self.schema, idFactory)
889 table.setMetadata(self.algMetadata)
890 results = self.detection.
run(
892 exposure=subtractedExposure,
893 doSmooth=
not self.config.doPreConvolve
896 if self.config.doMerge:
897 fpSet = results.fpSets.positive
898 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
899 self.config.growFootprint,
False)
900 diaSources = afwTable.SourceCatalog(table)
901 fpSet.makeSources(diaSources)
902 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
904 diaSources = results.sources
906 if self.config.doMeasurement:
907 newDipoleFitting = self.config.doDipoleFitting
908 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
909 if not newDipoleFitting:
911 self.measurement.
run(diaSources, subtractedExposure)
914 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
915 self.measurement.
run(diaSources, subtractedExposure, exposure,
916 subtractRes.matchedExposure)
918 self.measurement.
run(diaSources, subtractedExposure, exposure)
920 if self.config.doForcedMeasurement:
923 forcedSources = self.forcedMeasurement.generateMeasCat(
924 exposure, diaSources, subtractedExposure.getWcs())
925 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
926 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
927 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
928 "ip_diffim_forced_PsfFlux_instFlux",
True)
929 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
930 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
931 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
932 "ip_diffim_forced_PsfFlux_area",
True)
933 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
934 "ip_diffim_forced_PsfFlux_flag",
True)
935 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
936 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
937 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
938 "ip_diffim_forced_PsfFlux_flag_edge",
True)
939 for diaSource, forcedSource
in zip(diaSources, forcedSources):
940 diaSource.assign(forcedSource, mapper)
943 if self.config.doMatchSources:
944 if selectSources
is not None:
946 matchRadAsec = self.config.diaSourceMatchRadius
947 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
949 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
950 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
951 srcMatch
in srcMatches])
952 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
955 self.log.warn(
"Src product does not exist; cannot match with diaSources")
959 refAstromConfig = AstrometryConfig()
960 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
961 refAstrometer = AstrometryTask(refAstromConfig)
962 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
963 refMatches = astromRet.matches
964 if refMatches
is None:
965 self.log.warn(
"No diaSource matches with reference catalog")
968 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
970 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
971 refMatch
in refMatches])
974 for diaSource
in diaSources:
975 sid = diaSource.getId()
976 if sid
in srcMatchDict:
977 diaSource.set(
"srcMatchId", srcMatchDict[sid])
978 if sid
in refMatchDict:
979 diaSource.set(
"refMatchId", refMatchDict[sid])
981 if self.config.doAddMetrics
and self.config.doSelectSources:
982 self.log.info(
"Evaluating metrics and control sample")
985 for cell
in subtractRes.kernelCellSet.getCellList():
986 for cand
in cell.begin(
False):
987 kernelCandList.append(cand)
990 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
991 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
994 diffimTools.sourceTableToCandidateList(controlSources,
995 subtractRes.warpedExposure, exposure,
996 self.config.subtract.kernel.active,
997 self.config.subtract.kernel.active.detectionConfig,
998 self.log, doBuild=
True, basisList=basisList))
1000 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1001 subtractRes.backgroundModel, dof=nparam)
1002 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1003 subtractRes.backgroundModel)
1005 if self.config.doDetection:
1006 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1008 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1010 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1011 return pipeBase.Struct(
1012 subtractedExposure=subtractedExposure,
1013 warpedExposure=subtractRes.warpedExposure,
1014 matchedExposure=subtractRes.matchedExposure,
1015 subtractRes=subtractRes,
1016 diaSources=diaSources,
1017 selectSources=selectSources
1020 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1021 """Fit the relative astrometry between templateSources and selectSources
1026 Remove this method. It originally fit a new WCS to the template before calling register.run
1027 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1028 It remains because a subtask overrides it.
1030 results = self.register.
run(templateSources, templateExposure.getWcs(),
1031 templateExposure.getBBox(), selectSources)
1034 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1035 """Make debug plots and displays.
1039 Test and update for current debug display and slot names
1049 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1050 if not maskTransparency:
1051 maskTransparency = 0
1052 disp.setMaskTransparency(maskTransparency)
1054 if display
and showSubtracted:
1055 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1056 mi = subtractRes.subtractedExposure.getMaskedImage()
1057 x0, y0 = mi.getX0(), mi.getY0()
1058 with disp.Buffering():
1059 for s
in diaSources:
1060 x, y = s.getX() - x0, s.getY() - y0
1061 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1062 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1063 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1064 or s.get(
"base_PixelFlags_flag_crCenter")):
1066 elif (s.get(
"base_PixelFlags_flag_interpolated")
1067 or s.get(
"base_PixelFlags_flag_saturated")
1068 or s.get(
"base_PixelFlags_flag_cr")):
1072 disp.dot(ptype, x, y, size=4, ctype=ctype)
1073 lsstDebug.frame += 1
1075 if display
and showPixelResiduals
and selectSources:
1076 nonKernelSources = []
1077 for source
in selectSources:
1078 if source
not in kernelSources:
1079 nonKernelSources.append(source)
1081 diUtils.plotPixelResiduals(exposure,
1082 subtractRes.warpedExposure,
1083 subtractRes.subtractedExposure,
1084 subtractRes.kernelCellSet,
1085 subtractRes.psfMatchingKernel,
1086 subtractRes.backgroundModel,
1088 self.subtract.config.kernel.active.detectionConfig,
1090 diUtils.plotPixelResiduals(exposure,
1091 subtractRes.warpedExposure,
1092 subtractRes.subtractedExposure,
1093 subtractRes.kernelCellSet,
1094 subtractRes.psfMatchingKernel,
1095 subtractRes.backgroundModel,
1097 self.subtract.config.kernel.active.detectionConfig,
1099 if display
and showDiaSources:
1100 flagChecker = SourceFlagChecker(diaSources)
1101 isFlagged = [flagChecker(x)
for x
in diaSources]
1102 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1103 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1104 frame=lsstDebug.frame)
1105 lsstDebug.frame += 1
1107 if display
and showDipoles:
1108 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1109 frame=lsstDebug.frame)
1110 lsstDebug.frame += 1
1112 def _getConfigName(self):
1113 """Return the name of the config dataset
1115 return "%sDiff_config" % (self.config.coaddName,)
1117 def _getMetadataName(self):
1118 """Return the name of the metadata dataset
1120 return "%sDiff_metadata" % (self.config.coaddName,)
1122 def getSchemaCatalogs(self):
1123 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1124 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1127 def _makeArgumentParser(cls):
1128 """Create an argument parser
1130 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1132 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1133 help=
"Template data ID in case of calexp template,"
1134 " e.g. --templateId visit=6789")
1138 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1139 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1140 doc=
"Shift stars going into RegisterTask by this amount")
1141 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1142 doc=
"Perturb stars going into RegisterTask by this amount")
1144 def setDefaults(self):
1145 ImageDifferenceConfig.setDefaults(self)
1146 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1149 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1150 """!Image difference Task used in the Winter 2013 data challege.
1151 Enables testing the effects of registration shifts and scatter.
1153 For use with winter 2013 simulated images:
1154 Use --templateId visit=88868666 for sparse data
1155 --templateId visit=22222200 for dense data (g)
1156 --templateId visit=11111100 for dense data (i)
1158 ConfigClass = Winter2013ImageDifferenceConfig
1159 _DefaultName =
"winter2013ImageDifference"
1161 def __init__(self, **kwargs):
1162 ImageDifferenceTask.__init__(self, **kwargs)
1164 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1165 """Fit the relative astrometry between templateSources and selectSources"""
1166 if self.config.winter2013WcsShift > 0.0:
1168 self.config.winter2013WcsShift)
1169 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1170 for source
in templateSources:
1171 centroid = source.get(cKey)
1172 source.set(cKey, centroid + offset)
1173 elif self.config.winter2013WcsRms > 0.0:
1174 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1175 for source
in templateSources:
1176 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1177 self.config.winter2013WcsRms*numpy.random.normal())
1178 centroid = source.get(cKey)
1179 source.set(cKey, centroid + offset)
1181 results = self.register.
run(templateSources, templateExposure.getWcs(),
1182 templateExposure.getBBox(), selectSources)