38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 from lsst.obs.base
import ExposureIdInfo
49 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
55 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
56 defaultTemplates={
"coaddName":
"deep",
61 exposure = pipeBase.connectionTypes.Input(
62 doc=
"Input science exposure to subtract from.",
63 dimensions=(
"instrument",
"visit",
"detector"),
64 storageClass=
"ExposureF",
65 name=
"{fakesType}calexp"
76 skyMap = pipeBase.connectionTypes.Input(
77 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
78 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79 dimensions=(
"skymap", ),
80 storageClass=
"SkyMap",
82 coaddExposures = pipeBase.connectionTypes.Input(
83 doc=
"Input template to match and subtract from the exposure",
84 dimensions=(
"tract",
"patch",
"skymap",
"band"),
85 storageClass=
"ExposureF",
86 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
90 dcrCoadds = pipeBase.connectionTypes.Input(
91 doc=
"Input DCR template to match and subtract from the exposure",
92 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
93 storageClass=
"ExposureF",
94 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
98 outputSchema = pipeBase.connectionTypes.InitOutput(
99 doc=
"Schema (as an example catalog) for output DIASource catalog.",
100 storageClass=
"SourceCatalog",
101 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
103 subtractedExposure = pipeBase.connectionTypes.Output(
104 doc=
"Output difference image",
105 dimensions=(
"instrument",
"visit",
"detector"),
106 storageClass=
"ExposureF",
107 name=
"{fakesType}{coaddName}Diff_differenceExp",
109 warpedExposure = pipeBase.connectionTypes.Output(
110 doc=
"Warped template used to create `subtractedExposure`.",
111 dimensions=(
"instrument",
"visit",
"detector"),
112 storageClass=
"ExposureF",
113 name=
"{fakesType}{coaddName}Diff_warpedExp",
115 diaSources = pipeBase.connectionTypes.Output(
116 doc=
"Output detected diaSources on the difference image",
117 dimensions=(
"instrument",
"visit",
"detector"),
118 storageClass=
"SourceCatalog",
119 name=
"{fakesType}{coaddName}Diff_diaSrc",
122 def __init__(self, *, config=None):
123 super().__init__(config=config)
124 if config.coaddName ==
'dcr':
125 self.inputs.remove(
"coaddExposures")
127 self.inputs.remove(
"dcrCoadds")
133 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
134 pipelineConnections=ImageDifferenceTaskConnections):
135 """Config for ImageDifferenceTask
137 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
138 doc=
"Add background to calexp before processing it. "
139 "Useful as ipDiffim does background matching.")
140 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
141 doc=
"Use image-to-image registration to align template with "
143 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
144 doc=
"Writing debugging data for doUseRegister")
145 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
146 doc=
"Select stars to use for kernel fitting")
147 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Select stars of extreme color as part of the control sample")
149 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
150 doc=
"Select stars that are variable to be part "
151 "of the control sample")
152 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
153 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
154 doc=
"Convolve science image by its PSF before PSF-matching?")
155 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
156 doc=
"Scale variance of the template before PSF matching")
157 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=
True,
158 doc=
"Scale variance of the diffim before PSF matching. "
159 "You may do either this or template variance scaling, "
160 "or neither. (Doing both is a waste of CPU.)")
161 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
162 doc=
"Use a simple gaussian PSF model for pre-convolution "
163 "(else use fit PSF)? Ignored if doPreConvolve false.")
164 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
165 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
166 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
167 "kernel convolution? If True, also update the diffim PSF.")
168 doMerge = pexConfig.Field(dtype=bool, default=
True,
169 doc=
"Merge positive and negative diaSources with grow radius "
170 "set by growFootprint")
171 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
172 doc=
"Match diaSources with input calexp sources and ref catalog sources")
173 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
174 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
175 doForcedMeasurement = pexConfig.Field(
178 doc=
"Force photometer diaSource locations on PVI?")
179 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
180 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
181 doc=
"Write WCS, warped template coadd exposure?")
182 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
183 doc=
"Write warped and PSF-matched template coadd exposure?")
184 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
185 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
186 doc=
"Add columns to the source table to hold analysis metrics?")
188 coaddName = pexConfig.Field(
189 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
193 convolveTemplate = pexConfig.Field(
194 doc=
"Which image gets convolved (default = template)",
198 refObjLoader = pexConfig.ConfigurableField(
199 target=LoadIndexedReferenceObjectsTask,
200 doc=
"reference object loader",
202 astrometer = pexConfig.ConfigurableField(
203 target=AstrometryTask,
204 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
206 sourceSelector = pexConfig.ConfigurableField(
207 target=ObjectSizeStarSelectorTask,
208 doc=
"Source selection algorithm",
210 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
211 decorrelate = pexConfig.ConfigurableField(
212 target=DecorrelateALKernelSpatialTask,
213 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
214 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
218 doSpatiallyVarying = pexConfig.Field(
221 doc=
"Perform A&L decorrelation on a grid across the "
222 "image in order to allow for spatial variations. Zogy does not use this option."
224 detection = pexConfig.ConfigurableField(
225 target=SourceDetectionTask,
226 doc=
"Low-threshold detection for final measurement",
228 measurement = pexConfig.ConfigurableField(
229 target=DipoleFitTask,
230 doc=
"Enable updated dipole fitting method",
232 doApCorr = lsst.pex.config.Field(
235 doc=
"Run subtask to apply aperture corrections"
237 applyApCorr = lsst.pex.config.ConfigurableField(
238 target=ApplyApCorrTask,
239 doc=
"Subtask to apply aperture corrections"
241 forcedMeasurement = pexConfig.ConfigurableField(
242 target=ForcedMeasurementTask,
243 doc=
"Subtask to force photometer PVI at diaSource location.",
245 getTemplate = pexConfig.ConfigurableField(
246 target=GetCoaddAsTemplateTask,
247 doc=
"Subtask to retrieve template exposure and sources",
249 scaleVariance = pexConfig.ConfigurableField(
250 target=ScaleVarianceTask,
251 doc=
"Subtask to rescale the variance of the template "
252 "to the statistically expected level"
254 controlStepSize = pexConfig.Field(
255 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
259 controlRandomSeed = pexConfig.Field(
260 doc=
"Random seed for shuffing the control sample",
264 register = pexConfig.ConfigurableField(
266 doc=
"Task to enable image-to-image image registration (warping)",
268 kernelSourcesFromRef = pexConfig.Field(
269 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
273 templateSipOrder = pexConfig.Field(
274 dtype=int, default=2,
275 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
277 growFootprint = pexConfig.Field(
278 dtype=int, default=2,
279 doc=
"Grow positive and negative footprints by this amount before merging"
281 diaSourceMatchRadius = pexConfig.Field(
282 dtype=float, default=0.5,
283 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
286 def setDefaults(self):
289 self.subtract[
'al'].kernel.name =
"AL"
290 self.subtract[
'al'].kernel.active.fitForBackground =
True
291 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
292 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
293 self.doPreConvolve =
False
294 self.doMatchSources =
False
295 self.doAddMetrics =
False
296 self.doUseRegister =
False
299 self.detection.thresholdPolarity =
"both"
300 self.detection.thresholdValue = 5.5
301 self.detection.reEstimateBackground =
False
302 self.detection.thresholdType =
"pixel_stdev"
308 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
309 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
312 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
313 self.forcedMeasurement.copyColumns = {
314 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
315 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
316 self.forcedMeasurement.slots.shape =
None
319 random.seed(self.controlRandomSeed)
322 pexConfig.Config.validate(self)
323 if self.doAddMetrics
and not self.doSubtract:
324 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
325 if not self.doSubtract
and not self.doDetection:
326 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
327 if self.subtract.name ==
'zogy' and self.doAddMetrics:
328 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
329 if self.doMeasurement
and not self.doDetection:
330 raise ValueError(
"Cannot run source measurement without source detection.")
331 if self.doMerge
and not self.doDetection:
332 raise ValueError(
"Cannot run source merging without source detection.")
333 if self.doUseRegister
and not self.doSelectSources:
334 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
335 "Cannot run RegisterTask without selecting sources.")
336 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
337 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
338 "convolveTemplate=False is not supported.")
339 if hasattr(self.getTemplate,
"coaddName"):
340 if self.getTemplate.coaddName != self.coaddName:
341 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
342 if self.doScaleDiffimVariance
and self.doScaleTemplateVariance:
343 raise ValueError(
"Scaling the diffim variance and scaling the template variance "
344 "are both set. Please choose one or the other.")
347 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
350 def getTargetList(parsedCmd, **kwargs):
351 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
355 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
356 """Subtract an image from a template and measure the result
358 ConfigClass = ImageDifferenceConfig
359 RunnerClass = ImageDifferenceTaskRunner
360 _DefaultName =
"imageDifference"
362 def __init__(self, butler=None, **kwargs):
363 """!Construct an ImageDifference Task
365 @param[in] butler Butler object to use in constructing reference object loaders
367 super().__init__(**kwargs)
368 self.makeSubtask(
"getTemplate")
370 self.makeSubtask(
"subtract")
372 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
373 self.makeSubtask(
"decorrelate")
375 if self.config.doScaleTemplateVariance
or self.config.doScaleDiffimVariance:
376 self.makeSubtask(
"scaleVariance")
378 if self.config.doUseRegister:
379 self.makeSubtask(
"register")
380 self.schema = afwTable.SourceTable.makeMinimalSchema()
382 if self.config.doSelectSources:
383 self.makeSubtask(
"sourceSelector")
384 if self.config.kernelSourcesFromRef:
385 self.makeSubtask(
'refObjLoader', butler=butler)
386 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
388 self.algMetadata = dafBase.PropertyList()
389 if self.config.doDetection:
390 self.makeSubtask(
"detection", schema=self.schema)
391 if self.config.doMeasurement:
392 self.makeSubtask(
"measurement", schema=self.schema,
393 algMetadata=self.algMetadata)
394 if self.config.doApCorr:
395 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
396 if self.config.doForcedMeasurement:
397 self.schema.addField(
398 "ip_diffim_forced_PsfFlux_instFlux",
"D",
399 "Forced PSF flux measured on the direct image.",
401 self.schema.addField(
402 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
403 "Forced PSF flux error measured on the direct image.",
405 self.schema.addField(
406 "ip_diffim_forced_PsfFlux_area",
"F",
407 "Forced PSF flux effective area of PSF.",
409 self.schema.addField(
410 "ip_diffim_forced_PsfFlux_flag",
"Flag",
411 "Forced PSF flux general failure flag.")
412 self.schema.addField(
413 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
414 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
415 self.schema.addField(
416 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
417 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
418 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
419 if self.config.doMatchSources:
420 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
421 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
424 self.outputSchema = afwTable.SourceCatalog(self.schema)
425 self.outputSchema.getTable().setMetadata(self.algMetadata)
428 def makeIdFactory(expId, expBits):
429 """Create IdFactory instance for unique 64 bit diaSource id-s.
437 Number of used bits in ``expId``.
441 The diasource id-s consists of the ``expId`` stored fixed in the highest value
442 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
443 low value end of the integer.
447 idFactory: `lsst.afw.table.IdFactory`
449 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
451 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
452 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
453 inputRefs: pipeBase.InputQuantizedConnection,
454 outputRefs: pipeBase.OutputQuantizedConnection):
455 inputs = butlerQC.get(inputRefs)
456 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
457 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
459 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
460 if self.config.coaddName ==
'dcr':
461 templateExposures = inputRefs.dcrCoadds
463 templateExposures = inputRefs.coaddExposures
464 templateStruct = self.getTemplate.runQuantum(
465 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
468 outputs = self.run(exposure=inputs[
'exposure'],
469 templateExposure=templateStruct.exposure,
471 butlerQC.put(outputs, outputRefs)
474 def runDataRef(self, sensorRef, templateIdList=None):
475 """Subtract an image from a template coadd and measure the result.
477 Data I/O wrapper around `run` using the butler in Gen2.
481 sensorRef : `lsst.daf.persistence.ButlerDataRef`
482 Sensor-level butler data reference, used for the following data products:
489 - self.config.coaddName + "Coadd_skyMap"
490 - self.config.coaddName + "Coadd"
491 Input or output, depending on config:
492 - self.config.coaddName + "Diff_subtractedExp"
493 Output, depending on config:
494 - self.config.coaddName + "Diff_matchedExp"
495 - self.config.coaddName + "Diff_src"
499 results : `lsst.pipe.base.Struct`
500 Returns the Struct by `run`.
502 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
503 subtractedExposure =
None
505 calexpBackgroundExposure =
None
506 self.log.info(
"Processing %s" % (sensorRef.dataId))
511 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
512 expBits=sensorRef.get(
"ccdExposureId_bits"))
513 if self.config.doAddCalexpBackground:
514 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
517 exposure = sensorRef.get(
"calexp", immediate=
True)
520 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
522 if sensorRef.datasetExists(
"src"):
523 self.log.info(
"Source selection via src product")
525 selectSources = sensorRef.get(
"src")
527 if not self.config.doSubtract
and self.config.doDetection:
529 subtractedExposure = sensorRef.get(subtractedExposureName)
532 results = self.run(exposure=exposure,
533 selectSources=selectSources,
534 templateExposure=template.exposure,
535 templateSources=template.sources,
537 calexpBackgroundExposure=calexpBackgroundExposure,
538 subtractedExposure=subtractedExposure)
540 if self.config.doWriteSources
and results.diaSources
is not None:
541 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
542 if self.config.doWriteWarpedExp:
543 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
544 if self.config.doWriteMatchedExp:
545 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
546 if self.config.doAddMetrics
and self.config.doSelectSources:
547 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
548 if self.config.doWriteSubtractedExp:
549 sensorRef.put(results.subtractedExposure, subtractedExposureName)
553 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
554 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
555 """PSF matches, subtract two images and perform detection on the difference image.
559 exposure : `lsst.afw.image.ExposureF`, optional
560 The science exposure, the minuend in the image subtraction.
561 Can be None only if ``config.doSubtract==False``.
562 selectSources : `lsst.afw.table.SourceCatalog`, optional
563 Identified sources on the science exposure. This catalog is used to
564 select sources in order to perform the AL PSF matching on stamp images
565 around them. The selection steps depend on config options and whether
566 ``templateSources`` and ``matchingSources`` specified.
567 templateExposure : `lsst.afw.image.ExposureF`, optional
568 The template to be subtracted from ``exposure`` in the image subtraction.
569 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
570 The template exposure should cover the same sky area as the science exposure.
571 It is either a stich of patches of a coadd skymap image or a calexp
572 of the same pointing as the science exposure. Can be None only
573 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
574 templateSources : `lsst.afw.table.SourceCatalog`, optional
575 Identified sources on the template exposure.
576 idFactory : `lsst.afw.table.IdFactory`
577 Generator object to assign ids to detected sources in the difference image.
578 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
579 Background exposure to be added back to the science exposure
580 if ``config.doAddCalexpBackground==True``
581 subtractedExposure : `lsst.afw.image.ExposureF`, optional
582 If ``config.doSubtract==False`` and ``config.doDetection==True``,
583 performs the post subtraction source detection only on this exposure.
584 Otherwise should be None.
588 results : `lsst.pipe.base.Struct`
589 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
591 ``matchedExposure`` : `lsst.afw.image.ExposureF`
592 The matched PSF exposure.
593 ``subtractRes`` : `lsst.pipe.base.Struct`
594 The returned result structure of the ImagePsfMatchTask subtask.
595 ``diaSources`` : `lsst.afw.table.SourceCatalog`
596 The catalog of detected sources.
597 ``selectSources`` : `lsst.afw.table.SourceCatalog`
598 The input source catalog with optionally added Qa information.
602 The following major steps are included:
604 - warp template coadd to match WCS of image
605 - PSF match image to warped template
606 - subtract image from PSF-matched, warped template
610 For details about the image subtraction configuration modes
611 see `lsst.ip.diffim`.
614 controlSources =
None
618 exposureOrig = exposure
620 if self.config.doAddCalexpBackground:
621 mi = exposure.getMaskedImage()
622 mi += calexpBackgroundExposure.getImage()
624 if not exposure.hasPsf():
625 raise pipeBase.TaskError(
"Exposure has no psf")
626 sciencePsf = exposure.getPsf()
628 if self.config.doSubtract:
629 if self.config.doScaleTemplateVariance:
630 self.log.info(
"Rescaling template variance")
631 templateVarFactor = self.scaleVariance.
run(
632 templateExposure.getMaskedImage())
633 self.log.info(
"Template variance scaling factor: %.2f" % templateVarFactor)
634 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
636 if self.config.subtract.name ==
'zogy':
637 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
638 if self.config.doPreConvolve:
639 subtractedExposure = subtractRes.scoreExp
641 subtractedExposure = subtractRes.diffExp
642 subtractRes.subtractedExposure = subtractedExposure
643 subtractRes.matchedExposure =
None
645 elif self.config.subtract.name ==
'al':
647 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
648 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
656 if self.config.doPreConvolve:
657 convControl = afwMath.ConvolutionControl()
659 srcMI = exposure.maskedImage
660 exposure = exposure.clone()
662 if self.config.useGaussianForPreConvolution:
664 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
669 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
670 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
672 scienceSigmaPost = scienceSigmaOrig
677 if self.config.doSelectSources:
678 if selectSources
is None:
679 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
681 selectSources = self.subtract.getSelectSources(
683 sigma=scienceSigmaPost,
684 doSmooth=
not self.config.doPreConvolve,
688 if self.config.doAddMetrics:
691 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
692 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
693 targetFwhmPix=templateSigma*FwhmPerSigma))
700 kcQa = KernelCandidateQa(nparam)
701 selectSources = kcQa.addToSchema(selectSources)
702 if self.config.kernelSourcesFromRef:
704 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
705 matches = astromRet.matches
706 elif templateSources:
708 mc = afwTable.MatchControl()
709 mc.findOnlyClosest =
False
710 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
713 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
714 "but template sources not available. Cannot match science "
715 "sources with template sources. Run process* on data from "
716 "which templates are built.")
718 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
719 matches=matches).sourceCat
720 random.shuffle(kernelSources, random.random)
721 controlSources = kernelSources[::self.config.controlStepSize]
722 kernelSources = [k
for i, k
in enumerate(kernelSources)
723 if i % self.config.controlStepSize]
725 if self.config.doSelectDcrCatalog:
726 redSelector = DiaCatalogSourceSelectorTask(
727 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
729 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
730 controlSources.extend(redSources)
732 blueSelector = DiaCatalogSourceSelectorTask(
733 DiaCatalogSourceSelectorConfig(grMin=-99.999,
734 grMax=self.sourceSelector.config.grMin))
735 blueSources = blueSelector.selectStars(exposure, selectSources,
736 matches=matches).starCat
737 controlSources.extend(blueSources)
739 if self.config.doSelectVariableCatalog:
740 varSelector = DiaCatalogSourceSelectorTask(
741 DiaCatalogSourceSelectorConfig(includeVariable=
True))
742 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
743 controlSources.extend(varSources)
745 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
746 % (len(kernelSources), len(selectSources), len(controlSources)))
750 if self.config.doUseRegister:
751 self.log.info(
"Registering images")
753 if templateSources
is None:
757 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
758 templateSources = self.subtract.getSelectSources(
767 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
768 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
769 exposure.getWcs(), exposure.getBBox())
770 templateExposure = warpedExp
775 if self.config.doDebugRegister:
777 srcToMatch = {x.second.getId(): x.first
for x
in matches}
779 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
780 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
781 sids = [m.first.getId()
for m
in wcsResults.matches]
782 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
783 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
784 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
785 allresids = dict(zip(sids, zip(positions, residuals)))
787 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
788 wcsResults.wcs.pixelToSky(
789 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
790 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
791 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
792 for x
in sids
if x
in srcToMatch.keys()])
793 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
794 if s
in srcToMatch.keys()])
795 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
796 if s
in srcToMatch.keys()])
797 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
798 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
799 & (colors <= self.sourceSelector.config.grMax))
800 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
801 rms1Long = IqrToSigma*(
802 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
803 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
804 - numpy.percentile(dlat[idx1], 25))
805 rms2Long = IqrToSigma*(
806 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
807 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
808 - numpy.percentile(dlat[idx2], 25))
809 rms3Long = IqrToSigma*(
810 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
811 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
812 - numpy.percentile(dlat[idx3], 25))
813 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
814 (numpy.median(dlong[idx1]), rms1Long,
815 numpy.median(dlat[idx1]), rms1Lat))
816 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
817 (numpy.median(dlong[idx2]), rms2Long,
818 numpy.median(dlat[idx2]), rms2Lat))
819 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
820 (numpy.median(dlong[idx3]), rms3Long,
821 numpy.median(dlat[idx3]), rms3Lat))
823 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
824 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
825 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
826 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
827 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
828 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
830 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
831 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
832 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
833 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
834 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
835 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
842 self.log.info(
"Subtracting images")
843 subtractRes = self.subtract.subtractExposures(
844 templateExposure=templateExposure,
845 scienceExposure=exposure,
846 candidateList=kernelSources,
847 convolveTemplate=self.config.convolveTemplate,
848 doWarping=
not self.config.doUseRegister
850 subtractedExposure = subtractRes.subtractedExposure
852 if self.config.doDetection:
853 self.log.info(
"Computing diffim PSF")
856 if not subtractedExposure.hasPsf():
857 if self.config.convolveTemplate:
858 subtractedExposure.setPsf(exposure.getPsf())
860 subtractedExposure.setPsf(templateExposure.getPsf())
867 if self.config.doDecorrelation
and self.config.doSubtract:
869 if preConvPsf
is not None:
870 preConvKernel = preConvPsf.getLocalKernel()
871 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
873 subtractRes.psfMatchingKernel,
874 spatiallyVarying=self.config.doSpatiallyVarying,
875 preConvKernel=preConvKernel,
876 templateMatched=self.config.convolveTemplate)
877 subtractedExposure = decorrResult.correctedExposure
881 if self.config.doDetection:
882 self.log.info(
"Running diaSource detection")
885 if self.config.doScaleDiffimVariance:
886 self.log.info(
"Rescaling diffim variance")
887 diffimVarFactor = self.scaleVariance.
run(subtractedExposure.getMaskedImage())
888 self.log.info(
"Diffim variance scaling factor: %.2f" % diffimVarFactor)
889 self.metadata.add(
"scaleDiffimVarianceFactor", diffimVarFactor)
892 mask = subtractedExposure.getMaskedImage().getMask()
893 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
895 table = afwTable.SourceTable.make(self.schema, idFactory)
896 table.setMetadata(self.algMetadata)
897 results = self.detection.
run(
899 exposure=subtractedExposure,
900 doSmooth=
not self.config.doPreConvolve
903 if self.config.doMerge:
904 fpSet = results.fpSets.positive
905 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
906 self.config.growFootprint,
False)
907 diaSources = afwTable.SourceCatalog(table)
908 fpSet.makeSources(diaSources)
909 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
911 diaSources = results.sources
913 if self.config.doMeasurement:
914 newDipoleFitting = self.config.doDipoleFitting
915 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
916 if not newDipoleFitting:
918 self.measurement.
run(diaSources, subtractedExposure)
921 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
922 self.measurement.
run(diaSources, subtractedExposure, exposure,
923 subtractRes.matchedExposure)
925 self.measurement.
run(diaSources, subtractedExposure, exposure)
926 if self.config.doApCorr:
927 self.applyApCorr.
run(
929 apCorrMap=subtractedExposure.getInfo().getApCorrMap()
932 if self.config.doForcedMeasurement:
935 forcedSources = self.forcedMeasurement.generateMeasCat(
936 exposure, diaSources, subtractedExposure.getWcs())
937 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
938 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
939 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
940 "ip_diffim_forced_PsfFlux_instFlux",
True)
941 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
942 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
943 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
944 "ip_diffim_forced_PsfFlux_area",
True)
945 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
946 "ip_diffim_forced_PsfFlux_flag",
True)
947 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
948 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
949 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
950 "ip_diffim_forced_PsfFlux_flag_edge",
True)
951 for diaSource, forcedSource
in zip(diaSources, forcedSources):
952 diaSource.assign(forcedSource, mapper)
955 if self.config.doMatchSources:
956 if selectSources
is not None:
958 matchRadAsec = self.config.diaSourceMatchRadius
959 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
961 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
962 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
963 srcMatch
in srcMatches])
964 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
967 self.log.warn(
"Src product does not exist; cannot match with diaSources")
971 refAstromConfig = AstrometryConfig()
972 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
973 refAstrometer = AstrometryTask(refAstromConfig)
974 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
975 refMatches = astromRet.matches
976 if refMatches
is None:
977 self.log.warn(
"No diaSource matches with reference catalog")
980 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
982 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
983 refMatch
in refMatches])
986 for diaSource
in diaSources:
987 sid = diaSource.getId()
988 if sid
in srcMatchDict:
989 diaSource.set(
"srcMatchId", srcMatchDict[sid])
990 if sid
in refMatchDict:
991 diaSource.set(
"refMatchId", refMatchDict[sid])
993 if self.config.doAddMetrics
and self.config.doSelectSources:
994 self.log.info(
"Evaluating metrics and control sample")
997 for cell
in subtractRes.kernelCellSet.getCellList():
998 for cand
in cell.begin(
False):
999 kernelCandList.append(cand)
1002 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1003 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1006 diffimTools.sourceTableToCandidateList(controlSources,
1007 subtractRes.warpedExposure, exposure,
1008 self.config.subtract.kernel.active,
1009 self.config.subtract.kernel.active.detectionConfig,
1010 self.log, doBuild=
True, basisList=basisList))
1012 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1013 subtractRes.backgroundModel, dof=nparam)
1014 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1015 subtractRes.backgroundModel)
1017 if self.config.doDetection:
1018 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1020 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1022 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1023 return pipeBase.Struct(
1024 subtractedExposure=subtractedExposure,
1025 warpedExposure=subtractRes.warpedExposure,
1026 matchedExposure=subtractRes.matchedExposure,
1027 subtractRes=subtractRes,
1028 diaSources=diaSources,
1029 selectSources=selectSources
1032 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1033 """Fit the relative astrometry between templateSources and selectSources
1038 Remove this method. It originally fit a new WCS to the template before calling register.run
1039 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1040 It remains because a subtask overrides it.
1042 results = self.register.
run(templateSources, templateExposure.getWcs(),
1043 templateExposure.getBBox(), selectSources)
1046 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1047 """Make debug plots and displays.
1051 Test and update for current debug display and slot names
1061 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1062 if not maskTransparency:
1063 maskTransparency = 0
1064 disp.setMaskTransparency(maskTransparency)
1066 if display
and showSubtracted:
1067 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1068 mi = subtractRes.subtractedExposure.getMaskedImage()
1069 x0, y0 = mi.getX0(), mi.getY0()
1070 with disp.Buffering():
1071 for s
in diaSources:
1072 x, y = s.getX() - x0, s.getY() - y0
1073 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1074 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1075 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1076 or s.get(
"base_PixelFlags_flag_crCenter")):
1078 elif (s.get(
"base_PixelFlags_flag_interpolated")
1079 or s.get(
"base_PixelFlags_flag_saturated")
1080 or s.get(
"base_PixelFlags_flag_cr")):
1084 disp.dot(ptype, x, y, size=4, ctype=ctype)
1085 lsstDebug.frame += 1
1087 if display
and showPixelResiduals
and selectSources:
1088 nonKernelSources = []
1089 for source
in selectSources:
1090 if source
not in kernelSources:
1091 nonKernelSources.append(source)
1093 diUtils.plotPixelResiduals(exposure,
1094 subtractRes.warpedExposure,
1095 subtractRes.subtractedExposure,
1096 subtractRes.kernelCellSet,
1097 subtractRes.psfMatchingKernel,
1098 subtractRes.backgroundModel,
1100 self.subtract.config.kernel.active.detectionConfig,
1102 diUtils.plotPixelResiduals(exposure,
1103 subtractRes.warpedExposure,
1104 subtractRes.subtractedExposure,
1105 subtractRes.kernelCellSet,
1106 subtractRes.psfMatchingKernel,
1107 subtractRes.backgroundModel,
1109 self.subtract.config.kernel.active.detectionConfig,
1111 if display
and showDiaSources:
1112 flagChecker = SourceFlagChecker(diaSources)
1113 isFlagged = [flagChecker(x)
for x
in diaSources]
1114 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1115 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1116 frame=lsstDebug.frame)
1117 lsstDebug.frame += 1
1119 if display
and showDipoles:
1120 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1121 frame=lsstDebug.frame)
1122 lsstDebug.frame += 1
1124 def _getConfigName(self):
1125 """Return the name of the config dataset
1127 return "%sDiff_config" % (self.config.coaddName,)
1129 def _getMetadataName(self):
1130 """Return the name of the metadata dataset
1132 return "%sDiff_metadata" % (self.config.coaddName,)
1134 def getSchemaCatalogs(self):
1135 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1136 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1139 def _makeArgumentParser(cls):
1140 """Create an argument parser
1142 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1143 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1144 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1145 help=
"Template data ID in case of calexp template,"
1146 " e.g. --templateId visit=6789")
1150 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1151 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1152 doc=
"Shift stars going into RegisterTask by this amount")
1153 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1154 doc=
"Perturb stars going into RegisterTask by this amount")
1156 def setDefaults(self):
1157 ImageDifferenceConfig.setDefaults(self)
1158 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1161 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1162 """!Image difference Task used in the Winter 2013 data challege.
1163 Enables testing the effects of registration shifts and scatter.
1165 For use with winter 2013 simulated images:
1166 Use --templateId visit=88868666 for sparse data
1167 --templateId visit=22222200 for dense data (g)
1168 --templateId visit=11111100 for dense data (i)
1170 ConfigClass = Winter2013ImageDifferenceConfig
1171 _DefaultName =
"winter2013ImageDifference"
1173 def __init__(self, **kwargs):
1174 ImageDifferenceTask.__init__(self, **kwargs)
1176 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1177 """Fit the relative astrometry between templateSources and selectSources"""
1178 if self.config.winter2013WcsShift > 0.0:
1180 self.config.winter2013WcsShift)
1181 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1182 for source
in templateSources:
1183 centroid = source.get(cKey)
1184 source.set(cKey, centroid + offset)
1185 elif self.config.winter2013WcsRms > 0.0:
1186 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1187 for source
in templateSources:
1188 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1189 self.config.winter2013WcsRms*numpy.random.normal())
1190 centroid = source.get(cKey)
1191 source.set(cKey, centroid + offset)
1193 results = self.register.
run(templateSources, templateExposure.getWcs(),
1194 templateExposure.getBBox(), selectSources)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)