27 import lsst.pex.config
as pexConfig
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
53 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
54 defaultTemplates={
"coaddName":
"deep",
59 exposure = pipeBase.connectionTypes.Input(
60 doc=
"Input science exposure to subtract from.",
61 dimensions=(
"instrument",
"visit",
"detector"),
62 storageClass=
"ExposureF",
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
76 name=
"{skyMapName}Coadd_skyMap",
77 dimensions=(
"skymap", ),
78 storageClass=
"SkyMap",
80 coaddExposures = pipeBase.connectionTypes.Input(
81 doc=
"Input template to match and subtract from the exposure",
82 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
83 storageClass=
"ExposureF",
84 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
88 dcrCoadds = pipeBase.connectionTypes.Input(
89 doc=
"Input DCR template to match and subtract from the exposure",
90 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
91 storageClass=
"ExposureF",
92 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter",
"subfilter"),
96 outputSchema = pipeBase.connectionTypes.InitOutput(
97 doc=
"Schema (as an example catalog) for output DIASource catalog.",
98 storageClass=
"SourceCatalog",
99 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
101 subtractedExposure = pipeBase.connectionTypes.Output(
102 doc=
"Output difference image",
103 dimensions=(
"instrument",
"visit",
"detector"),
104 storageClass=
"ExposureF",
105 name=
"{fakesType}{coaddName}Diff_differenceExp",
107 warpedExposure = pipeBase.connectionTypes.Output(
108 doc=
"Warped template used to create `subtractedExposure`.",
109 dimensions=(
"instrument",
"visit",
"detector"),
110 storageClass=
"ExposureF",
111 name=
"{fakesType}{coaddName}Diff_warpedExp",
113 diaSources = pipeBase.connectionTypes.Output(
114 doc=
"Output detected diaSources on the difference image",
115 dimensions=(
"instrument",
"visit",
"detector"),
116 storageClass=
"SourceCatalog",
117 name=
"{fakesType}{coaddName}Diff_diaSrc",
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
122 if config.coaddName ==
'dcr':
123 self.inputs.remove(
"coaddExposures")
125 self.inputs.remove(
"dcrCoadds")
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132 pipelineConnections=ImageDifferenceTaskConnections):
133 """Config for ImageDifferenceTask
135 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
136 doc=
"Add background to calexp before processing it. "
137 "Useful as ipDiffim does background matching.")
138 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
139 doc=
"Use image-to-image registration to align template with "
141 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Writing debugging data for doUseRegister")
143 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
144 doc=
"Select stars to use for kernel fitting")
145 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
146 doc=
"Select stars of extreme color as part of the control sample")
147 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Select stars that are variable to be part "
149 "of the control sample")
150 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
151 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
152 doc=
"Convolve science image by its PSF before PSF-matching?")
153 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
154 doc=
"Scale variance of the template before PSF matching")
155 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
156 doc=
"Use a simple gaussian PSF model for pre-convolution "
157 "(else use fit PSF)? Ignored if doPreConvolve false.")
158 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
159 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
160 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
161 "kernel convolution? If True, also update the diffim PSF.")
162 doMerge = pexConfig.Field(dtype=bool, default=
True,
163 doc=
"Merge positive and negative diaSources with grow radius "
164 "set by growFootprint")
165 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
166 doc=
"Match diaSources with input calexp sources and ref catalog sources")
167 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
168 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
169 doForcedMeasurement = pexConfig.Field(
172 doc=
"Force photometer diaSource locations on PVI?")
173 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
174 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
175 doc=
"Write WCS, warped template coadd exposure?")
176 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
177 doc=
"Write warped and PSF-matched template coadd exposure?")
178 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
179 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
180 doc=
"Add columns to the source table to hold analysis metrics?")
182 coaddName = pexConfig.Field(
183 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
187 convolveTemplate = pexConfig.Field(
188 doc=
"Which image gets convolved (default = template)",
192 refObjLoader = pexConfig.ConfigurableField(
193 target=LoadIndexedReferenceObjectsTask,
194 doc=
"reference object loader",
196 astrometer = pexConfig.ConfigurableField(
197 target=AstrometryTask,
198 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
200 sourceSelector = pexConfig.ConfigurableField(
201 target=ObjectSizeStarSelectorTask,
202 doc=
"Source selection algorithm",
204 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
205 decorrelate = pexConfig.ConfigurableField(
206 target=DecorrelateALKernelSpatialTask,
207 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
211 doSpatiallyVarying = pexConfig.Field(
214 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
215 "image in order to allow for spatial variations"
217 detection = pexConfig.ConfigurableField(
218 target=SourceDetectionTask,
219 doc=
"Low-threshold detection for final measurement",
221 measurement = pexConfig.ConfigurableField(
222 target=DipoleFitTask,
223 doc=
"Enable updated dipole fitting method",
225 forcedMeasurement = pexConfig.ConfigurableField(
226 target=ForcedMeasurementTask,
227 doc=
"Subtask to force photometer PVI at diaSource location.",
229 getTemplate = pexConfig.ConfigurableField(
230 target=GetCoaddAsTemplateTask,
231 doc=
"Subtask to retrieve template exposure and sources",
233 scaleVariance = pexConfig.ConfigurableField(
234 target=ScaleVarianceTask,
235 doc=
"Subtask to rescale the variance of the template "
236 "to the statistically expected level"
238 controlStepSize = pexConfig.Field(
239 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
243 controlRandomSeed = pexConfig.Field(
244 doc=
"Random seed for shuffing the control sample",
248 register = pexConfig.ConfigurableField(
250 doc=
"Task to enable image-to-image image registration (warping)",
252 kernelSourcesFromRef = pexConfig.Field(
253 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
257 templateSipOrder = pexConfig.Field(
258 dtype=int, default=2,
259 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
261 growFootprint = pexConfig.Field(
262 dtype=int, default=2,
263 doc=
"Grow positive and negative footprints by this amount before merging"
265 diaSourceMatchRadius = pexConfig.Field(
266 dtype=float, default=0.5,
267 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
270 def setDefaults(self):
273 self.subtract[
'al'].kernel.name =
"AL"
274 self.subtract[
'al'].kernel.active.fitForBackground =
True
275 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
276 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
277 self.doPreConvolve =
False
278 self.doMatchSources =
False
279 self.doAddMetrics =
False
280 self.doUseRegister =
False
283 self.detection.thresholdPolarity =
"both"
284 self.detection.thresholdValue = 5.5
285 self.detection.reEstimateBackground =
False
286 self.detection.thresholdType =
"pixel_stdev"
292 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
293 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
296 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
297 self.forcedMeasurement.copyColumns = {
298 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
299 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
300 self.forcedMeasurement.slots.shape =
None
303 random.seed(self.controlRandomSeed)
306 pexConfig.Config.validate(self)
307 if self.doAddMetrics
and not self.doSubtract:
308 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
309 if not self.doSubtract
and not self.doDetection:
310 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
311 if self.subtract.name ==
'zogy' and self.doAddMetrics:
312 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
313 if self.doMeasurement
and not self.doDetection:
314 raise ValueError(
"Cannot run source measurement without source detection.")
315 if self.doMerge
and not self.doDetection:
316 raise ValueError(
"Cannot run source merging without source detection.")
317 if self.doUseRegister
and not self.doSelectSources:
318 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
319 "Cannot run RegisterTask without selecting sources.")
320 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
321 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
322 "convolveTemplate=False is not supported.")
323 if hasattr(self.getTemplate,
"coaddName"):
324 if self.getTemplate.coaddName != self.coaddName:
325 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
331 def getTargetList(parsedCmd, **kwargs):
332 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337 """Subtract an image from a template and measure the result
339 ConfigClass = ImageDifferenceConfig
340 RunnerClass = ImageDifferenceTaskRunner
341 _DefaultName =
"imageDifference"
343 def __init__(self, butler=None, **kwargs):
344 """!Construct an ImageDifference Task
346 @param[in] butler Butler object to use in constructing reference object loaders
348 super().__init__(**kwargs)
349 self.makeSubtask(
"getTemplate")
351 self.makeSubtask(
"subtract")
353 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
354 self.makeSubtask(
"decorrelate")
356 if self.config.doScaleTemplateVariance:
357 self.makeSubtask(
"scaleVariance")
359 if self.config.doUseRegister:
360 self.makeSubtask(
"register")
361 self.schema = afwTable.SourceTable.makeMinimalSchema()
363 if self.config.doSelectSources:
364 self.makeSubtask(
"sourceSelector")
365 if self.config.kernelSourcesFromRef:
366 self.makeSubtask(
'refObjLoader', butler=butler)
367 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
369 self.algMetadata = dafBase.PropertyList()
370 if self.config.doDetection:
371 self.makeSubtask(
"detection", schema=self.schema)
372 if self.config.doMeasurement:
373 self.makeSubtask(
"measurement", schema=self.schema,
374 algMetadata=self.algMetadata)
375 if self.config.doForcedMeasurement:
376 self.schema.addField(
377 "ip_diffim_forced_PsfFlux_instFlux",
"D",
378 "Forced PSF flux measured on the direct image.")
379 self.schema.addField(
380 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
381 "Forced PSF flux error measured on the direct image.")
382 self.schema.addField(
383 "ip_diffim_forced_PsfFlux_area",
"F",
384 "Forced PSF flux effective area of PSF.",
386 self.schema.addField(
387 "ip_diffim_forced_PsfFlux_flag",
"Flag",
388 "Forced PSF flux general failure flag.")
389 self.schema.addField(
390 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
391 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392 self.schema.addField(
393 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
394 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
396 if self.config.doMatchSources:
397 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
398 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
401 self.outputSchema = afwTable.SourceCatalog(self.schema)
402 self.outputSchema.getTable().setMetadata(self.algMetadata)
405 def makeIdFactory(expId, expBits):
406 """Create IdFactory instance for unique 64 bit diaSource id-s.
414 Number of used bits in ``expId``.
418 The diasource id-s consists of the ``expId`` stored fixed in the highest value
419 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420 low value end of the integer.
424 idFactory: `lsst.afw.table.IdFactory`
426 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
429 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430 inputRefs: pipeBase.InputQuantizedConnection,
431 outputRefs: pipeBase.OutputQuantizedConnection):
432 inputs = butlerQC.get(inputRefs)
433 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
434 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
436 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437 if self.config.coaddName ==
'dcr':
438 templateExposures = inputRefs.dcrCoadds
440 templateExposures = inputRefs.coaddExposures
441 templateStruct = self.getTemplate.runQuantum(
442 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
445 outputs = self.run(exposure=inputs[
'exposure'],
446 templateExposure=templateStruct.exposure,
448 butlerQC.put(outputs, outputRefs)
451 def runDataRef(self, sensorRef, templateIdList=None):
452 """Subtract an image from a template coadd and measure the result.
454 Data I/O wrapper around `run` using the butler in Gen2.
458 sensorRef : `lsst.daf.persistence.ButlerDataRef`
459 Sensor-level butler data reference, used for the following data products:
466 - self.config.coaddName + "Coadd_skyMap"
467 - self.config.coaddName + "Coadd"
468 Input or output, depending on config:
469 - self.config.coaddName + "Diff_subtractedExp"
470 Output, depending on config:
471 - self.config.coaddName + "Diff_matchedExp"
472 - self.config.coaddName + "Diff_src"
476 results : `lsst.pipe.base.Struct`
477 Returns the Struct by `run`.
479 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
480 subtractedExposure =
None
482 calexpBackgroundExposure =
None
483 self.log.info(
"Processing %s" % (sensorRef.dataId))
488 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
489 expBits=sensorRef.get(
"ccdExposureId_bits"))
490 if self.config.doAddCalexpBackground:
491 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
494 exposure = sensorRef.get(
"calexp", immediate=
True)
497 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
499 if sensorRef.datasetExists(
"src"):
500 self.log.info(
"Source selection via src product")
502 selectSources = sensorRef.get(
"src")
504 if not self.config.doSubtract
and self.config.doDetection:
506 subtractedExposure = sensorRef.get(subtractedExposureName)
509 results = self.run(exposure=exposure,
510 selectSources=selectSources,
511 templateExposure=template.exposure,
512 templateSources=template.sources,
514 calexpBackgroundExposure=calexpBackgroundExposure,
515 subtractedExposure=subtractedExposure)
517 if self.config.doWriteSources
and results.diaSources
is not None:
518 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
519 if self.config.doWriteWarpedExp:
520 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
521 if self.config.doWriteMatchedExp:
522 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
523 if self.config.doAddMetrics
and self.config.doSelectSources:
524 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
525 if self.config.doWriteSubtractedExp:
526 sensorRef.put(results.subtractedExposure, subtractedExposureName)
529 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
530 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
531 """PSF matches, subtract two images and perform detection on the difference image.
535 exposure : `lsst.afw.image.ExposureF`, optional
536 The science exposure, the minuend in the image subtraction.
537 Can be None only if ``config.doSubtract==False``.
538 selectSources : `lsst.afw.table.SourceCatalog`, optional
539 Identified sources on the science exposure. This catalog is used to
540 select sources in order to perform the AL PSF matching on stamp images
541 around them. The selection steps depend on config options and whether
542 ``templateSources`` and ``matchingSources`` specified.
543 templateExposure : `lsst.afw.image.ExposureF`, optional
544 The template to be subtracted from ``exposure`` in the image subtraction.
545 The template exposure should cover the same sky area as the science exposure.
546 It is either a stich of patches of a coadd skymap image or a calexp
547 of the same pointing as the science exposure. Can be None only
548 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
549 templateSources : `lsst.afw.table.SourceCatalog`, optional
550 Identified sources on the template exposure.
551 idFactory : `lsst.afw.table.IdFactory`
552 Generator object to assign ids to detected sources in the difference image.
553 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
554 Background exposure to be added back to the science exposure
555 if ``config.doAddCalexpBackground==True``
556 subtractedExposure : `lsst.afw.image.ExposureF`, optional
557 If ``config.doSubtract==False`` and ``config.doDetection==True``,
558 performs the post subtraction source detection only on this exposure.
559 Otherwise should be None.
563 results : `lsst.pipe.base.Struct`
564 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
566 ``matchedExposure`` : `lsst.afw.image.ExposureF`
567 The matched PSF exposure.
568 ``subtractRes`` : `lsst.pipe.base.Struct`
569 The returned result structure of the ImagePsfMatchTask subtask.
570 ``diaSources`` : `lsst.afw.table.SourceCatalog`
571 The catalog of detected sources.
572 ``selectSources`` : `lsst.afw.table.SourceCatalog`
573 The input source catalog with optionally added Qa information.
577 The following major steps are included:
579 - warp template coadd to match WCS of image
580 - PSF match image to warped template
581 - subtract image from PSF-matched, warped template
585 For details about the image subtraction configuration modes
586 see `lsst.ip.diffim`.
589 controlSources =
None
593 if self.config.doAddCalexpBackground:
594 mi = exposure.getMaskedImage()
595 mi += calexpBackgroundExposure.getImage()
597 if not exposure.hasPsf():
598 raise pipeBase.TaskError(
"Exposure has no psf")
599 sciencePsf = exposure.getPsf()
601 if self.config.doSubtract:
602 if self.config.doScaleTemplateVariance:
603 templateVarFactor = self.scaleVariance.
run(
604 templateExposure.getMaskedImage())
605 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
607 if self.config.subtract.name ==
'zogy':
608 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
609 if self.config.doPreConvolve:
610 subtractedExposure = subtractRes.scoreExp
612 subtractedExposure = subtractRes.diffExp
613 subtractRes.subtractedExposure = subtractedExposure
614 subtractRes.matchedExposure =
None
616 elif self.config.subtract.name ==
'al':
618 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
619 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
627 if self.config.doPreConvolve:
628 convControl = afwMath.ConvolutionControl()
630 srcMI = exposure.getMaskedImage()
631 exposureOrig = exposure.clone()
632 destMI = srcMI.Factory(srcMI.getDimensions())
634 if self.config.useGaussianForPreConvolution:
636 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
641 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
642 exposure.setMaskedImage(destMI)
643 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
645 scienceSigmaPost = scienceSigmaOrig
646 exposureOrig = exposure
651 if self.config.doSelectSources:
652 if selectSources
is None:
653 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
655 selectSources = self.subtract.getSelectSources(
657 sigma=scienceSigmaPost,
658 doSmooth=
not self.config.doPreConvolve,
662 if self.config.doAddMetrics:
665 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
666 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
667 targetFwhmPix=templateSigma*FwhmPerSigma))
674 kcQa = KernelCandidateQa(nparam)
675 selectSources = kcQa.addToSchema(selectSources)
676 if self.config.kernelSourcesFromRef:
678 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
679 matches = astromRet.matches
680 elif templateSources:
682 mc = afwTable.MatchControl()
683 mc.findOnlyClosest =
False
684 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
687 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
688 "but template sources not available. Cannot match science "
689 "sources with template sources. Run process* on data from "
690 "which templates are built.")
692 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
693 matches=matches).sourceCat
694 random.shuffle(kernelSources, random.random)
695 controlSources = kernelSources[::self.config.controlStepSize]
696 kernelSources = [k
for i, k
in enumerate(kernelSources)
697 if i % self.config.controlStepSize]
699 if self.config.doSelectDcrCatalog:
700 redSelector = DiaCatalogSourceSelectorTask(
701 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
703 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
704 controlSources.extend(redSources)
706 blueSelector = DiaCatalogSourceSelectorTask(
707 DiaCatalogSourceSelectorConfig(grMin=-99.999,
708 grMax=self.sourceSelector.config.grMin))
709 blueSources = blueSelector.selectStars(exposure, selectSources,
710 matches=matches).starCat
711 controlSources.extend(blueSources)
713 if self.config.doSelectVariableCatalog:
714 varSelector = DiaCatalogSourceSelectorTask(
715 DiaCatalogSourceSelectorConfig(includeVariable=
True))
716 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
717 controlSources.extend(varSources)
719 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
720 % (len(kernelSources), len(selectSources), len(controlSources)))
724 if self.config.doUseRegister:
725 self.log.info(
"Registering images")
727 if templateSources
is None:
731 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
732 templateSources = self.subtract.getSelectSources(
741 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
742 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
743 exposure.getWcs(), exposure.getBBox())
744 templateExposure = warpedExp
749 if self.config.doDebugRegister:
751 srcToMatch = {x.second.getId(): x.first
for x
in matches}
753 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
754 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
755 sids = [m.first.getId()
for m
in wcsResults.matches]
756 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
757 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
758 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
759 allresids = dict(zip(sids, zip(positions, residuals)))
761 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
762 wcsResults.wcs.pixelToSky(
763 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
764 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
765 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
766 for x
in sids
if x
in srcToMatch.keys()])
767 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
768 if s
in srcToMatch.keys()])
769 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
770 if s
in srcToMatch.keys()])
771 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
772 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
773 & (colors <= self.sourceSelector.config.grMax))
774 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
775 rms1Long = IqrToSigma*(
776 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
777 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
778 - numpy.percentile(dlat[idx1], 25))
779 rms2Long = IqrToSigma*(
780 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
781 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
782 - numpy.percentile(dlat[idx2], 25))
783 rms3Long = IqrToSigma*(
784 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
785 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
786 - numpy.percentile(dlat[idx3], 25))
787 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
788 (numpy.median(dlong[idx1]), rms1Long,
789 numpy.median(dlat[idx1]), rms1Lat))
790 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
791 (numpy.median(dlong[idx2]), rms2Long,
792 numpy.median(dlat[idx2]), rms2Lat))
793 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
794 (numpy.median(dlong[idx3]), rms3Long,
795 numpy.median(dlat[idx3]), rms3Lat))
797 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
798 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
799 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
800 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
801 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
802 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
804 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
805 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
806 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
807 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
808 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
809 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
816 self.log.info(
"Subtracting images")
817 subtractRes = self.subtract.subtractExposures(
818 templateExposure=templateExposure,
819 scienceExposure=exposure,
820 candidateList=kernelSources,
821 convolveTemplate=self.config.convolveTemplate,
822 doWarping=
not self.config.doUseRegister
824 subtractedExposure = subtractRes.subtractedExposure
826 if self.config.doDetection:
827 self.log.info(
"Computing diffim PSF")
830 if not subtractedExposure.hasPsf():
831 if self.config.convolveTemplate:
832 subtractedExposure.setPsf(exposure.getPsf())
834 subtractedExposure.setPsf(templateExposure.getPsf())
841 if self.config.doDecorrelation
and self.config.doSubtract:
843 if preConvPsf
is not None:
844 preConvKernel = preConvPsf.getLocalKernel()
845 if self.config.convolveTemplate:
846 self.log.info(
"Decorrelation after template image convolution")
847 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
849 subtractRes.psfMatchingKernel,
850 spatiallyVarying=self.config.doSpatiallyVarying,
851 preConvKernel=preConvKernel)
853 self.log.info(
"Decorrelation after science image convolution")
854 decorrResult = self.decorrelate.
run(subtractRes.warpedExposure, exposureOrig,
856 subtractRes.psfMatchingKernel,
857 spatiallyVarying=self.config.doSpatiallyVarying,
858 preConvKernel=preConvKernel)
859 subtractedExposure = decorrResult.correctedExposure
863 if self.config.doDetection:
864 self.log.info(
"Running diaSource detection")
866 mask = subtractedExposure.getMaskedImage().getMask()
867 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
869 table = afwTable.SourceTable.make(self.schema, idFactory)
870 table.setMetadata(self.algMetadata)
871 results = self.detection.
run(
873 exposure=subtractedExposure,
874 doSmooth=
not self.config.doPreConvolve
877 if self.config.doMerge:
878 fpSet = results.fpSets.positive
879 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
880 self.config.growFootprint,
False)
881 diaSources = afwTable.SourceCatalog(table)
882 fpSet.makeSources(diaSources)
883 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
885 diaSources = results.sources
887 if self.config.doMeasurement:
888 newDipoleFitting = self.config.doDipoleFitting
889 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
890 if not newDipoleFitting:
892 self.measurement.
run(diaSources, subtractedExposure)
895 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
896 self.measurement.
run(diaSources, subtractedExposure, exposure,
897 subtractRes.matchedExposure)
899 self.measurement.
run(diaSources, subtractedExposure, exposure)
901 if self.config.doForcedMeasurement:
904 forcedSources = self.forcedMeasurement.generateMeasCat(
905 exposure, diaSources, subtractedExposure.getWcs())
906 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
907 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
908 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
909 "ip_diffim_forced_PsfFlux_instFlux",
True)
910 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
911 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
912 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
913 "ip_diffim_forced_PsfFlux_area",
True)
914 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
915 "ip_diffim_forced_PsfFlux_flag",
True)
916 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
917 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
918 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
919 "ip_diffim_forced_PsfFlux_flag_edge",
True)
920 for diaSource, forcedSource
in zip(diaSources, forcedSources):
921 diaSource.assign(forcedSource, mapper)
924 if self.config.doMatchSources:
925 if selectSources
is not None:
927 matchRadAsec = self.config.diaSourceMatchRadius
928 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
930 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
931 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
932 srcMatch
in srcMatches])
933 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
936 self.log.warn(
"Src product does not exist; cannot match with diaSources")
940 refAstromConfig = AstrometryConfig()
941 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
942 refAstrometer = AstrometryTask(refAstromConfig)
943 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
944 refMatches = astromRet.matches
945 if refMatches
is None:
946 self.log.warn(
"No diaSource matches with reference catalog")
949 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
951 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
952 refMatch
in refMatches])
955 for diaSource
in diaSources:
956 sid = diaSource.getId()
957 if sid
in srcMatchDict:
958 diaSource.set(
"srcMatchId", srcMatchDict[sid])
959 if sid
in refMatchDict:
960 diaSource.set(
"refMatchId", refMatchDict[sid])
962 if self.config.doAddMetrics
and self.config.doSelectSources:
963 self.log.info(
"Evaluating metrics and control sample")
966 for cell
in subtractRes.kernelCellSet.getCellList():
967 for cand
in cell.begin(
False):
968 kernelCandList.append(cand)
971 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
972 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
975 diffimTools.sourceTableToCandidateList(controlSources,
976 subtractRes.warpedExposure, exposure,
977 self.config.subtract.kernel.active,
978 self.config.subtract.kernel.active.detectionConfig,
979 self.log, doBuild=
True, basisList=basisList))
981 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
982 subtractRes.backgroundModel, dof=nparam)
983 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
984 subtractRes.backgroundModel)
986 if self.config.doDetection:
987 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
989 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
991 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
992 return pipeBase.Struct(
993 subtractedExposure=subtractedExposure,
994 warpedExposure=subtractRes.warpedExposure,
995 matchedExposure=subtractRes.matchedExposure,
996 subtractRes=subtractRes,
997 diaSources=diaSources,
998 selectSources=selectSources
1001 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1002 """Fit the relative astrometry between templateSources and selectSources
1007 Remove this method. It originally fit a new WCS to the template before calling register.run
1008 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1009 It remains because a subtask overrides it.
1011 results = self.register.
run(templateSources, templateExposure.getWcs(),
1012 templateExposure.getBBox(), selectSources)
1015 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1016 """Make debug plots and displays.
1020 Test and update for current debug display and slot names
1030 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1031 if not maskTransparency:
1032 maskTransparency = 0
1033 disp.setMaskTransparency(maskTransparency)
1035 if display
and showSubtracted:
1036 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1037 mi = subtractRes.subtractedExposure.getMaskedImage()
1038 x0, y0 = mi.getX0(), mi.getY0()
1039 with disp.Buffering():
1040 for s
in diaSources:
1041 x, y = s.getX() - x0, s.getY() - y0
1042 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1043 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1044 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1045 or s.get(
"base_PixelFlags_flag_crCenter")):
1047 elif (s.get(
"base_PixelFlags_flag_interpolated")
1048 or s.get(
"base_PixelFlags_flag_saturated")
1049 or s.get(
"base_PixelFlags_flag_cr")):
1053 disp.dot(ptype, x, y, size=4, ctype=ctype)
1054 lsstDebug.frame += 1
1056 if display
and showPixelResiduals
and selectSources:
1057 nonKernelSources = []
1058 for source
in selectSources:
1059 if source
not in kernelSources:
1060 nonKernelSources.append(source)
1062 diUtils.plotPixelResiduals(exposure,
1063 subtractRes.warpedExposure,
1064 subtractRes.subtractedExposure,
1065 subtractRes.kernelCellSet,
1066 subtractRes.psfMatchingKernel,
1067 subtractRes.backgroundModel,
1069 self.subtract.config.kernel.active.detectionConfig,
1071 diUtils.plotPixelResiduals(exposure,
1072 subtractRes.warpedExposure,
1073 subtractRes.subtractedExposure,
1074 subtractRes.kernelCellSet,
1075 subtractRes.psfMatchingKernel,
1076 subtractRes.backgroundModel,
1078 self.subtract.config.kernel.active.detectionConfig,
1080 if display
and showDiaSources:
1081 flagChecker = SourceFlagChecker(diaSources)
1082 isFlagged = [flagChecker(x)
for x
in diaSources]
1083 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1084 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1085 frame=lsstDebug.frame)
1086 lsstDebug.frame += 1
1088 if display
and showDipoles:
1089 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1090 frame=lsstDebug.frame)
1091 lsstDebug.frame += 1
1093 def _getConfigName(self):
1094 """Return the name of the config dataset
1096 return "%sDiff_config" % (self.config.coaddName,)
1098 def _getMetadataName(self):
1099 """Return the name of the metadata dataset
1101 return "%sDiff_metadata" % (self.config.coaddName,)
1103 def getSchemaCatalogs(self):
1104 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1105 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1108 def _makeArgumentParser(cls):
1109 """Create an argument parser
1111 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1112 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1113 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1114 help=
"Template data ID in case of calexp template,"
1115 " e.g. --templateId visit=6789")
1119 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1120 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1121 doc=
"Shift stars going into RegisterTask by this amount")
1122 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1123 doc=
"Perturb stars going into RegisterTask by this amount")
1125 def setDefaults(self):
1126 ImageDifferenceConfig.setDefaults(self)
1127 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1130 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1131 """!Image difference Task used in the Winter 2013 data challege.
1132 Enables testing the effects of registration shifts and scatter.
1134 For use with winter 2013 simulated images:
1135 Use --templateId visit=88868666 for sparse data
1136 --templateId visit=22222200 for dense data (g)
1137 --templateId visit=11111100 for dense data (i)
1139 ConfigClass = Winter2013ImageDifferenceConfig
1140 _DefaultName =
"winter2013ImageDifference"
1142 def __init__(self, **kwargs):
1143 ImageDifferenceTask.__init__(self, **kwargs)
1145 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1146 """Fit the relative astrometry between templateSources and selectSources"""
1147 if self.config.winter2013WcsShift > 0.0:
1149 self.config.winter2013WcsShift)
1150 cKey = templateSources[0].getTable().getCentroidKey()
1151 for source
in templateSources:
1152 centroid = source.get(cKey)
1153 source.set(cKey, centroid + offset)
1154 elif self.config.winter2013WcsRms > 0.0:
1155 cKey = templateSources[0].getTable().getCentroidKey()
1156 for source
in templateSources:
1157 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1158 self.config.winter2013WcsRms*numpy.random.normal())
1159 centroid = source.get(cKey)
1160 source.set(cKey, centroid + offset)
1162 results = self.register.
run(templateSources, templateExposure.getWcs(),
1163 templateExposure.getBBox(), selectSources)