38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
48 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
54 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
55 defaultTemplates={
"coaddName":
"deep",
60 exposure = pipeBase.connectionTypes.Input(
61 doc=
"Input science exposure to subtract from.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}calexp"
75 skyMap = pipeBase.connectionTypes.Input(
76 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 dimensions=(
"skymap", ),
79 storageClass=
"SkyMap",
81 coaddExposures = pipeBase.connectionTypes.Input(
82 doc=
"Input template to match and subtract from the exposure",
83 dimensions=(
"tract",
"patch",
"skymap",
"band"),
84 storageClass=
"ExposureF",
85 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
89 dcrCoadds = pipeBase.connectionTypes.Input(
90 doc=
"Input DCR template to match and subtract from the exposure",
91 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
92 storageClass=
"ExposureF",
93 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
97 outputSchema = pipeBase.connectionTypes.InitOutput(
98 doc=
"Schema (as an example catalog) for output DIASource catalog.",
99 storageClass=
"SourceCatalog",
100 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
102 subtractedExposure = pipeBase.connectionTypes.Output(
103 doc=
"Output difference image",
104 dimensions=(
"instrument",
"visit",
"detector"),
105 storageClass=
"ExposureF",
106 name=
"{fakesType}{coaddName}Diff_differenceExp",
108 warpedExposure = pipeBase.connectionTypes.Output(
109 doc=
"Warped template used to create `subtractedExposure`.",
110 dimensions=(
"instrument",
"visit",
"detector"),
111 storageClass=
"ExposureF",
112 name=
"{fakesType}{coaddName}Diff_warpedExp",
114 diaSources = pipeBase.connectionTypes.Output(
115 doc=
"Output detected diaSources on the difference image",
116 dimensions=(
"instrument",
"visit",
"detector"),
117 storageClass=
"SourceCatalog",
118 name=
"{fakesType}{coaddName}Diff_diaSrc",
121 def __init__(self, *, config=None):
122 super().__init__(config=config)
123 if config.coaddName ==
'dcr':
124 self.inputs.remove(
"coaddExposures")
126 self.inputs.remove(
"dcrCoadds")
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133 pipelineConnections=ImageDifferenceTaskConnections):
134 """Config for ImageDifferenceTask
136 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
137 doc=
"Add background to calexp before processing it. "
138 "Useful as ipDiffim does background matching.")
139 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
140 doc=
"Use image-to-image registration to align template with "
142 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
143 doc=
"Writing debugging data for doUseRegister")
144 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
145 doc=
"Select stars to use for kernel fitting")
146 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
147 doc=
"Select stars of extreme color as part of the control sample")
148 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
149 doc=
"Select stars that are variable to be part "
150 "of the control sample")
151 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
152 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
153 doc=
"Convolve science image by its PSF before PSF-matching?")
154 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
155 doc=
"Scale variance of the template before PSF matching")
156 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=
False,
157 doc=
"Scale variance of the diffim before PSF matching. "
158 "You may do either this or template variance scaling, "
159 "or neither. (Doing both is a waste of CPU.)")
160 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
161 doc=
"Use a simple gaussian PSF model for pre-convolution "
162 "(else use fit PSF)? Ignored if doPreConvolve false.")
163 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
164 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
165 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
166 "kernel convolution? If True, also update the diffim PSF.")
167 doMerge = pexConfig.Field(dtype=bool, default=
True,
168 doc=
"Merge positive and negative diaSources with grow radius "
169 "set by growFootprint")
170 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
171 doc=
"Match diaSources with input calexp sources and ref catalog sources")
172 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
173 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
174 doForcedMeasurement = pexConfig.Field(
177 doc=
"Force photometer diaSource locations on PVI?")
178 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
179 doWriteWarpedExp = pexConfig.Field(dtype=bool, default=
False,
180 doc=
"Write WCS, warped template coadd exposure?")
181 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
182 doc=
"Write warped and PSF-matched template coadd exposure?")
183 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
184 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
185 doc=
"Add columns to the source table to hold analysis metrics?")
187 coaddName = pexConfig.Field(
188 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
192 convolveTemplate = pexConfig.Field(
193 doc=
"Which image gets convolved (default = template)",
197 refObjLoader = pexConfig.ConfigurableField(
198 target=LoadIndexedReferenceObjectsTask,
199 doc=
"reference object loader",
201 astrometer = pexConfig.ConfigurableField(
202 target=AstrometryTask,
203 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
205 sourceSelector = pexConfig.ConfigurableField(
206 target=ObjectSizeStarSelectorTask,
207 doc=
"Source selection algorithm",
209 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
210 decorrelate = pexConfig.ConfigurableField(
211 target=DecorrelateALKernelSpatialTask,
212 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
216 doSpatiallyVarying = pexConfig.Field(
219 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
220 "image in order to allow for spatial variations"
222 detection = pexConfig.ConfigurableField(
223 target=SourceDetectionTask,
224 doc=
"Low-threshold detection for final measurement",
226 measurement = pexConfig.ConfigurableField(
227 target=DipoleFitTask,
228 doc=
"Enable updated dipole fitting method",
230 forcedMeasurement = pexConfig.ConfigurableField(
231 target=ForcedMeasurementTask,
232 doc=
"Subtask to force photometer PVI at diaSource location.",
234 getTemplate = pexConfig.ConfigurableField(
235 target=GetCoaddAsTemplateTask,
236 doc=
"Subtask to retrieve template exposure and sources",
238 scaleVariance = pexConfig.ConfigurableField(
239 target=ScaleVarianceTask,
240 doc=
"Subtask to rescale the variance of the template "
241 "to the statistically expected level"
243 controlStepSize = pexConfig.Field(
244 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
248 controlRandomSeed = pexConfig.Field(
249 doc=
"Random seed for shuffing the control sample",
253 register = pexConfig.ConfigurableField(
255 doc=
"Task to enable image-to-image image registration (warping)",
257 kernelSourcesFromRef = pexConfig.Field(
258 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
262 templateSipOrder = pexConfig.Field(
263 dtype=int, default=2,
264 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
266 growFootprint = pexConfig.Field(
267 dtype=int, default=2,
268 doc=
"Grow positive and negative footprints by this amount before merging"
270 diaSourceMatchRadius = pexConfig.Field(
271 dtype=float, default=0.5,
272 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
275 def setDefaults(self):
278 self.subtract[
'al'].kernel.name =
"AL"
279 self.subtract[
'al'].kernel.active.fitForBackground =
True
280 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
281 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
282 self.doPreConvolve =
False
283 self.doMatchSources =
False
284 self.doAddMetrics =
False
285 self.doUseRegister =
False
288 self.detection.thresholdPolarity =
"both"
289 self.detection.thresholdValue = 5.5
290 self.detection.reEstimateBackground =
False
291 self.detection.thresholdType =
"pixel_stdev"
297 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
298 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
301 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
302 self.forcedMeasurement.copyColumns = {
303 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
304 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
305 self.forcedMeasurement.slots.shape =
None
308 random.seed(self.controlRandomSeed)
311 pexConfig.Config.validate(self)
312 if self.doAddMetrics
and not self.doSubtract:
313 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
314 if not self.doSubtract
and not self.doDetection:
315 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
316 if self.subtract.name ==
'zogy' and self.doAddMetrics:
317 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
318 if self.doMeasurement
and not self.doDetection:
319 raise ValueError(
"Cannot run source measurement without source detection.")
320 if self.doMerge
and not self.doDetection:
321 raise ValueError(
"Cannot run source merging without source detection.")
322 if self.doUseRegister
and not self.doSelectSources:
323 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
324 "Cannot run RegisterTask without selecting sources.")
325 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
326 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
327 "convolveTemplate=False is not supported.")
328 if hasattr(self.getTemplate,
"coaddName"):
329 if self.getTemplate.coaddName != self.coaddName:
330 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
331 if self.doScaleDiffimVariance
and self.doScaleTemplateVariance:
332 raise ValueError(
"Scaling the diffim variance and scaling the template variance "
333 "are both set. Please choose one or the other.")
336 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
339 def getTargetList(parsedCmd, **kwargs):
340 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
344 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
345 """Subtract an image from a template and measure the result
347 ConfigClass = ImageDifferenceConfig
348 RunnerClass = ImageDifferenceTaskRunner
349 _DefaultName =
"imageDifference"
351 def __init__(self, butler=None, **kwargs):
352 """!Construct an ImageDifference Task
354 @param[in] butler Butler object to use in constructing reference object loaders
356 super().__init__(**kwargs)
357 self.makeSubtask(
"getTemplate")
359 self.makeSubtask(
"subtract")
361 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
362 self.makeSubtask(
"decorrelate")
364 if self.config.doScaleTemplateVariance
or self.config.doScaleDiffimVariance:
365 self.makeSubtask(
"scaleVariance")
367 if self.config.doUseRegister:
368 self.makeSubtask(
"register")
369 self.schema = afwTable.SourceTable.makeMinimalSchema()
371 if self.config.doSelectSources:
372 self.makeSubtask(
"sourceSelector")
373 if self.config.kernelSourcesFromRef:
374 self.makeSubtask(
'refObjLoader', butler=butler)
375 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
377 self.algMetadata = dafBase.PropertyList()
378 if self.config.doDetection:
379 self.makeSubtask(
"detection", schema=self.schema)
380 if self.config.doMeasurement:
381 self.makeSubtask(
"measurement", schema=self.schema,
382 algMetadata=self.algMetadata)
383 if self.config.doForcedMeasurement:
384 self.schema.addField(
385 "ip_diffim_forced_PsfFlux_instFlux",
"D",
386 "Forced PSF flux measured on the direct image.",
388 self.schema.addField(
389 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
390 "Forced PSF flux error measured on the direct image.",
392 self.schema.addField(
393 "ip_diffim_forced_PsfFlux_area",
"F",
394 "Forced PSF flux effective area of PSF.",
396 self.schema.addField(
397 "ip_diffim_forced_PsfFlux_flag",
"Flag",
398 "Forced PSF flux general failure flag.")
399 self.schema.addField(
400 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
401 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
402 self.schema.addField(
403 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
404 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
405 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
406 if self.config.doMatchSources:
407 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
408 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
411 self.outputSchema = afwTable.SourceCatalog(self.schema)
412 self.outputSchema.getTable().setMetadata(self.algMetadata)
415 def makeIdFactory(expId, expBits):
416 """Create IdFactory instance for unique 64 bit diaSource id-s.
424 Number of used bits in ``expId``.
428 The diasource id-s consists of the ``expId`` stored fixed in the highest value
429 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
430 low value end of the integer.
434 idFactory: `lsst.afw.table.IdFactory`
436 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
438 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
439 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
440 inputRefs: pipeBase.InputQuantizedConnection,
441 outputRefs: pipeBase.OutputQuantizedConnection):
442 inputs = butlerQC.get(inputRefs)
443 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
444 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
446 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
447 if self.config.coaddName ==
'dcr':
448 templateExposures = inputRefs.dcrCoadds
450 templateExposures = inputRefs.coaddExposures
451 templateStruct = self.getTemplate.runQuantum(
452 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
455 outputs = self.run(exposure=inputs[
'exposure'],
456 templateExposure=templateStruct.exposure,
458 butlerQC.put(outputs, outputRefs)
461 def runDataRef(self, sensorRef, templateIdList=None):
462 """Subtract an image from a template coadd and measure the result.
464 Data I/O wrapper around `run` using the butler in Gen2.
468 sensorRef : `lsst.daf.persistence.ButlerDataRef`
469 Sensor-level butler data reference, used for the following data products:
476 - self.config.coaddName + "Coadd_skyMap"
477 - self.config.coaddName + "Coadd"
478 Input or output, depending on config:
479 - self.config.coaddName + "Diff_subtractedExp"
480 Output, depending on config:
481 - self.config.coaddName + "Diff_matchedExp"
482 - self.config.coaddName + "Diff_src"
486 results : `lsst.pipe.base.Struct`
487 Returns the Struct by `run`.
489 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
490 subtractedExposure =
None
492 calexpBackgroundExposure =
None
493 self.log.info(
"Processing %s" % (sensorRef.dataId))
498 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
499 expBits=sensorRef.get(
"ccdExposureId_bits"))
500 if self.config.doAddCalexpBackground:
501 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
504 exposure = sensorRef.get(
"calexp", immediate=
True)
507 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
509 if sensorRef.datasetExists(
"src"):
510 self.log.info(
"Source selection via src product")
512 selectSources = sensorRef.get(
"src")
514 if not self.config.doSubtract
and self.config.doDetection:
516 subtractedExposure = sensorRef.get(subtractedExposureName)
519 results = self.run(exposure=exposure,
520 selectSources=selectSources,
521 templateExposure=template.exposure,
522 templateSources=template.sources,
524 calexpBackgroundExposure=calexpBackgroundExposure,
525 subtractedExposure=subtractedExposure)
527 if self.config.doWriteSources
and results.diaSources
is not None:
528 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
529 if self.config.doWriteWarpedExp:
530 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
531 if self.config.doWriteMatchedExp:
532 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
533 if self.config.doAddMetrics
and self.config.doSelectSources:
534 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
535 if self.config.doWriteSubtractedExp:
536 sensorRef.put(results.subtractedExposure, subtractedExposureName)
540 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
541 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
542 """PSF matches, subtract two images and perform detection on the difference image.
546 exposure : `lsst.afw.image.ExposureF`, optional
547 The science exposure, the minuend in the image subtraction.
548 Can be None only if ``config.doSubtract==False``.
549 selectSources : `lsst.afw.table.SourceCatalog`, optional
550 Identified sources on the science exposure. This catalog is used to
551 select sources in order to perform the AL PSF matching on stamp images
552 around them. The selection steps depend on config options and whether
553 ``templateSources`` and ``matchingSources`` specified.
554 templateExposure : `lsst.afw.image.ExposureF`, optional
555 The template to be subtracted from ``exposure`` in the image subtraction.
556 The template exposure should cover the same sky area as the science exposure.
557 It is either a stich of patches of a coadd skymap image or a calexp
558 of the same pointing as the science exposure. Can be None only
559 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
560 templateSources : `lsst.afw.table.SourceCatalog`, optional
561 Identified sources on the template exposure.
562 idFactory : `lsst.afw.table.IdFactory`
563 Generator object to assign ids to detected sources in the difference image.
564 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
565 Background exposure to be added back to the science exposure
566 if ``config.doAddCalexpBackground==True``
567 subtractedExposure : `lsst.afw.image.ExposureF`, optional
568 If ``config.doSubtract==False`` and ``config.doDetection==True``,
569 performs the post subtraction source detection only on this exposure.
570 Otherwise should be None.
574 results : `lsst.pipe.base.Struct`
575 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
577 ``matchedExposure`` : `lsst.afw.image.ExposureF`
578 The matched PSF exposure.
579 ``subtractRes`` : `lsst.pipe.base.Struct`
580 The returned result structure of the ImagePsfMatchTask subtask.
581 ``diaSources`` : `lsst.afw.table.SourceCatalog`
582 The catalog of detected sources.
583 ``selectSources`` : `lsst.afw.table.SourceCatalog`
584 The input source catalog with optionally added Qa information.
588 The following major steps are included:
590 - warp template coadd to match WCS of image
591 - PSF match image to warped template
592 - subtract image from PSF-matched, warped template
596 For details about the image subtraction configuration modes
597 see `lsst.ip.diffim`.
600 controlSources =
None
604 if self.config.doAddCalexpBackground:
605 mi = exposure.getMaskedImage()
606 mi += calexpBackgroundExposure.getImage()
608 if not exposure.hasPsf():
609 raise pipeBase.TaskError(
"Exposure has no psf")
610 sciencePsf = exposure.getPsf()
612 if self.config.doSubtract:
613 if self.config.doScaleTemplateVariance:
614 self.log.info(
"Rescaling template variance")
615 templateVarFactor = self.scaleVariance.
run(
616 templateExposure.getMaskedImage())
617 self.log.info(
"Template variance scaling factor: %.2f" % templateVarFactor)
618 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
620 if self.config.subtract.name ==
'zogy':
621 subtractRes = self.subtract.
run(exposure, templateExposure, doWarping=
True)
622 if self.config.doPreConvolve:
623 subtractedExposure = subtractRes.scoreExp
625 subtractedExposure = subtractRes.diffExp
626 subtractRes.subtractedExposure = subtractedExposure
627 subtractRes.matchedExposure =
None
629 elif self.config.subtract.name ==
'al':
631 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
632 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
640 if self.config.doPreConvolve:
641 convControl = afwMath.ConvolutionControl()
643 srcMI = exposure.getMaskedImage()
644 exposureOrig = exposure.clone()
645 destMI = srcMI.Factory(srcMI.getDimensions())
647 if self.config.useGaussianForPreConvolution:
649 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
654 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
655 exposure.setMaskedImage(destMI)
656 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
658 scienceSigmaPost = scienceSigmaOrig
659 exposureOrig = exposure
664 if self.config.doSelectSources:
665 if selectSources
is None:
666 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
668 selectSources = self.subtract.getSelectSources(
670 sigma=scienceSigmaPost,
671 doSmooth=
not self.config.doPreConvolve,
675 if self.config.doAddMetrics:
678 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
679 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
680 targetFwhmPix=templateSigma*FwhmPerSigma))
687 kcQa = KernelCandidateQa(nparam)
688 selectSources = kcQa.addToSchema(selectSources)
689 if self.config.kernelSourcesFromRef:
691 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
692 matches = astromRet.matches
693 elif templateSources:
695 mc = afwTable.MatchControl()
696 mc.findOnlyClosest =
False
697 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
700 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
701 "but template sources not available. Cannot match science "
702 "sources with template sources. Run process* on data from "
703 "which templates are built.")
705 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
706 matches=matches).sourceCat
707 random.shuffle(kernelSources, random.random)
708 controlSources = kernelSources[::self.config.controlStepSize]
709 kernelSources = [k
for i, k
in enumerate(kernelSources)
710 if i % self.config.controlStepSize]
712 if self.config.doSelectDcrCatalog:
713 redSelector = DiaCatalogSourceSelectorTask(
714 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
716 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
717 controlSources.extend(redSources)
719 blueSelector = DiaCatalogSourceSelectorTask(
720 DiaCatalogSourceSelectorConfig(grMin=-99.999,
721 grMax=self.sourceSelector.config.grMin))
722 blueSources = blueSelector.selectStars(exposure, selectSources,
723 matches=matches).starCat
724 controlSources.extend(blueSources)
726 if self.config.doSelectVariableCatalog:
727 varSelector = DiaCatalogSourceSelectorTask(
728 DiaCatalogSourceSelectorConfig(includeVariable=
True))
729 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
730 controlSources.extend(varSources)
732 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
733 % (len(kernelSources), len(selectSources), len(controlSources)))
737 if self.config.doUseRegister:
738 self.log.info(
"Registering images")
740 if templateSources
is None:
744 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
745 templateSources = self.subtract.getSelectSources(
754 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
755 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
756 exposure.getWcs(), exposure.getBBox())
757 templateExposure = warpedExp
762 if self.config.doDebugRegister:
764 srcToMatch = {x.second.getId(): x.first
for x
in matches}
766 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
767 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
768 sids = [m.first.getId()
for m
in wcsResults.matches]
769 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
770 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
771 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
772 allresids = dict(zip(sids, zip(positions, residuals)))
774 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
775 wcsResults.wcs.pixelToSky(
776 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
777 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
778 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
779 for x
in sids
if x
in srcToMatch.keys()])
780 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
781 if s
in srcToMatch.keys()])
782 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
783 if s
in srcToMatch.keys()])
784 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
785 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
786 & (colors <= self.sourceSelector.config.grMax))
787 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
788 rms1Long = IqrToSigma*(
789 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
790 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
791 - numpy.percentile(dlat[idx1], 25))
792 rms2Long = IqrToSigma*(
793 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
794 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
795 - numpy.percentile(dlat[idx2], 25))
796 rms3Long = IqrToSigma*(
797 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
798 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
799 - numpy.percentile(dlat[idx3], 25))
800 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
801 (numpy.median(dlong[idx1]), rms1Long,
802 numpy.median(dlat[idx1]), rms1Lat))
803 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
804 (numpy.median(dlong[idx2]), rms2Long,
805 numpy.median(dlat[idx2]), rms2Lat))
806 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
807 (numpy.median(dlong[idx3]), rms3Long,
808 numpy.median(dlat[idx3]), rms3Lat))
810 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
811 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
812 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
813 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
814 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
815 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
817 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
818 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
819 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
820 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
821 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
822 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
829 self.log.info(
"Subtracting images")
830 subtractRes = self.subtract.subtractExposures(
831 templateExposure=templateExposure,
832 scienceExposure=exposure,
833 candidateList=kernelSources,
834 convolveTemplate=self.config.convolveTemplate,
835 doWarping=
not self.config.doUseRegister
837 subtractedExposure = subtractRes.subtractedExposure
839 if self.config.doDetection:
840 self.log.info(
"Computing diffim PSF")
843 if not subtractedExposure.hasPsf():
844 if self.config.convolveTemplate:
845 subtractedExposure.setPsf(exposure.getPsf())
847 subtractedExposure.setPsf(templateExposure.getPsf())
854 if self.config.doDecorrelation
and self.config.doSubtract:
856 if preConvPsf
is not None:
857 preConvKernel = preConvPsf.getLocalKernel()
858 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
860 subtractRes.psfMatchingKernel,
861 spatiallyVarying=self.config.doSpatiallyVarying,
862 preConvKernel=preConvKernel,
863 templateMatched=self.config.convolveTemplate)
864 subtractedExposure = decorrResult.correctedExposure
868 if self.config.doDetection:
869 self.log.info(
"Running diaSource detection")
872 if self.config.doScaleDiffimVariance:
873 self.log.info(
"Rescaling diffim variance")
874 diffimVarFactor = self.scaleVariance.
run(subtractedExposure.getMaskedImage())
875 self.log.info(
"Diffim variance scaling factor: %.2f" % diffimVarFactor)
876 self.metadata.add(
"scaleDiffimVarianceFactor", diffimVarFactor)
879 mask = subtractedExposure.getMaskedImage().getMask()
880 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
882 table = afwTable.SourceTable.make(self.schema, idFactory)
883 table.setMetadata(self.algMetadata)
884 results = self.detection.
run(
886 exposure=subtractedExposure,
887 doSmooth=
not self.config.doPreConvolve
890 if self.config.doMerge:
891 fpSet = results.fpSets.positive
892 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
893 self.config.growFootprint,
False)
894 diaSources = afwTable.SourceCatalog(table)
895 fpSet.makeSources(diaSources)
896 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
898 diaSources = results.sources
900 if self.config.doMeasurement:
901 newDipoleFitting = self.config.doDipoleFitting
902 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
903 if not newDipoleFitting:
905 self.measurement.
run(diaSources, subtractedExposure)
908 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
909 self.measurement.
run(diaSources, subtractedExposure, exposure,
910 subtractRes.matchedExposure)
912 self.measurement.
run(diaSources, subtractedExposure, exposure)
914 if self.config.doForcedMeasurement:
917 forcedSources = self.forcedMeasurement.generateMeasCat(
918 exposure, diaSources, subtractedExposure.getWcs())
919 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
920 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
921 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
922 "ip_diffim_forced_PsfFlux_instFlux",
True)
923 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
924 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
925 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
926 "ip_diffim_forced_PsfFlux_area",
True)
927 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
928 "ip_diffim_forced_PsfFlux_flag",
True)
929 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
930 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
931 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
932 "ip_diffim_forced_PsfFlux_flag_edge",
True)
933 for diaSource, forcedSource
in zip(diaSources, forcedSources):
934 diaSource.assign(forcedSource, mapper)
937 if self.config.doMatchSources:
938 if selectSources
is not None:
940 matchRadAsec = self.config.diaSourceMatchRadius
941 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
943 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
944 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
945 srcMatch
in srcMatches])
946 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
949 self.log.warn(
"Src product does not exist; cannot match with diaSources")
953 refAstromConfig = AstrometryConfig()
954 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
955 refAstrometer = AstrometryTask(refAstromConfig)
956 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
957 refMatches = astromRet.matches
958 if refMatches
is None:
959 self.log.warn(
"No diaSource matches with reference catalog")
962 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
964 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
965 refMatch
in refMatches])
968 for diaSource
in diaSources:
969 sid = diaSource.getId()
970 if sid
in srcMatchDict:
971 diaSource.set(
"srcMatchId", srcMatchDict[sid])
972 if sid
in refMatchDict:
973 diaSource.set(
"refMatchId", refMatchDict[sid])
975 if self.config.doAddMetrics
and self.config.doSelectSources:
976 self.log.info(
"Evaluating metrics and control sample")
979 for cell
in subtractRes.kernelCellSet.getCellList():
980 for cand
in cell.begin(
False):
981 kernelCandList.append(cand)
984 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
985 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
988 diffimTools.sourceTableToCandidateList(controlSources,
989 subtractRes.warpedExposure, exposure,
990 self.config.subtract.kernel.active,
991 self.config.subtract.kernel.active.detectionConfig,
992 self.log, doBuild=
True, basisList=basisList))
994 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
995 subtractRes.backgroundModel, dof=nparam)
996 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
997 subtractRes.backgroundModel)
999 if self.config.doDetection:
1000 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1002 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1004 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1005 return pipeBase.Struct(
1006 subtractedExposure=subtractedExposure,
1007 warpedExposure=subtractRes.warpedExposure,
1008 matchedExposure=subtractRes.matchedExposure,
1009 subtractRes=subtractRes,
1010 diaSources=diaSources,
1011 selectSources=selectSources
1014 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1015 """Fit the relative astrometry between templateSources and selectSources
1020 Remove this method. It originally fit a new WCS to the template before calling register.run
1021 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1022 It remains because a subtask overrides it.
1024 results = self.register.
run(templateSources, templateExposure.getWcs(),
1025 templateExposure.getBBox(), selectSources)
1028 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1029 """Make debug plots and displays.
1033 Test and update for current debug display and slot names
1043 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1044 if not maskTransparency:
1045 maskTransparency = 0
1046 disp.setMaskTransparency(maskTransparency)
1048 if display
and showSubtracted:
1049 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1050 mi = subtractRes.subtractedExposure.getMaskedImage()
1051 x0, y0 = mi.getX0(), mi.getY0()
1052 with disp.Buffering():
1053 for s
in diaSources:
1054 x, y = s.getX() - x0, s.getY() - y0
1055 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1056 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1057 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1058 or s.get(
"base_PixelFlags_flag_crCenter")):
1060 elif (s.get(
"base_PixelFlags_flag_interpolated")
1061 or s.get(
"base_PixelFlags_flag_saturated")
1062 or s.get(
"base_PixelFlags_flag_cr")):
1066 disp.dot(ptype, x, y, size=4, ctype=ctype)
1067 lsstDebug.frame += 1
1069 if display
and showPixelResiduals
and selectSources:
1070 nonKernelSources = []
1071 for source
in selectSources:
1072 if source
not in kernelSources:
1073 nonKernelSources.append(source)
1075 diUtils.plotPixelResiduals(exposure,
1076 subtractRes.warpedExposure,
1077 subtractRes.subtractedExposure,
1078 subtractRes.kernelCellSet,
1079 subtractRes.psfMatchingKernel,
1080 subtractRes.backgroundModel,
1082 self.subtract.config.kernel.active.detectionConfig,
1084 diUtils.plotPixelResiduals(exposure,
1085 subtractRes.warpedExposure,
1086 subtractRes.subtractedExposure,
1087 subtractRes.kernelCellSet,
1088 subtractRes.psfMatchingKernel,
1089 subtractRes.backgroundModel,
1091 self.subtract.config.kernel.active.detectionConfig,
1093 if display
and showDiaSources:
1094 flagChecker = SourceFlagChecker(diaSources)
1095 isFlagged = [flagChecker(x)
for x
in diaSources]
1096 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1097 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1098 frame=lsstDebug.frame)
1099 lsstDebug.frame += 1
1101 if display
and showDipoles:
1102 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1103 frame=lsstDebug.frame)
1104 lsstDebug.frame += 1
1106 def _getConfigName(self):
1107 """Return the name of the config dataset
1109 return "%sDiff_config" % (self.config.coaddName,)
1111 def _getMetadataName(self):
1112 """Return the name of the metadata dataset
1114 return "%sDiff_metadata" % (self.config.coaddName,)
1116 def getSchemaCatalogs(self):
1117 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1118 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1121 def _makeArgumentParser(cls):
1122 """Create an argument parser
1124 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1125 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1126 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1127 help=
"Template data ID in case of calexp template,"
1128 " e.g. --templateId visit=6789")
1132 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1133 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1134 doc=
"Shift stars going into RegisterTask by this amount")
1135 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1136 doc=
"Perturb stars going into RegisterTask by this amount")
1138 def setDefaults(self):
1139 ImageDifferenceConfig.setDefaults(self)
1140 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1143 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1144 """!Image difference Task used in the Winter 2013 data challege.
1145 Enables testing the effects of registration shifts and scatter.
1147 For use with winter 2013 simulated images:
1148 Use --templateId visit=88868666 for sparse data
1149 --templateId visit=22222200 for dense data (g)
1150 --templateId visit=11111100 for dense data (i)
1152 ConfigClass = Winter2013ImageDifferenceConfig
1153 _DefaultName =
"winter2013ImageDifference"
1155 def __init__(self, **kwargs):
1156 ImageDifferenceTask.__init__(self, **kwargs)
1158 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1159 """Fit the relative astrometry between templateSources and selectSources"""
1160 if self.config.winter2013WcsShift > 0.0:
1162 self.config.winter2013WcsShift)
1163 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1164 for source
in templateSources:
1165 centroid = source.get(cKey)
1166 source.set(cKey, centroid + offset)
1167 elif self.config.winter2013WcsRms > 0.0:
1168 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1169 for source
in templateSources:
1170 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1171 self.config.winter2013WcsRms*numpy.random.normal())
1172 centroid = source.get(cKey)
1173 source.set(cKey, centroid + offset)
1175 results = self.register.
run(templateSources, templateExposure.getWcs(),
1176 templateExposure.getBBox(), selectSources)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)