28 import lsst.meas.algorithms
as measAlg
30 import lsst.pex.config
as pexConfig
31 import lsst.pipe.base
as pipeBase
33 from .imageMapReduce
import (ImageMapReduceConfig, ImageMapper,
35 from .imagePsfMatch
import (ImagePsfMatchTask, ImagePsfMatchConfig,
36 subtractAlgorithmRegistry)
38 __all__ = [
"ZogyTask",
"ZogyConfig",
39 "ZogyMapper",
"ZogyMapReduceConfig",
40 "ZogyImagePsfMatchConfig",
"ZogyImagePsfMatchTask"]
43 """Tasks for performing the "Proper image subtraction" algorithm of 44 Zackay, et al. (2016), hereafter simply referred to as 'ZOGY (2016)'. 46 `ZogyTask` contains methods to perform the basic estimation of the 47 ZOGY diffim `D`, its updated PSF, and the variance-normalized 48 likelihood image `S_corr`. We have implemented ZOGY using the 49 proscribed methodology, computing all convolutions in Fourier space, 50 and also variants in which the convolutions are performed in real 51 (image) space. The former is faster and results in fewer artifacts 52 when the PSFs are noisy (i.e., measured, for example, via 53 `PsfEx`). The latter is presumed to be preferred as it can account for 54 masks correctly with fewer "ringing" artifacts from edge effects or 55 saturated stars, but noisy PSFs result in their own smaller 56 artifacts. Removal of these artifacts is a subject of continuing 57 research. Currently, we "pad" the PSFs when performing the 58 subtractions in real space, which reduces, but does not entirely 59 eliminate these artifacts. 61 All methods in `ZogyTask` assume template and science images are 62 already accurately photometrically and astrometrically registered. 64 `ZogyMapper` is a wrapper which runs `ZogyTask` in the 65 `ImageMapReduce` framework, computing of ZOGY diffim's on small, 66 overlapping sub-images, thereby enabling complete ZOGY diffim's which 67 account for spatially-varying noise and PSFs across the two input 68 exposures. An example of the use of this task is in the `testZogy.py` 74 """Configuration parameters for the ZogyTask 76 inImageSpace = pexConfig.Field(
79 doc=
"Perform all convolutions in real (image) space rather than Fourier space. " 80 "Currently if True, this results in artifacts when using real (noisy) PSFs." 83 padSize = pexConfig.Field(
86 doc=
"Number of pixels to pad PSFs to avoid artifacts (when inImageSpace is True)" 89 templateFluxScaling = pexConfig.Field(
92 doc=
"Template flux scaling factor (Fr in ZOGY paper)" 95 scienceFluxScaling = pexConfig.Field(
98 doc=
"Science flux scaling factor (Fn in ZOGY paper)" 101 scaleByCalibration = pexConfig.Field(
104 doc=
"Compute the flux normalization scaling based on the image calibration." 105 "This overrides 'templateFluxScaling' and 'scienceFluxScaling'." 108 doTrimKernels = pexConfig.Field(
111 doc=
"Trim kernels for image-space ZOGY. Speeds up convolutions and shrinks artifacts. " 112 "Subject of future research." 115 doFilterPsfs = pexConfig.Field(
118 doc=
"Filter PSFs for image-space ZOGY. Aids in reducing artifacts. " 119 "Subject of future research." 122 ignoreMaskPlanes = pexConfig.ListField(
124 default=(
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE"),
125 doc=
"Mask planes to ignore for statistics" 133 """Task to perform ZOGY proper image subtraction. See module-level documentation for 136 In all methods, im1 is R (reference, or template) and im2 is N (new, or science). 138 ConfigClass = ZogyConfig
139 _DefaultName =
"ip_diffim_Zogy" 141 def __init__(self, templateExposure=None, scienceExposure=None, sig1=None, sig2=None,
142 psf1=None, psf2=None, *args, **kwargs):
143 """Create the ZOGY task. 147 templateExposure : `lsst.afw.image.Exposure` 148 Template exposure ("Reference image" in ZOGY (2016)). 149 scienceExposure : `lsst.afw.image.Exposure` 150 Science exposure ("New image" in ZOGY (2016)). Must have already been 151 registered and photmetrically matched to template. 153 (Optional) sqrt(variance) of `templateExposure`. If `None`, it is 154 computed from the sqrt(mean) of the `templateExposure` variance image. 156 (Optional) sqrt(variance) of `scienceExposure`. If `None`, it is 157 computed from the sqrt(mean) of the `scienceExposure` variance image. 158 psf1 : 2D `numpy.array` 159 (Optional) 2D array containing the PSF image for the template. If 160 `None`, it is extracted from the PSF taken at the center of `templateExposure`. 161 psf2 : 2D `numpy.array` 162 (Optional) 2D array containing the PSF image for the science img. If 163 `None`, it is extracted from the PSF taken at the center of `scienceExposure`. 165 additional arguments to be passed to 166 `lsst.pipe.base.Task` 168 additional keyword arguments to be passed to 169 `lsst.pipe.base.Task` 171 pipeBase.Task.__init__(self, *args, **kwargs)
173 self.
setup(templateExposure=templateExposure, scienceExposure=scienceExposure,
174 sig1=sig1, sig2=sig2, psf1=psf1, psf2=psf2, *args, **kwargs)
176 def setup(self, templateExposure=None, scienceExposure=None, sig1=None, sig2=None,
177 psf1=None, psf2=None, correctBackground=False, *args, **kwargs):
178 """Set up the ZOGY task. 182 templateExposure : `lsst.afw.image.Exposure` 183 Template exposure ("Reference image" in ZOGY (2016)). 184 scienceExposure : `lsst.afw.image.Exposure` 185 Science exposure ("New image" in ZOGY (2016)). Must have already been 186 registered and photometrically matched to template. 188 (Optional) sqrt(variance) of `templateExposure`. If `None`, it is 189 computed from the sqrt(mean) of the `templateExposure` variance image. 191 (Optional) sqrt(variance) of `scienceExposure`. If `None`, it is 192 computed from the sqrt(mean) of the `scienceExposure` variance image. 193 psf1 : 2D `numpy.array` 194 (Optional) 2D array containing the PSF image for the template. If 195 `None`, it is extracted from the PSF taken at the center of `templateExposure`. 196 psf2 : 2D `numpy.array` 197 (Optional) 2D array containing the PSF image for the science img. If 198 `None`, it is extracted from the PSF taken at the center of `scienceExposure`. 199 correctBackground : `bool` 200 (Optional) subtract sigma-clipped mean of exposures. Zogy doesn't correct 201 nonzero backgrounds (unlike AL) so subtract them here. 203 additional arguments to be passed to 204 `lsst.pipe.base.Task` 206 additional keyword arguments to be passed to 207 `lsst.pipe.base.Task` 209 if self.
template is None and templateExposure
is None:
211 if self.
science is None and scienceExposure
is None:
220 self.
statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(
221 self.config.ignoreMaskPlanes))
224 self.
im2 = self.
science.getMaskedImage().getImage().getArray()
228 def selectPsf(psf, exposure):
233 xcen = (bbox1.getBeginX() + bbox1.getEndX()) / 2.
234 ycen = (bbox1.getBeginY() + bbox1.getEndY()) / 2.
235 return exposure.getPsf().computeKernelImage(
geom.Point2D(xcen, ycen)).getArray()
245 if (pShape1[0] < pShape2[0]):
246 psf1 = np.pad(psf1, ((0, pShape2[0] - pShape1[0]), (0, 0)), mode=
'constant', constant_values=0.)
247 elif (pShape2[0] < pShape1[0]):
248 psf2 = np.pad(psf2, ((0, pShape1[0] - pShape2[0]), (0, 0)), mode=
'constant', constant_values=0.)
249 if (pShape1[1] < pShape2[1]):
250 psf1 = np.pad(psf1, ((0, 0), (0, pShape2[1] - pShape1[1])), mode=
'constant', constant_values=0.)
251 elif (pShape2[1] < pShape1[1]):
252 psf2 = np.pad(psf2, ((0, 0), (0, pShape1[1] - pShape2[1])), mode=
'constant', constant_values=0.)
255 maxLoc1 = np.unravel_index(np.argmax(psf1), psf1.shape)
256 maxLoc2 = np.unravel_index(np.argmax(psf2), psf2.shape)
258 while (maxLoc1[0] != maxLoc2[0])
or (maxLoc1[1] != maxLoc2[1]):
259 if maxLoc1[0] > maxLoc2[0]:
260 psf2[1:, :] = psf2[:-1, :]
261 elif maxLoc1[0] < maxLoc2[0]:
262 psf1[1:, :] = psf1[:-1, :]
263 if maxLoc1[1] > maxLoc2[1]:
264 psf2[:, 1:] = psf2[:, :-1]
265 elif maxLoc1[1] < maxLoc2[1]:
266 psf1[:, 1:] = psf1[:, :-1]
267 maxLoc1 = np.unravel_index(np.argmax(psf1), psf1.shape)
268 maxLoc2 = np.unravel_index(np.argmax(psf2), psf2.shape)
271 psf1[psf1 < MIN_KERNEL] = MIN_KERNEL
272 psf2[psf2 < MIN_KERNEL] = MIN_KERNEL
281 if np.isnan(self.
sig1)
or self.
sig1 == 0:
283 if np.isnan(self.
sig2)
or self.
sig2 == 0:
287 if correctBackground:
288 def _subtractImageMean(exposure):
289 """Compute the sigma-clipped mean of the image of `exposure`.""" 290 mi = exposure.getMaskedImage()
293 mean = statObj.getValue(afwMath.MEANCLIP)
294 if not np.isnan(mean):
298 _subtractImageMean(self.
science)
301 self.
Fr = self.config.templateFluxScaling
302 self.
Fn = self.config.scienceFluxScaling
304 if self.config.scaleByCalibration:
305 calib_template = self.
template.getPhotoCalib()
306 calib_science = self.
science.getPhotoCalib()
307 if calib_template
is None:
308 self.log.warning(
"No calibration information available for template image.")
309 if calib_science
is None:
310 self.log.warning(
"No calibration information available for science image.")
311 if calib_template
is None or calib_science
is None:
312 self.log.warning(
"Due to lack of calibration information, " 313 "reverting to templateFluxScaling and scienceFluxScaling.")
315 self.
Fr = 1/calib_template.getCalibrationMean()
316 self.
Fn = 1/calib_science.getCalibrationMean()
318 self.log.info(
"Setting template image scaling to Fr=%f" % self.
Fr)
319 self.log.info(
"Setting science image scaling to Fn=%f" % self.
Fn)
323 def _computeVarianceMean(self, exposure):
324 """Compute the sigma-clipped mean of the variance image of `exposure`. 327 exposure.getMaskedImage().getMask(),
329 var = statObj.getValue(afwMath.MEANCLIP)
333 def _padPsfToSize(psf, size):
334 """Zero-pad `psf` to the dimensions given by `size`. 338 psf : 2D `numpy.array` 339 Input psf to be padded 341 Two element list containing the dimensions to pad the `psf` to 345 psf : 2D `numpy.array` 346 The padded copy of the input `psf`. 348 newArr = np.zeros(size)
350 offset = [size[0]//2 - psf.shape[0]//2, size[1]//2 - psf.shape[1]//2]
351 tmp = newArr[offset[0]:(psf.shape[0] + offset[0]), offset[1]:(psf.shape[1] + offset[1])]
356 """Compute standard ZOGY quantities used by (nearly) all methods. 358 Many of the ZOGY calculations require similar quantities, including 359 FFTs of the PSFs, and the "denominator" term (e.g. in eq. 13 of 360 ZOGY manuscript (2016). This function consolidates many of those 365 psf1 : 2D `numpy.array` 366 (Optional) Input psf of template, override if already padded 367 psf2 : 2D `numpy.array` 368 (Optional) Input psf of science image, override if already padded 369 padSize : `int`, optional 370 Number of pixels to pad the image on each side with zeroes. 374 A `lsst.pipe.base.Struct` containing: 375 - Pr : 2D `numpy.array`, the (possibly zero-padded) template PSF 376 - Pn : 2D `numpy.array`, the (possibly zero-padded) science PSF 377 - Pr_hat : 2D `numpy.array`, the FFT of `Pr` 378 - Pn_hat : 2D `numpy.array`, the FFT of `Pn` 379 - denom : 2D `numpy.array`, the denominator of equation (13) in ZOGY (2016) manuscript 380 - Fd : `float`, the relative flux scaling factor between science and template 382 psf1 = self.
im1_psf if psf1
is None else psf1
383 psf2 = self.
im2_psf if psf2
is None else psf2
384 padSize = self.
padSize if padSize
is None else padSize
387 Pr = ZogyTask._padPsfToSize(psf1, (psf1.shape[0] + padSize, psf1.shape[1] + padSize))
388 Pn = ZogyTask._padPsfToSize(psf2, (psf2.shape[0] + padSize, psf2.shape[1] + padSize))
390 psf1[np.abs(psf1) <= MIN_KERNEL] = MIN_KERNEL
391 psf2[np.abs(psf2) <= MIN_KERNEL] = MIN_KERNEL
394 Pr_hat = np.fft.fft2(Pr)
395 Pr_hat2 = np.conj(Pr_hat) * Pr_hat
396 Pn_hat = np.fft.fft2(Pn)
397 Pn_hat2 = np.conj(Pn_hat) * Pn_hat
398 denom = np.sqrt((sigN**2 * self.
Fr**2 * Pr_hat2) + (sigR**2 * self.
Fn**2 * Pn_hat2))
399 Fd = self.
Fr * self.
Fn / np.sqrt(sigN**2 * self.
Fr**2 + sigR**2 * self.
Fn**2)
401 res = pipeBase.Struct(
402 Pr=Pr, Pn=Pn, Pr_hat=Pr_hat, Pn_hat=Pn_hat, denom=denom, Fd=Fd
407 r"""Compute ZOGY diffim `D` as proscribed in ZOGY (2016) manuscript 411 debug : `bool`, optional 412 If set to True, filter the kernels by setting the edges to zero. 413 returnMatchedTemplate : `bool`, optional 414 Calculate the template image. 415 If not set, the returned template will be None. 419 In all functions, im1 is R (reference, or template) and im2 is N (new, or science) 420 Compute the ZOGY eqn. (13): 424 \widehat{D} = \frac{Fr\widehat{Pr}\widehat{N} - 425 F_n\widehat{Pn}\widehat{R}}{\sqrt{\sigma_n^2 Fr^2 426 \|\widehat{Pr}\|^2 + \sigma_r^2 F_n^2 \|\widehat{Pn}\|^2}} 428 where :math:`D` is the optimal difference image, :math:`R` and :math:`N` are the 429 reference and "new" image, respectively, :math:`Pr` and :math:`P_n` are their 430 PSFs, :math:`Fr` and :math:`Fn` are their flux-based zero-points (which we 431 will set to one here), :math:`\sigma_r^2` and :math:`\sigma_n^2` are their 432 variance, and :math:`\widehat{D}` denotes the FT of :math:`D`. 436 result : `lsst.pipe.base.Struct` 437 Result struct with components: 439 - ``D`` : 2D `numpy.array`, the proper image difference 440 - ``D_var`` : 2D `numpy.array`, the variance image for `D` 443 psf1 = ZogyTask._padPsfToSize(self.
im1_psf, self.
im1.shape)
444 psf2 = ZogyTask._padPsfToSize(self.
im2_psf, self.
im2.shape)
448 def _filterKernel(K, trim_amount):
451 K[:ps, :] = K[-ps:, :] = 0
452 K[:, :ps] = K[:, -ps:] = 0
455 Kr_hat = self.
Fr * preqs.Pr_hat / preqs.denom
456 Kn_hat = self.
Fn * preqs.Pn_hat / preqs.denom
457 if debug
and self.config.doTrimKernels:
460 ps = (Kn_hat.shape[1] - 80)//2
461 Kn = _filterKernel(np.fft.ifft2(Kn_hat), ps)
462 Kn_hat = np.fft.fft2(Kn)
463 Kr = _filterKernel(np.fft.ifft2(Kr_hat), ps)
464 Kr_hat = np.fft.fft2(Kr)
466 def processImages(im1, im2, doAdd=False):
468 im1[np.isinf(im1)] = np.nan
469 im1[np.isnan(im1)] = np.nanmean(im1)
470 im2[np.isinf(im2)] = np.nan
471 im2[np.isnan(im2)] = np.nanmean(im2)
473 R_hat = np.fft.fft2(im1)
474 N_hat = np.fft.fft2(im2)
476 D_hat = Kr_hat * N_hat
477 D_hat_R = Kn_hat * R_hat
483 D = np.fft.ifft2(D_hat)
484 D = np.fft.ifftshift(D.real) / preqs.Fd
487 if returnMatchedTemplate:
488 R = np.fft.ifft2(D_hat_R)
489 R = np.fft.ifftshift(R.real) / preqs.Fd
494 D, R = processImages(self.
im1, self.
im2, doAdd=
False)
496 D_var, R_var = processImages(self.
im1_var, self.
im2_var, doAdd=
True)
498 return pipeBase.Struct(D=D, D_var=D_var, R=R, R_var=R_var)
500 def _doConvolve(self, exposure, kernel, recenterKernel=False):
501 """Convolve an Exposure with a decorrelation convolution kernel. 505 exposure : `lsst.afw.image.Exposure` 506 Input exposure to be convolved. 507 kernel : `numpy.array` 508 2D `numpy.array` to convolve the image with 509 recenterKernel : `bool`, optional 510 Force the kernel center to the pixel with the maximum value. 514 A new `lsst.afw.image.Exposure` with the convolved pixels and the (possibly 519 - We optionally re-center the kernel if necessary and return the possibly 522 kernelImg = afwImage.ImageD(kernel.shape[1], kernel.shape[0])
523 kernelImg.getArray()[:, :] = kernel
526 maxloc = np.unravel_index(np.argmax(kernel), kernel.shape)
527 kern.setCtrX(maxloc[0])
528 kern.setCtrY(maxloc[1])
529 outExp = exposure.clone()
531 maxInterpolationDistance=0)
533 afwMath.convolve(outExp.getMaskedImage(), exposure.getMaskedImage(), kern, convCntrl)
534 except AttributeError:
542 """Compute ZOGY diffim `D` using image-space convlutions 544 This method is still being debugged as it results in artifacts 545 when the PSFs are noisy (see module-level docstring). Thus 546 there are several options still enabled by the `debug` flag, 547 which are disabled by defult. 552 The amount to pad the PSFs by 554 Flag to enable debugging tests and options 558 D : `lsst.afw.Exposure` 559 the proper image difference, including correct variance, 567 Kr_hat = (preqs.Pr_hat + delta) / (preqs.denom + delta)
568 Kn_hat = (preqs.Pn_hat + delta) / (preqs.denom + delta)
569 Kr = np.fft.ifft2(Kr_hat).real
570 Kr = np.roll(np.roll(Kr, -1, 0), -1, 1)
571 Kn = np.fft.ifft2(Kn_hat).real
572 Kn = np.roll(np.roll(Kn, -1, 0), -1, 1)
574 def _trimKernel(self, K, trim_amount):
578 K = K[ps:-ps, ps:-ps]
581 padSize = self.
padSize if padSize
is None else padSize
583 if debug
and self.config.doTrimKernels:
586 Kn = _trimKernel(Kn, padSize)
587 Kr = _trimKernel(Kr, padSize)
593 tmp = D.getMaskedImage()
594 tmp -= exp1.getMaskedImage()
596 return pipeBase.Struct(D=D, R=exp1)
598 def _setNewPsf(self, exposure, psfArr):
599 """Utility method to set an exposure's PSF when provided as a 2-d numpy.array 601 bbox = exposure.getBBox()
602 center = ((bbox.getBeginX() + bbox.getEndX()) // 2., (bbox.getBeginY() + bbox.getEndY()) // 2.)
604 psfI = afwImage.ImageD(psfArr.shape[1], psfArr.shape[0])
605 psfI.getArray()[:, :] = psfArr
607 psfNew = measAlg.KernelPsf(psfK, center)
608 exposure.setPsf(psfNew)
612 returnMatchedTemplate=False, **kwargs):
613 """Wrapper method to compute ZOGY proper diffim 615 This method should be used as the public interface for 616 computing the ZOGY diffim. 620 inImageSpace : `bool` 621 Override config `inImageSpace` parameter 623 Override config `padSize` parameter 624 returnMatchedTemplate : `bool` 625 Include the PSF-matched template in the results Struct 627 additional keyword arguments to be passed to 628 `computeDiffimFourierSpace` or `computeDiffimImageSpace`. 632 An lsst.pipe.base.Struct containing: 633 - D : `lsst.afw.Exposure` 634 the proper image difference, including correct variance, 636 - R : `lsst.afw.Exposure` 637 If `returnMatchedTemplate` is True, the PSF-matched template 641 inImageSpace = self.config.inImageSpace
if inImageSpace
is None else inImageSpace
643 padSize = self.
padSize if padSize
is None else padSize
646 if returnMatchedTemplate:
651 D.getMaskedImage().getImage().getArray()[:, :] = res.D
652 D.getMaskedImage().getVariance().getArray()[:, :] = res.D_var
653 if returnMatchedTemplate:
655 R.getMaskedImage().getImage().getArray()[:, :] = res.R
656 R.getMaskedImage().getVariance().getArray()[:, :] = res.R_var
660 return pipeBase.Struct(D=D, R=R)
663 """Compute the ZOGY diffim PSF (ZOGY manuscript eq. 14) 668 Override config `padSize` parameter 670 Return the FFT of the diffim PSF (do not inverse-FFT it) 671 psf1 : 2D `numpy.array` 672 (Optional) Input psf of template, override if already padded 673 psf2 : 2D `numpy.array` 674 (Optional) Input psf of science image, override if already padded 678 Pd : 2D `numpy.array` 679 The diffim PSF (or FFT of PSF if `keepFourier=True`) 681 preqs = self.
computePrereqs(psf1=psf1, psf2=psf2, padSize=padSize)
683 Pd_hat_numerator = (self.
Fr * self.
Fn * preqs.Pr_hat * preqs.Pn_hat)
684 Pd_hat = Pd_hat_numerator / (preqs.Fd * preqs.denom)
689 Pd = np.fft.ifft2(Pd_hat)
690 Pd = np.fft.ifftshift(Pd).real
694 def _computeVarAstGradients(self, xVarAst=0., yVarAst=0., inImageSpace=False,
695 R_hat=None, Kr_hat=None, Kr=None,
696 N_hat=None, Kn_hat=None, Kn=None):
697 """Compute the astrometric noise correction terms 699 Compute the correction for estimated astrometric noise as 700 proscribed in ZOGY (2016), section 3.3. All convolutions 701 performed either in real (image) or Fourier space. 705 xVarAst, yVarAst : `float` 706 estimated astrometric noise (variance of astrometric registration errors) 707 inImageSpace : `bool` 708 Perform all convolutions in real (image) space rather than Fourier space 709 R_hat : 2-D `numpy.array` 710 (Optional) FFT of template image, only required if `inImageSpace=False` 711 Kr_hat : 2-D `numpy.array` 712 FFT of Kr kernel (eq. 28 of ZOGY (2016)), only required if `inImageSpace=False` 713 Kr : 2-D `numpy.array` 714 Kr kernel (eq. 28 of ZOGY (2016)), only required if `inImageSpace=True`. 715 Kr is associated with the template (reference). 716 N_hat : 2-D `numpy.array` 717 FFT of science image, only required if `inImageSpace=False` 718 Kn_hat : 2-D `numpy.array` 719 FFT of Kn kernel (eq. 29 of ZOGY (2016)), only required if `inImageSpace=False` 720 Kn : 2-D `numpy.array` 721 Kn kernel (eq. 29 of ZOGY (2016)), only required if `inImageSpace=True`. 722 Kn is associated with the science (new) image. 726 VastSR, VastSN : 2-D `numpy.array` 727 Arrays containing the values in eqs. 30 and 32 of ZOGY (2016). 730 if xVarAst + yVarAst > 0:
733 S_R = S_R.getMaskedImage().getImage().getArray()
735 S_R = np.fft.ifft2(R_hat * Kr_hat)
736 gradRx, gradRy = np.gradient(S_R)
737 VastSR = xVarAst * gradRx**2. + yVarAst * gradRy**2.
741 S_N = S_N.getMaskedImage().getImage().getArray()
743 S_N = np.fft.ifft2(N_hat * Kn_hat)
744 gradNx, gradNy = np.gradient(S_N)
745 VastSN = xVarAst * gradNx**2. + yVarAst * gradNy**2.
747 return VastSR, VastSN
750 """Compute corrected likelihood image, optimal for source detection 752 Compute ZOGY S_corr image. This image can be thresholded for 753 detection without optimal filtering, and the variance image is 754 corrected to account for astrometric noise (errors in 755 astrometric registration whether systematic or due to effects 756 such as DCR). The calculations here are all performed in 757 Fourier space, as proscribed in ZOGY (2016). 761 xVarAst, yVarAst : `float` 762 estimated astrometric noise (variance of astrometric registration errors) 766 result : `lsst.pipe.base.Struct` 767 Result struct with components: 769 - ``S`` : `numpy.array`, the likelihood image S (eq. 12 of ZOGY (2016)) 770 - ``S_var`` : the corrected variance image (denominator of eq. 25 of ZOGY (2016)) 771 - ``Dpsf`` : the PSF of the diffim D, likely never to be used. 775 """Replace any NaNs or Infs with the mean of the image.""" 776 isbad = ~np.isfinite(im)
779 im[isbad] = np.nanmean(im)
782 self.
im1 = fix_nans(self.
im1)
783 self.
im2 = fix_nans(self.
im2)
788 psf1 = ZogyTask._padPsfToSize(self.
im1_psf, self.
im1.shape)
789 psf2 = ZogyTask._padPsfToSize(self.
im2_psf, self.
im2.shape)
794 R_hat = np.fft.fft2(self.
im1)
795 N_hat = np.fft.fft2(self.
im2)
796 D_hat = self.
Fr * preqs.Pr_hat * N_hat - self.
Fn * preqs.Pn_hat * R_hat
799 Pd_hat = self.
computeDiffimPsf(padSize=0, keepFourier=
True, psf1=psf1, psf2=psf2)
800 Pd_bar = np.conj(Pd_hat)
801 S = np.fft.ifft2(D_hat * Pd_bar)
805 Pn_hat2 = np.conj(preqs.Pn_hat) * preqs.Pn_hat
806 Kr_hat = self.
Fr * self.
Fn**2. * np.conj(preqs.Pr_hat) * Pn_hat2 / preqs.denom**2.
807 Pr_hat2 = np.conj(preqs.Pr_hat) * preqs.Pr_hat
808 Kn_hat = self.
Fn * self.
Fr**2. * np.conj(preqs.Pn_hat) * Pr_hat2 / preqs.denom**2.
810 Kr_hat2 = np.fft.fft2(np.fft.ifft2(Kr_hat)**2.)
811 Kn_hat2 = np.fft.fft2(np.fft.ifft2(Kn_hat)**2.)
812 var1c_hat = Kr_hat2 * np.fft.fft2(self.
im1_var)
813 var2c_hat = Kn_hat2 * np.fft.fft2(self.
im2_var)
817 R_hat=R_hat, Kr_hat=Kr_hat,
818 N_hat=N_hat, Kn_hat=Kn_hat)
820 S_var = np.sqrt(np.fft.ifftshift(np.fft.ifft2(var1c_hat + var2c_hat)) + fGradR + fGradN)
823 S = np.fft.ifftshift(np.fft.ifft2(Kn_hat * N_hat - Kr_hat * R_hat))
827 return pipeBase.Struct(S=S.real, S_var=S_var.real, Dpsf=Pd)
830 """Compute corrected likelihood image, optimal for source detection 832 Compute ZOGY S_corr image. This image can be thresholded for 833 detection without optimal filtering, and the variance image is 834 corrected to account for astrometric noise (errors in 835 astrometric registration whether systematic or due to effects 836 such as DCR). The calculations here are all performed in 841 xVarAst, yVarAst : `float` 842 estimated astrometric noise (variance of astrometric registration errors) 846 A `lsst.pipe.base.Struct` containing: 847 - S : `lsst.afw.image.Exposure`, the likelihood exposure S (eq. 12 of ZOGY (2016)), 848 including corrected variance, masks, and PSF 849 - D : `lsst.afw.image.Exposure`, the proper image difference, including correct 850 variance, masks, and PSF 855 padSize = self.
padSize if padSize
is None else padSize
859 Pd_bar = np.fliplr(np.flipud(Pd))
861 tmp = S.getMaskedImage()
866 Pn_hat2 = np.conj(preqs.Pn_hat) * preqs.Pn_hat
867 Kr_hat = self.
Fr * self.
Fn**2. * np.conj(preqs.Pr_hat) * Pn_hat2 / preqs.denom**2.
868 Pr_hat2 = np.conj(preqs.Pr_hat) * preqs.Pr_hat
869 Kn_hat = self.
Fn * self.
Fr**2. * np.conj(preqs.Pn_hat) * Pr_hat2 / preqs.denom**2.
871 Kr = np.fft.ifft2(Kr_hat).real
872 Kr = np.roll(np.roll(Kr, -1, 0), -1, 1)
873 Kn = np.fft.ifft2(Kn_hat).real
874 Kn = np.roll(np.roll(Kn, -1, 0), -1, 1)
882 Smi = S.getMaskedImage()
884 S_var = np.sqrt(var1c.getArray() + var2c.getArray() + fGradR + fGradN)
885 S.getMaskedImage().getVariance().getArray()[:, :] = S_var
889 return pipeBase.Struct(S=S, D=D)
891 def computeScorr(self, xVarAst=0., yVarAst=0., inImageSpace=None, padSize=0, **kwargs):
892 """Wrapper method to compute ZOGY corrected likelihood image, optimal for 895 This method should be used as the public interface for 896 computing the ZOGY S_corr. 900 xVarAst, yVarAst : `float` 901 estimated astrometric noise (variance of astrometric registration errors) 902 inImageSpace : `bool` 903 Override config `inImageSpace` parameter 905 Override config `padSize` parameter 909 S : `lsst.afw.image.Exposure` 910 The likelihood exposure S (eq. 12 of ZOGY (2016)), 911 including corrected variance, masks, and PSF 913 inImageSpace = self.config.inImageSpace
if inImageSpace
is None else inImageSpace
921 S.getMaskedImage().getImage().getArray()[:, :] = res.S
922 S.getMaskedImage().getVariance().getArray()[:, :] = res.S_var
925 return pipeBase.Struct(S=S)
929 """Task to be used as an ImageMapper for performing 930 ZOGY image subtraction on a grid of subimages. 932 ConfigClass = ZogyConfig
933 _DefaultName =
'ip_diffim_ZogyMapper' 936 ImageMapper.__init__(self, *args, **kwargs)
938 def run(self, subExposure, expandedSubExposure, fullBBox, template,
940 """Perform ZOGY proper image subtraction on sub-images 942 This method performs ZOGY proper image subtraction on 943 `subExposure` using local measures for image variances and 944 PSF. `subExposure` is a sub-exposure of the science image. It 945 also requires the corresponding sub-exposures of the template 946 (`template`). The operations are actually performed on 947 `expandedSubExposure` to allow for invalid edge pixels arising 948 from convolutions, which are then removed. 952 subExposure : `lsst.afw.image.Exposure` 953 the sub-exposure of the diffim 954 expandedSubExposure : `lsst.afw.image.Exposure` 955 the expanded sub-exposure upon which to operate 956 fullBBox : `lsst.geom.Box2I` 957 the bounding box of the original exposure 958 template : `lsst.afw.image.Exposure` 959 the template exposure, from which a corresponding sub-exposure 962 additional keyword arguments propagated from 963 `ImageMapReduceTask.run`. These include: 966 Compute and return the corrected likelihood image S_corr 967 rather than the proper image difference 968 ``inImageSpace`` : `bool` 969 Perform all convolutions in real (image) space rather than 970 in Fourier space. This option currently leads to artifacts 971 when using real (measured and noisy) PSFs, thus it is set 972 to `False` by default. 973 These kwargs may also include arguments to be propagated to 974 `ZogyTask.computeDiffim` and `ZogyTask.computeScorr`. 978 result : `lsst.pipe.base.Struct` 979 Result struct with components: 981 ``subExposure``: Either the subExposure of the proper image difference ``D``, 982 or (if `doScorr==True`) the corrected likelihood exposure ``S``. 986 This `run` method accepts parameters identical to those of 987 `ImageMapper.run`, since it is called from the 988 `ImageMapperTask`. See that class for more information. 990 bbox = subExposure.getBBox()
991 center = ((bbox.getBeginX() + bbox.getEndX()) // 2., (bbox.getBeginY() + bbox.getEndY()) // 2.)
994 imageSpace = kwargs.pop(
'inImageSpace',
False)
995 doScorr = kwargs.pop(
'doScorr',
False)
996 sigmas = kwargs.pop(
'sigmas',
None)
997 padSize = kwargs.pop(
'padSize', 7)
1000 subExp2 = expandedSubExposure
1003 subExp1 = template.Factory(template, expandedSubExposure.getBBox())
1009 sig1, sig2 = sigmas[0], sigmas[1]
1011 def _makePsfSquare(psf):
1013 if psf.shape[0] < psf.shape[1]:
1014 psf = np.pad(psf, ((0, psf.shape[1] - psf.shape[0]), (0, 0)), mode=
'constant',
1016 elif psf.shape[0] > psf.shape[1]:
1017 psf = np.pad(psf, ((0, 0), (0, psf.shape[0] - psf.shape[1])), mode=
'constant',
1021 psf2 = subExp2.getPsf().computeKernelImage(center).getArray()
1022 psf2 = _makePsfSquare(psf2)
1024 psf1 = template.getPsf().computeKernelImage(center).getArray()
1025 psf1 = _makePsfSquare(psf1)
1028 if subExp1.getDimensions()[0] < psf1.shape[0]
or subExp1.getDimensions()[1] < psf1.shape[1]:
1029 return pipeBase.Struct(subExposure=subExposure)
1031 def _filterPsf(psf):
1032 """Filter a noisy Psf to remove artifacts. Subject of future research.""" 1034 if psf.shape[0] == 41:
1037 psf[0:10, :] = psf[:, 0:10] = psf[31:41, :] = psf[:, 31:41] = 0
1042 psf1b = psf2b =
None 1043 if self.config.doFilterPsfs:
1045 psf1b = _filterPsf(psf1)
1046 psf2b = _filterPsf(psf2)
1049 if imageSpace
is True:
1050 config.inImageSpace = imageSpace
1051 config.padSize = padSize
1052 task =
ZogyTask(templateExposure=subExp1, scienceExposure=subExp2,
1053 sig1=sig1, sig2=sig2, psf1=psf1b, psf2=psf2b, config=config)
1056 res = task.computeDiffim(**kwargs)
1059 res = task.computeScorr(**kwargs)
1062 outExp = D.Factory(D, subExposure.getBBox())
1063 out = pipeBase.Struct(subExposure=outExp)
1068 """Config to be passed to ImageMapReduceTask 1070 This config targets the imageMapper to use the ZogyMapper. 1072 mapper = pexConfig.ConfigurableField(
1073 doc=
'Zogy task to run on each sub-image',
1079 """Config for the ZogyImagePsfMatchTask""" 1081 zogyConfig = pexConfig.ConfigField(
1083 doc=
'ZogyTask config to use when running on complete exposure (non spatially-varying)',
1086 zogyMapReduceConfig = pexConfig.ConfigField(
1087 dtype=ZogyMapReduceConfig,
1088 doc=
'ZogyMapReduce config to use when running Zogy on each sub-image (spatially-varying)',
1100 """Task to perform Zogy PSF matching and image subtraction. 1102 This class inherits from ImagePsfMatchTask to contain the _warper 1103 subtask and related methods. 1106 ConfigClass = ZogyImagePsfMatchConfig
1109 ImagePsfMatchTask.__init__(self, *args, **kwargs)
1111 def _computeImageMean(self, exposure):
1112 """Compute the sigma-clipped mean of the pixels image of `exposure`. 1115 statsControl.setNumSigmaClip(3.)
1116 statsControl.setNumIter(3)
1117 ignoreMaskPlanes = (
"INTRP",
"EDGE",
"DETECTED",
"SAT",
"CR",
"BAD",
"NO_DATA",
"DETECTED_NEGATIVE")
1118 statsControl.setAndMask(afwImage.Mask.getPlaneBitMask(ignoreMaskPlanes))
1120 exposure.getMaskedImage().getMask(),
1121 afwMath.MEANCLIP | afwMath.MEDIAN, statsControl)
1122 mn = statObj.getValue(afwMath.MEANCLIP)
1123 med = statObj.getValue(afwMath.MEDIAN)
1127 doWarping=True, spatiallyVarying=True, inImageSpace=False,
1128 doPreConvolve=False):
1129 """Register, PSF-match, and subtract two Exposures using the ZOGY algorithm. 1131 Do the following, in order: 1132 - Warp templateExposure to match scienceExposure, if their WCSs do not already match 1133 - Compute subtracted exposure ZOGY image subtraction algorithm on the two exposures 1137 templateExposure : `lsst.afw.image.Exposure` 1138 exposure to PSF-match to scienceExposure. The exposure's mean value is subtracted 1140 scienceExposure : `lsst.afw.image.Exposure` 1141 reference Exposure. The exposure's mean value is subtracted in-place. 1143 what to do if templateExposure's and scienceExposure's WCSs do not match: 1144 - if True then warp templateExposure to match scienceExposure 1145 - if False then raise an Exception 1146 spatiallyVarying : `bool` 1147 If True, perform the operation over a grid of patches across the two exposures 1148 inImageSpace : `bool` 1149 If True, perform the Zogy convolutions in image space rather than in frequency space. 1150 doPreConvolve : `bool` 1151 ***Currently not implemented.*** If True assume we are to compute the match filter-convolved 1152 exposure which can be thresholded for detection. In the case of Zogy this would mean 1153 we compute the Scorr image. 1157 A `lsst.pipe.base.Struct` containing these fields: 1158 - subtractedExposure: subtracted Exposure 1159 - warpedExposure: templateExposure after warping to match scienceExposure (if doWarping true) 1164 self.log.info(
"Exposure means=%f, %f; median=%f, %f:" % (mn1[0], mn2[0], mn1[1], mn2[1]))
1165 if not np.isnan(mn1[0])
and np.abs(mn1[0]) > 1:
1166 mi = templateExposure.getMaskedImage()
1168 if not np.isnan(mn2[0])
and np.abs(mn2[0]) > 1:
1169 mi = scienceExposure.getMaskedImage()
1172 self.log.info(
'Running Zogy algorithm: spatiallyVarying=%r' % spatiallyVarying)
1174 if not self.
_validateWcs(templateExposure, scienceExposure):
1176 self.log.info(
"Astrometrically registering template to science image")
1179 scienceExposure.getWcs())
1180 psfWarped = measAlg.WarpedPsf(templateExposure.getPsf(), xyTransform)
1181 templateExposure = self.
_warper.warpExposure(scienceExposure.getWcs(),
1183 destBBox=scienceExposure.getBBox())
1185 templateExposure.setPsf(psfWarped)
1187 self.log.error(
"ERROR: Input images not registered")
1188 raise RuntimeError(
"Input images not registered")
1191 return exp.getMaskedImage().getMask()
1194 return exp.getMaskedImage().getImage().getArray()
1196 if self.config.zogyConfig.inImageSpace:
1198 self.log.info(
'Running Zogy algorithm: inImageSpace=%r' % inImageSpace)
1199 if spatiallyVarying:
1200 config = self.config.zogyMapReduceConfig
1202 results = task.run(scienceExposure, template=templateExposure, inImageSpace=inImageSpace,
1203 doScorr=doPreConvolve, forceEvenSized=
False)
1204 results.D = results.exposure
1211 config = self.config.zogyConfig
1212 task =
ZogyTask(scienceExposure=scienceExposure, templateExposure=templateExposure,
1214 if not doPreConvolve:
1215 results = task.computeDiffim(inImageSpace=inImageSpace)
1216 results.matchedExposure = results.R
1218 results = task.computeScorr(inImageSpace=inImageSpace)
1219 results.D = results.S
1222 mask = results.D.getMaskedImage().getMask()
1223 mask |= scienceExposure.getMaskedImage().getMask()
1224 mask |= templateExposure.getMaskedImage().getMask()
1225 results.D.getMaskedImage().getMask()[:, :] = mask
1226 badBitsNan = mask.addMaskPlane(
'UNMASKEDNAN')
1227 resultsArr = results.D.getMaskedImage().getMask().getArray()
1228 resultsArr[np.isnan(resultsArr)] |= badBitsNan
1229 resultsArr[np.isnan(scienceExposure.getMaskedImage().getImage().getArray())] |= badBitsNan
1230 resultsArr[np.isnan(templateExposure.getMaskedImage().getImage().getArray())] |= badBitsNan
1232 results.subtractedExposure = results.D
1233 results.warpedExposure = templateExposure
1237 doWarping=True, spatiallyVarying=True, inImageSpace=False,
1238 doPreConvolve=False):
1239 raise NotImplementedError
1242 subtractAlgorithmRegistry.register(
'zogy', ZogyImagePsfMatchTask)
def _computeImageMean(self, exposure)
def _computeVarAstGradients(self, xVarAst=0., yVarAst=0., inImageSpace=False, R_hat=None, Kr_hat=None, Kr=None, N_hat=None, Kn_hat=None, Kn=None)
def computeDiffim(self, inImageSpace=None, padSize=None, returnMatchedTemplate=False, kwargs)
def computeDiffimFourierSpace(self, debug=False, returnMatchedTemplate=False, kwargs)
def __init__(self, args, kwargs)
def computePrereqs(self, psf1=None, psf2=None, padSize=0)
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
def run(self, subExposure, expandedSubExposure, fullBBox, template, kwargs)
def subtractExposures(self, templateExposure, scienceExposure, doWarping=True, spatiallyVarying=True, inImageSpace=False, doPreConvolve=False)
def setup(self, templateExposure=None, scienceExposure=None, sig1=None, sig2=None, psf1=None, psf2=None, correctBackground=False, args, kwargs)
def __init__(self, templateExposure=None, scienceExposure=None, sig1=None, sig2=None, psf1=None, psf2=None, args, kwargs)
def computeScorrFourierSpace(self, xVarAst=0., yVarAst=0., kwargs)
std::shared_ptr< TransformPoint2ToPoint2 > makeWcsPairTransform(SkyWcs const &src, SkyWcs const &dst)
def _computeVarianceMean(self, exposure)
def computeDiffimPsf(self, padSize=0, keepFourier=False, psf1=None, psf2=None)
def _doConvolve(self, exposure, kernel, recenterKernel=False)
def _setNewPsf(self, exposure, psfArr)
def subtractMaskedImages(self, templateExposure, scienceExposure, doWarping=True, spatiallyVarying=True, inImageSpace=False, doPreConvolve=False)
def computeScorr(self, xVarAst=0., yVarAst=0., inImageSpace=None, padSize=0, kwargs)
def computeDiffimImageSpace(self, padSize=None, debug=False, kwargs)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, bool doNormalize, bool doCopyEdge=false)
def _validateWcs(self, templateExposure, scienceExposure)
def computeScorrImageSpace(self, xVarAst=0., yVarAst=0., padSize=None, kwargs)
def __init__(self, args, kwargs)