22__all__ = [
"MaskStreaksConfig",
"MaskStreaksTask",
"setDetectionMask"]
27from lsst.utils.timer
import timeMethod
33from skimage.feature
import canny
34from sklearn.cluster
import KMeans
36from dataclasses
import dataclass
39def setDetectionMask(maskedImage, forceSlowBin=False, binning=None, detectedPlane="DETECTED",
40 badMaskPlanes=(
"NO_DATA",
"INTRP",
"BAD",
"SAT",
"EDGE"), detectionThreshold=5):
41 """Make detection mask and set the mask plane.
43 Creat a binary image from a masked image by setting all data
with signal-to-
44 noise below some threshold to zero,
and all data above the threshold to one.
45 If the binning parameter has been set, this procedure will be preceded by a
46 weighted binning of the data
in order to smooth the result, after which the
47 result
is scaled back to the original dimensions. Set the detection mask
48 plane
with this binary image.
53 Image to be (optionally) binned
and converted.
54 forceSlowBin : `bool`, optional
55 Force usage of slower binning method to check that the two methods
57 binning : `int`, optional
58 Number of pixels by which to bin image.
59 detectedPlane : `str`, optional
60 Name of mask
with pixels that were detected above threshold
in image.
61 badMaskPlanes : `set`, optional
62 Names of masks
with pixels that are rejected.
63 detectionThreshold : `float`, optional
64 Boundary
in signal-to-noise between non-detections
and detections
for
65 making a binary image
from the original input image.
67 data = maskedImage.image.array
68 weights = 1 / maskedImage.variance.array
69 mask = maskedImage.getMask()
71 detectionMask = ((mask.array & mask.getPlaneBitMask(detectedPlane)))
72 badPixelMask = mask.getPlaneBitMask(badMaskPlanes)
73 badMask = (mask.array & badPixelMask) > 0
74 fitMask = detectionMask.astype(bool) & ~badMask
76 fitData = np.copy(data)
78 fitWeights = np.copy(weights)
79 fitWeights[~fitMask] = 0
83 ymax, xmax = fitData.shape
84 if (ymax % binning == 0)
and (xmax % binning == 0)
and (
not forceSlowBin):
86 binNumeratorReshape = (fitData * fitWeights).reshape(ymax // binning, binning,
87 xmax // binning, binning)
88 binDenominatorReshape = fitWeights.reshape(binNumeratorReshape.shape)
89 binnedNumerator = binNumeratorReshape.sum(axis=3).sum(axis=1)
90 binnedDenominator = binDenominatorReshape.sum(axis=3).sum(axis=1)
93 warnings.warn(
'Using slow binning method--consider choosing a binsize that evenly divides '
94 f
'into the image size, so that {ymax} mod binning == 0 '
95 f
'and {xmax} mod binning == 0')
96 xarray = np.arange(xmax)
97 yarray = np.arange(ymax)
98 xmesh, ymesh = np.meshgrid(xarray, yarray)
99 xbins = np.arange(0, xmax + binning, binning)
100 ybins = np.arange(0, ymax + binning, binning)
101 numerator = fitWeights * fitData
102 binnedNumerator, *_ = scipy.stats.binned_statistic_2d(ymesh.ravel(), xmesh.ravel(),
103 numerator.ravel(), statistic=
'sum',
105 binnedDenominator, *_ = scipy.stats.binned_statistic_2d(ymesh.ravel(), xmesh.ravel(),
106 fitWeights.ravel(), statistic=
'sum',
108 binnedData = np.zeros(binnedNumerator.shape)
109 ind = binnedDenominator != 0
110 np.divide(binnedNumerator, binnedDenominator, out=binnedData, where=ind)
111 binnedWeight = binnedDenominator
112 binMask = (binnedData * binnedWeight**0.5) > detectionThreshold
113 tmpOutputMask = binMask.repeat(binning, axis=0)[:ymax]
114 outputMask = tmpOutputMask.repeat(binning, axis=1)[:, :xmax]
116 outputMask = (fitData * fitWeights**0.5) > detectionThreshold
119 maskedImage.mask.array &= ~maskedImage.mask.getPlaneBitMask(detectedPlane)
122 maskedImage.mask.array[outputMask] |= maskedImage.mask.getPlaneBitMask(detectedPlane)
127 """A simple data class to describe a line profile. The parameter `rho`
128 describes the distance from the center of the image, `theta` describes
129 the angle,
and `sigma` describes the width of the line.
138 """Collection of `Line` objects.
143 Array of `Line` rho parameters.
144 thetas : `np.ndarray`
145 Array of `Line` theta parameters.
146 sigmas : `np.ndarray`, optional
147 Array of `Line` sigma parameters.
152 sigmas = np.zeros(len(rhos))
154 self.
_lines = [
Line(rho, theta, sigma)
for (rho, theta, sigma)
in
155 zip(rhos, thetas, sigmas)]
167 joinedString =
", ".join(
str(line)
for line
in self.
_lines)
168 return textwrap.shorten(joinedString, width=160, placeholder=
"...")
172 return np.array([line.rho
for line
in self.
_lines])
176 return np.array([line.theta
for line
in self.
_lines])
179 """Add line to current collection of lines.
184 `Line` to add to current collection of lines
190 """Construct and/or fit a model for a linear streak.
192 This assumes a simple model for a streak,
in which the streak
193 follows a straight line
in pixels space,
with a Moffat-shaped profile. The
194 model
is fit to data using a Newton-Raphson style minimization algorithm.
195 The initial guess
for the line parameters
is assumed to be fairly accurate,
196 so only a narrow band of pixels around the initial line estimate
is used
in
197 fitting the model, which provides a significant speed-up over using all the
198 data. The
class can also be used just to construct a model for the
data with
199 a line following the given coordinates.
205 weights : `np.ndarray`
207 line : `Line`, optional
208 Guess
for position of line. Data far
from line guess
is masked out.
209 Defaults to
None,
in which case only data
with `weights` = 0
is masked
216 self._ymax, self.
_xmax = data.shape
218 xrange = np.arange(self.
_xmax) - self.
_xmax / 2.
219 yrange = np.arange(self._ymax) - self._ymax / 2.
220 self.
_rhoMax = ((0.5 * self._ymax)**2 + (0.5 * self.
_xmax)**2)**0.5
221 self._xmesh, self.
_ymesh = np.meshgrid(xrange, yrange)
228 """Set mask around the image region near the line.
233 Parameters of line in the image.
237 radtheta = np.deg2rad(line.theta)
238 distance = (np.cos(radtheta) * self._xmesh + np.sin(radtheta) * self.
_ymesh - line.rho)
239 m = (abs(distance) < 5 * line.sigma)
250 def _makeMaskedProfile(self, line, fitFlux=True):
251 """Construct the line model in the masked region and calculate its
257 Parameters of line profile for which to make profile
in the masked
260 Fit the amplitude of the line profile to the data.
265 Model
in the masked region.
266 dModel : `np.ndarray`
267 Derivative of the model
in the masked region.
269 invSigma = line.sigma**-1
271 radtheta = np.deg2rad(line.theta)
272 costheta = np.cos(radtheta)
273 sintheta = np.sin(radtheta)
274 distance = (costheta * self.
_mxmesh + sintheta * self.
_mymesh - line.rho)
275 distanceSquared = distance**2
279 dDistanceSqdRho = 2 * distance * (-np.ones_like(self.
_mxmesh))
280 dDistanceSqdTheta = (2 * distance * (-sintheta * self.
_mxmesh + costheta * self.
_mymesh) * drad)
283 profile = (1 + distanceSquared * invSigma**2)**-2.5
284 dProfile = -2.5 * (1 + distanceSquared * invSigma**2)**-3.5
296 model = flux * profile
299 fluxdProfile = flux * dProfile
300 fluxdProfileInvSigma = fluxdProfile * invSigma**2
301 dModeldRho = fluxdProfileInvSigma * dDistanceSqdRho
302 dModeldTheta = fluxdProfileInvSigma * dDistanceSqdTheta
303 dModeldInvSigma = fluxdProfile * distanceSquared * 2 * invSigma
305 dModel = np.array([dModeldRho, dModeldTheta, dModeldInvSigma])
309 """Construct the line profile model.
314 Parameters of the line profile to model.
315 fitFlux : `bool`, optional
316 Fit the amplitude of the line profile to the data.
320 finalModel : `np.ndarray`
321 Model for line profile.
324 finalModel = np.zeros((self._ymax, self._xmax), dtype=self._dtype)
328 def _lineChi2(self, line, grad=True):
329 """Construct the chi2 between the data and the model.
334 `Line` parameters for which to build model
and calculate chi2.
335 grad : `bool`, optional
336 Whether
or not to
return the gradient
and hessian.
341 Reduced chi2 of the model.
342 reducedDChi : `np.ndarray`
343 Derivative of the chi2
with respect to rho, theta, invSigma.
344 reducedHessianChi : `np.ndarray`
345 Hessian of the chi2
with respect to rho, theta, invSigma.
355 hessianChi2 = (2 * self.
_maskWeights * dModel[:,
None, :] * dModel[
None, :, :]).sum(axis=2)
360 return reducedChi, reducedDChi, reducedHessianChi
362 def fit(self, dChi2Tol=0.1, maxIter=100, log=None):
363 """Perform Newton-Raphson minimization to find line parameters.
365 This method takes advantage of having known derivative and Hessian of
366 the multivariate function to quickly
and efficiently find the minimum.
367 This
is more efficient than the scipy implementation of the Newton-
368 Raphson method, which doesn
't take advantage of the Hessian matrix. The
369 method here also performs a line search in the direction of the steepest
370 derivative at each iteration, which reduces the number of iterations
375 dChi2Tol : `float`, optional
376 Change
in Chi2 tolerated
for fit convergence.
377 maxIter : `int`, optional
378 Maximum number of fit iterations allowed. The fit should converge
in
379 ~10 iterations, depending on the value of dChi2Tol, but this
380 maximum provides a backup.
381 log : `lsst.utils.logging.LsstLogAdapter`, optional
382 Logger to use
for reporting more details
for fitting failures.
386 outline : `np.ndarray`
387 Coordinates
and inverse width of fit line.
389 Reduced Chi2 of model fit to data.
391 Boolean where `
False` corresponds to a successful fit.
401 def line_search(c, dx):
403 testLine =
Line(testx[0], testx[1], testx[2]**-1)
404 return self.
_lineChi2(testLine, grad=
False)
406 while abs(dChi2) > dChi2Tol:
407 line =
Line(x[0], x[1], x[2]**-1)
411 if not np.isfinite(A).all():
414 log.warning(
"Hessian matrix has non-finite elements.")
416 dChi2 = oldChi2 - chi2
418 cholesky = scipy.linalg.cho_factor(A)
419 except np.linalg.LinAlgError:
422 log.warning(
"Hessian matrix is not invertible.")
424 dx = scipy.linalg.cho_solve(cholesky, b)
426 factor, fmin, _, _ = scipy.optimize.brent(line_search, args=(dx,), full_output=
True, tol=0.05)
428 if (abs(x[0]) > 1.5 * self.
_rhoMax)
or (iter > maxIter):
434 outline =
Line(x[0], x[1], abs(x[2])**-1)
436 return outline, chi2, fitFailure
440 """Configuration parameters for `MaskStreaksTask`.
443 minimumKernelHeight = pexConfig.Field(
444 doc="Minimum height of the streak-finding kernel relative to the tallest kernel",
448 absMinimumKernelHeight = pexConfig.Field(
449 doc=
"Minimum absolute height of the streak-finding kernel",
453 clusterMinimumSize = pexConfig.Field(
454 doc=
"Minimum size in pixels of detected clusters",
458 clusterMinimumDeviation = pexConfig.Field(
459 doc=
"Allowed deviation (in pixels) from a straight line for a detected "
464 delta = pexConfig.Field(
465 doc=
"Stepsize in angle-radius parameter space",
469 nSigma = pexConfig.Field(
470 doc=
"Number of sigmas from center of kernel to include in voting "
475 rhoBinSize = pexConfig.Field(
476 doc=
"Binsize in pixels for position parameter rho when finding "
477 "clusters of detected lines",
481 thetaBinSize = pexConfig.Field(
482 doc=
"Binsize in degrees for angle parameter theta when finding "
483 "clusters of detected lines",
487 invSigma = pexConfig.Field(
488 doc=
"Inverse of the Moffat sigma parameter (in units of pixels)"
489 "describing the profile of the streak",
493 footprintThreshold = pexConfig.Field(
494 doc=
"Threshold at which to determine edge of line, in units of "
499 dChi2Tolerance = pexConfig.Field(
500 doc=
"Absolute difference in Chi2 between iterations of line profile"
501 "fitting that is acceptable for convergence",
505 detectedMaskPlane = pexConfig.Field(
506 doc=
"Name of mask with pixels above detection threshold, used for first"
507 "estimate of streak locations",
511 streaksMaskPlane = pexConfig.Field(
512 doc=
"Name of mask plane holding detected streaks",
519 """Find streaks or other straight lines in image data.
521 Nearby objects passing through the field of view of the telescope leave a
522 bright trail in images. This
class uses the Kernel Hough Transform (KHT)
523 (Fernandes
and Oliveira, 2007), implemented
in `lsst.houghtransform`. The
524 procedure works by taking a binary image, either provided
as put
or produced
525 from the input data image, using a Canny filter to make an image of the
526 edges
in the original image, then running the KHT on the edge image. The KHT
527 identifies clusters of non-zero points, breaks those clusters of points into
528 straight lines, keeps clusters
with a size greater than the user-set
529 threshold, then performs a voting procedure to find the best-fit coordinates
530 of any straight lines. Given the results of the KHT algorithm, clusters of
531 lines are identified
and grouped (generally these correspond to the two
532 edges of a strea)
and a profile
is fit to the streak
in the original
536 ConfigClass = MaskStreaksConfig
537 _DefaultName = "maskStreaks"
541 """Find streaks in a masked image.
546 The image in which to search
for streaks.
550 result : `lsst.pipe.base.Struct`
551 Results
as a struct
with attributes:
554 Lines identified by kernel hough transform.
556 Lines grouped into clusters
in rho-theta space.
558 Final result
for lines after line-profile fit.
560 2-d boolean mask where detected lines are
True.
562 mask = maskedImage.getMask()
563 detectionMask = (mask.array & mask.getPlaneBitMask(self.config.detectedMaskPlane))
568 if len(self.
lines) == 0:
569 lineMask = np.zeros(detectionMask.shape, dtype=bool)
574 fitLines, lineMask = self.
_fitProfile(clusters, maskedImage)
577 outputMask = lineMask & detectionMask.astype(bool)
579 return pipeBase.Struct(
581 lineClusters=clusters,
582 originalLines=self.
lines,
587 def run(self, maskedImage):
588 """Find and mask streaks in a masked image.
590 Finds streaks in the image
and modifies maskedImage
in place by adding a
591 mask plane
with any identified streaks.
596 The image
in which to search
for streaks. The mask detection plane
597 corresponding to `config.detectedMaskPlane` must be set
with the
602 result : `lsst.pipe.base.Struct`
603 Results
as a struct
with attributes:
606 Lines identified by kernel hough transform.
608 Lines grouped into clusters
in rho-theta space.
610 Final result
for lines after line-profile fit.
612 streaks = self.find(maskedImage)
614 maskedImage.mask.addMaskPlane(self.config.streaksMaskPlane)
615 maskedImage.mask.array[streaks.mask] |= maskedImage.mask.getPlaneBitMask(self.config.streaksMaskPlane)
617 return pipeBase.Struct(
619 lineClusters=streaks.lineClusters,
620 originalLines=streaks.originalLines,
623 def _cannyFilter(self, image):
624 """Apply a canny filter to the data in order to detect edges.
629 2-d image data on which to run filter.
633 cannyData : `np.ndarray`
634 2-d image of edges found in input image.
636 filterData = image.astype(int)
637 return canny(filterData, low_threshold=0, high_threshold=1, sigma=0.1)
639 def _runKHT(self, image):
640 """Run Kernel Hough Transform on image.
645 2-d image data on which to detect lines.
649 result : `LineCollection`
650 Collection of detected lines, with their detected rho
and theta
653 lines = lsst.kht.find_lines(image, self.config.clusterMinimumSize,
654 self.config.clusterMinimumDeviation, self.config.delta,
655 self.config.minimumKernelHeight, self.config.nSigma,
656 self.config.absMinimumKernelHeight)
657 self.log.info("The Kernel Hough Transform detected %s line(s)", len(lines))
661 def _findClusters(self, lines):
662 """Group lines that are close in parameter space and likely describe
667 lines : `LineCollection`
668 Collection of lines to group into clusters.
672 result : `LineCollection`
673 Average `Line` for each cluster of `Line`s
in the input
680 x = lines.rhos / self.config.rhoBinSize
681 y = lines.thetas / self.config.thetaBinSize
682 X = np.array([x, y]).T
691 kmeans = KMeans(n_clusters=nClusters).fit(X)
692 clusterStandardDeviations = np.zeros((nClusters, 2))
693 for c
in range(nClusters):
694 inCluster = X[kmeans.labels_ == c]
695 clusterStandardDeviations[c] = np.std(inCluster, axis=0)
697 if (clusterStandardDeviations <= 1).all():
702 finalClusters = kmeans.cluster_centers_.T
705 finalRhos = finalClusters[0] * self.config.rhoBinSize
706 finalThetas = finalClusters[1] * self.config.thetaBinSize
708 self.log.info(
"Lines were grouped into %s potential streak(s)", len(finalRhos))
712 def _fitProfile(self, lines, maskedImage):
713 """Fit the profile of the streak.
715 Given the initial parameters of detected lines, fit a model for the
716 streak to the original (non-binary image). The assumed model
is a
717 straight line
with a Moffat profile.
721 lines : `LineCollection`
722 Collection of guesses
for `Line`s detected
in the image.
724 Original image to be used to fit profile of streak.
728 lineFits : `LineCollection`
729 Collection of `Line` profiles fit to the data.
730 finalMask : `np.ndarray`
731 2d mask array
with detected streaks=1.
733 data = maskedImage.image.array
734 weights = maskedImage.variance.array**-1
736 weights[~np.isfinite(weights) | ~np.isfinite(data)] = 0
739 finalLineMasks = [np.zeros(data.shape, dtype=bool)]
742 line.sigma = self.config.invSigma**-1
745 if lineModel.lineMaskSize == 0:
748 fit, chi2, fitFailure = lineModel.fit(dChi2Tol=self.config.dChi2Tolerance, log=self.log)
750 self.log.warning(
"Streak fit failed.")
754 if ((abs(fit.rho - line.rho) > 2 * self.config.rhoBinSize)
755 or (abs(fit.theta - line.theta) > 2 * self.config.thetaBinSize)):
757 self.log.warning(
"Streak fit moved too far from initial estimate. Line will be dropped.")
762 self.log.debug(
"Best fit streak parameters are rho=%.2f, theta=%.2f, and sigma=%.2f", fit.rho,
763 fit.theta, fit.sigma)
766 lineModel.setLineMask(fit)
767 finalModel = lineModel.makeProfile(fit)
769 finalModelMax = abs(finalModel).max()
770 finalLineMask = abs(finalModel) > self.config.footprintThreshold
772 if not finalLineMask.any():
775 fit.finalModelMax = finalModelMax
777 finalLineMasks.append(finalLineMask)
780 finalMask = np.array(finalLineMasks).any(axis=0)
781 nMaskedPixels = finalMask.sum()
782 percentMasked = (nMaskedPixels / finalMask.size) * 100
783 self.log.info(
"%d streak(s) fit, with %d pixels masked (%0.2f%% of image)", nFinalLines,
784 nMaskedPixels, percentMasked)
786 return lineFits, finalMask
def __init__(self, rhos, thetas, sigmas=None)
def append(self, newLine)
def __getitem__(self, index)
def __init__(self, data, weights, line=None)
def makeProfile(self, line, fitFlux=True)
def _makeMaskedProfile(self, line, fitFlux=True)
def _lineChi2(self, line, grad=True)
def fit(self, dChi2Tol=0.1, maxIter=100, log=None)
def setLineMask(self, line)
def find(self, maskedImage)
def _fitProfile(self, lines, maskedImage)
def _findClusters(self, lines)
def _cannyFilter(self, image)
def setDetectionMask(maskedImage, forceSlowBin=False, binning=None, detectedPlane="DETECTED", badMaskPlanes=("NO_DATA", "INTRP", "BAD", "SAT", "EDGE"), detectionThreshold=5)