24 from __future__
import absolute_import, division, print_function
25 from builtins
import zip
26 from builtins
import input
27 from builtins
import range
30 import lsst.afw.geom
as afwGeom
31 import lsst.afw.image
as afwImage
32 import lsst.afw.math
as afwMath
33 import lsst.afw.display.ds9
as ds9
35 from lsst.pipe.base
import Task, Struct, timeMethod
36 from lsst.pex.config
import Config, Field, ListField, ConfigField
40 """Produce a new frame number each time"""
47 """Options for measuring fringes on an exposure"""
48 badMaskPlanes = ListField(dtype=str, default=[
"SAT"], doc=
"Ignore pixels with these masks")
49 stat = Field(dtype=int, default=int(afwMath.MEDIAN), doc=
"Statistic to use")
50 clip = Field(dtype=float, default=3.0, doc=
"Sigma clip threshold")
51 iterations = Field(dtype=int, default=3, doc=
"Number of fitting iterations")
52 rngSeedOffset = Field(dtype=int, default=0,
53 doc=
"Offset to the random number generator seed (full seed includes exposure ID)")
57 """Fringe subtraction options"""
58 filters = ListField(dtype=str, default=[], doc=
"Only fringe-subtract these filters")
59 num = Field(dtype=int, default=30000, doc=
"Number of fringe measurements")
60 small = Field(dtype=int, default=3, doc=
"Half-size of small (fringe) measurements (pixels)")
61 large = Field(dtype=int, default=30, doc=
"Half-size of large (background) measurements (pixels)")
62 iterations = Field(dtype=int, default=20, doc=
"Number of fitting iterations")
63 clip = Field(dtype=float, default=3.0, doc=
"Sigma clip threshold")
64 stats = ConfigField(dtype=FringeStatisticsConfig, doc=
"Statistics for measuring fringes")
65 pedestal = Field(dtype=bool, default=
False, doc=
"Remove fringe pedestal?")
69 """Task to remove fringes from a science exposure
71 We measure fringe amplitudes at random positions on the science exposure
72 and at the same positions on the (potentially multiple) fringe frames
73 and solve for the scales simultaneously.
75 ConfigClass = FringeConfig
78 """Read the fringe frame(s)
80 The current implementation assumes only a single fringe frame and
81 will have to be updated to support multi-mode fringe subtraction.
83 This implementation could be optimised by persisting the fringe
86 @param dataRef Data reference for the science exposure
87 @param assembler An instance of AssembleCcdTask (for assembling fringe frames)
88 @return Struct(fringes: fringe exposure or list of fringe exposures;
89 seed: 32-bit uint derived from ccdExposureId for random number generator
92 fringe = dataRef.get(
"fringe", immediate=
True)
93 except Exception
as e:
94 raise RuntimeError(
"Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e))
95 if assembler
is not None:
96 fringe = assembler.assembleCcd(fringe)
98 seed = self.config.stats.rngSeedOffset + dataRef.get(
"ccdExposureId", immediate=
True)
102 return Struct(fringes=fringe,
106 def run(self, exposure, fringes, seed=None):
107 """Remove fringes from the provided science exposure.
109 Primary method of FringeTask. Fringes are only subtracted if the
110 science exposure has a filter listed in the configuration.
112 @param exposure Science exposure from which to remove fringes
113 @param fringes Exposure or list of Exposures
114 @param seed 32-bit unsigned integer for random number generator
117 display = lsstDebug.Info(__name__).display
123 seed = self.config.stats.rngSeedOffset
124 rng = numpy.random.RandomState(seed=seed)
126 if not hasattr(fringes,
'__iter__'):
129 mask = exposure.getMaskedImage().getMask()
130 for fringe
in fringes:
131 fringe.getMaskedImage().getMask().__ior__(mask)
132 if self.config.pedestal:
138 fluxes = numpy.ndarray([self.config.num, len(fringes)])
139 for i, f
in enumerate(fringes):
140 fluxes[:, i] = self.
measureExposure(f, positions, title=
"Fringe frame")
142 expFringes = self.
measureExposure(exposure, positions, title=
"Science")
143 solution = self.
solve(expFringes, fluxes)
144 self.
subtract(exposure, fringes, solution)
146 ds9.mtv(exposure, title=
"Fringe subtracted", frame=
getFrame())
150 """Remove fringes from the provided science exposure.
152 Retrieve fringes from butler dataRef provided and remove from
153 provided science exposure.
154 Fringes are only subtracted if the science exposure has a filter
155 listed in the configuration.
157 @param exposure Science exposure from which to remove fringes
158 @param dataRef Data reference for the science exposure
159 @param assembler An instance of AssembleCcdTask (for assembling fringe frames)
163 fringeStruct = self.
readFringes(dataRef, assembler=assembler)
164 self.
run(exposure, **fringeStruct.getDict())
167 """Check whether we should fringe-subtract the science exposure"""
168 return exposure.getFilter().getName()
in self.config.filters
171 """Remove pedestal from fringe exposure"""
172 stats = afwMath.StatisticsControl()
173 stats.setNumSigmaClip(self.config.stats.clip)
174 stats.setNumIter(self.config.stats.iterations)
175 mi = fringe.getMaskedImage()
176 pedestal = afwMath.makeStatistics(mi, afwMath.MEDIAN, stats).getValue()
177 self.log.info(
"Removing fringe pedestal: %f", pedestal)
181 """Generate a random distribution of positions for measuring fringe amplitudes"""
182 start = self.config.large
183 num = self.config.num
184 width = exposure.getWidth() - self.config.large
185 height = exposure.getHeight() - self.config.large
186 return numpy.array([rng.randint(start, width, size=num),
187 rng.randint(start, height, size=num)]).swapaxes(0, 1)
191 """Measure fringe amplitudes for an exposure
193 The fringe amplitudes are measured as the statistic within a square
194 aperture. The statistic within a larger aperture are subtracted so
195 as to remove the background.
197 @param exposure Exposure to measure
198 @param positions Array of (x,y) for fringe measurement
199 @param title Title for display
200 @return Array of fringe measurements
202 stats = afwMath.StatisticsControl()
203 stats.setNumSigmaClip(self.config.stats.clip)
204 stats.setNumIter(self.config.stats.iterations)
205 stats.setAndMask(exposure.getMaskedImage().getMask().getPlaneBitMask(self.config.stats.badMaskPlanes))
207 num = self.config.num
208 fringes = numpy.ndarray(num)
212 small =
measure(exposure.getMaskedImage(), x, y, self.config.small, self.config.stats.stat, stats)
213 large =
measure(exposure.getMaskedImage(), x, y, self.config.large, self.config.stats.stat, stats)
214 fringes[i] = small - large
217 display = lsstDebug.Info(__name__).display
220 ds9.mtv(exposure, frame=frame, title=title)
222 with ds9.Buffering():
223 for x, y
in positions:
224 corners = numpy.array([[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]) + [[x, y]]
225 ds9.line(corners * self.config.small, frame=frame, ctype=
"green")
226 ds9.line(corners * self.config.large, frame=frame, ctype=
"blue")
232 """Solve (with iterative clipping) for the scale factors
234 @param science Array of science exposure fringe amplitudes
235 @param fringes Array of arrays of fringe frame fringe amplitudes
236 @return Array of scale factors for the fringe frames
239 doPlot = lsstDebug.Info(__name__).plot
241 origNum = len(science)
243 good = numpy.where(numpy.logical_and(numpy.isfinite(science), numpy.any(numpy.isfinite(fringes), 1)))
244 science = science[good]
245 fringes = fringes[good]
246 oldNum = len(science)
250 good =
select(science, self.config.clip)
251 for ff
in range(fringes.shape[1]):
252 good &=
select(fringes[:, ff], self.config.clip)
253 science = science[good]
254 fringes = fringes[good]
255 oldNum = len(science)
257 for i
in range(self.config.iterations):
258 solution = self.
_solve(science, fringes)
259 resid = science - numpy.sum(solution * fringes, 1)
261 good = numpy.logical_not(abs(resid) > self.config.clip * rms)
262 self.log.debug(
"Iteration %d: RMS=%f numGood=%d", i, rms, good.sum())
263 self.log.debug(
"Solution %d: %s", i, solution)
267 import matplotlib.pyplot
as plot
268 for j
in range(fringes.shape[1]):
272 fig.canvas._tkcanvas._root().lift()
275 ax = fig.add_subplot(1, 1, 1)
276 adjust = science.copy()
277 others = set(range(fringes.shape[1]))
280 adjust -= solution[k] * fringes[:, k]
281 ax.plot(fringes[:, j], adjust,
'r.')
282 xmin = fringes[:, j].min()
283 xmax = fringes[:, j].max()
284 ymin = solution[j] * xmin
285 ymax = solution[j] * xmax
286 ax.plot([xmin, xmax], [ymin, ymax],
'b-')
287 ax.set_title(
"Fringe %d: %f" % (j, solution[j]))
288 ax.set_xlabel(
"Fringe amplitude")
289 ax.set_ylabel(
"Science amplitude")
290 ax.set_autoscale_on(
False)
291 ax.set_xbound(lower=xmin, upper=xmax)
292 ax.set_ybound(lower=ymin, upper=ymax)
295 ans = input(
"Enter or c to continue [chp]").lower()
296 if ans
in (
"",
"c",):
302 print(
"h[elp] c[ontinue] p[db]")
308 good = numpy.where(good)
309 science = science[good]
310 fringes = fringes[good]
313 solution = self.
_solve(science, fringes)
314 self.log.info(
"Fringe solution: %s RMS: %f Good: %d/%d", solution, rms, len(science), origNum)
317 def _solve(self, science, fringes):
318 """Solve for the scale factors
320 @param science Array of science exposure fringe amplitudes
321 @param fringes Array of arrays of fringe frame fringe amplitudes
322 @return Array of scale factors for the fringe frames
324 return afwMath.LeastSquares.fromDesignMatrix(fringes, science,
325 afwMath.LeastSquares.DIRECT_SVD).getSolution()
328 """Subtract the fringes
330 @param science Science exposure
331 @param fringes List of fringe frames
332 @param solution Array of scale factors for the fringe frames
334 if len(solution) != len(fringes):
335 raise RuntimeError(
"Number of fringe frames (%s) != number of scale factors (%s)" %
336 (len(fringes), len(solution)))
338 for s, f
in zip(solution, fringes):
339 science.getMaskedImage().scaledMinus(s, f.getMaskedImage())
342 def measure(mi, x, y, size, statistic, stats):
343 """Measure a statistic within an aperture
345 @param mi MaskedImage to measure
346 @param x, y Center for aperture
347 @param size Size of aperture
348 @param statistic Statistic to measure
349 @param stats StatisticsControl object
350 @return Value of statistic within aperture
352 bbox = afwGeom.Box2I(afwGeom.Point2I(int(x) - size, int(y - size)), afwGeom.Extent2I(2*size, 2*size))
353 subImage = mi.Factory(mi, bbox, afwImage.LOCAL)
354 return afwMath.makeStatistics(subImage, statistic, stats).getValue()
358 """Calculate a robust standard deviation of an array of values
360 @param vector Array of values
361 @return Standard deviation
363 q1, q3 = numpy.percentile(vector, (25, 75))
368 """Select values within 'clip' standard deviations of the median
370 Returns a boolean array.
372 q1, q2, q3 = numpy.percentile(vector, (25, 50, 75))
373 return numpy.abs(vector - q2) < clip*0.74*(q3 - q1)