Coverage for python/lsst/ip/isr/fringe.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy
24import lsst.geom
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.afw.display as afwDisplay
29from lsst.pipe.base import Task, Struct, timeMethod
30from lsst.pex.config import Config, Field, ListField, ConfigField
32afwDisplay.setDefaultMaskTransparency(75)
35def getFrame():
36 """Produce a new frame number each time"""
37 getFrame.frame += 1
38 return getFrame.frame
41getFrame.frame = 0
44class FringeStatisticsConfig(Config):
45 """Options for measuring fringes on an exposure"""
46 badMaskPlanes = ListField(dtype=str, default=["SAT"], doc="Ignore pixels with these masks")
47 stat = Field(dtype=int, default=int(afwMath.MEDIAN), doc="Statistic to use")
48 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold")
49 iterations = Field(dtype=int, default=3, doc="Number of fitting iterations")
50 rngSeedOffset = Field(dtype=int, default=0,
51 doc="Offset to the random number generator seed (full seed includes exposure ID)")
54class FringeConfig(Config):
55 """Fringe subtraction options"""
56 filters = ListField(dtype=str, default=[], doc="Only fringe-subtract these filters")
57 num = Field(dtype=int, default=30000, doc="Number of fringe measurements")
58 small = Field(dtype=int, default=3, doc="Half-size of small (fringe) measurements (pixels)")
59 large = Field(dtype=int, default=30, doc="Half-size of large (background) measurements (pixels)")
60 iterations = Field(dtype=int, default=20, doc="Number of fitting iterations")
61 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold")
62 stats = ConfigField(dtype=FringeStatisticsConfig, doc="Statistics for measuring fringes")
63 pedestal = Field(dtype=bool, default=False, doc="Remove fringe pedestal?")
66class FringeTask(Task):
67 """Task to remove fringes from a science exposure
69 We measure fringe amplitudes at random positions on the science exposure
70 and at the same positions on the (potentially multiple) fringe frames
71 and solve for the scales simultaneously.
72 """
73 ConfigClass = FringeConfig
74 _DefaultName = 'isrFringe'
76 def readFringes(self, dataRef, assembler=None):
77 """Read the fringe frame(s), and pack data into a Struct
79 The current implementation assumes only a single fringe frame and
80 will have to be updated to support multi-mode fringe subtraction.
82 This implementation could be optimised by persisting the fringe
83 positions and fluxes.
85 Parameters
86 ----------
87 dataRef : `daf.butler.butlerSubset.ButlerDataRef`
88 Butler reference for the exposure that will have fringing
89 removed.
90 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
91 An instance of AssembleCcdTask (for assembling fringe
92 frames).
94 Returns
95 -------
96 fringeData : `pipeBase.Struct`
97 Struct containing fringe data:
98 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
99 Calibration fringe files containing master fringe frames.
100 - ``seed`` : `int`, optional
101 Seed for random number generation.
102 """
103 try:
104 fringe = dataRef.get("fringe", immediate=True)
105 except Exception as e:
106 raise RuntimeError("Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e))
108 return self.loadFringes(fringe, assembler)
110 def loadFringes(self, fringeExp, expId=0, assembler=None):
111 """Pack the fringe data into a Struct.
113 This method moves the struct parsing code into a butler
114 generation agnostic handler.
116 Parameters
117 ----------
118 fringeExp : `lsst.afw.exposure.Exposure`
119 The exposure containing the fringe data.
120 expId : `int`, optional
121 Exposure id to be fringe corrected, used to set RNG seed.
122 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
123 An instance of AssembleCcdTask (for assembling fringe
124 frames).
126 Returns
127 -------
128 fringeData : `pipeBase.Struct`
129 Struct containing fringe data:
130 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
131 Calibration fringe files containing master fringe frames.
132 - ``seed`` : `int`, optional
133 Seed for random number generation.
134 """
135 if assembler is not None:
136 fringeExp = assembler.assembleCcd(fringeExp)
138 if expId is None:
139 seed = self.config.stats.rngSeedOffset
140 else:
141 print(f"{self.config.stats.rngSeedOffset} {expId}")
142 seed = self.config.stats.rngSeedOffset + expId
144 # Seed for numpy.random.RandomState must be convertable to a 32 bit unsigned integer
145 seed %= 2**32
147 return Struct(fringes=fringeExp,
148 seed=seed)
150 @timeMethod
151 def run(self, exposure, fringes, seed=None):
152 """Remove fringes from the provided science exposure.
154 Primary method of FringeTask. Fringes are only subtracted if the
155 science exposure has a filter listed in the configuration.
157 Parameters
158 ----------
159 exposure : `lsst.afw.image.Exposure`
160 Science exposure from which to remove fringes.
161 fringes : `lsst.afw.image.Exposure` or `list` thereof
162 Calibration fringe files containing master fringe frames.
163 seed : `int`, optional
164 Seed for random number generation.
166 Returns
167 -------
168 solution : `np.array`
169 Fringe solution amplitudes for each input fringe frame.
170 rms : `float`
171 RMS error for the fit solution for this exposure.
172 """
173 import lsstDebug
174 display = lsstDebug.Info(__name__).display
176 if not self.checkFilter(exposure):
177 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
178 return
180 if seed is None:
181 seed = self.config.stats.rngSeedOffset
182 rng = numpy.random.RandomState(seed=seed)
184 if not hasattr(fringes, '__iter__'):
185 fringes = [fringes]
187 mask = exposure.getMaskedImage().getMask()
188 for fringe in fringes:
189 fringe.getMaskedImage().getMask().__ior__(mask)
190 if self.config.pedestal:
191 self.removePedestal(fringe)
193 positions = self.generatePositions(fringes[0], rng)
194 fluxes = numpy.ndarray([self.config.num, len(fringes)])
195 for i, f in enumerate(fringes):
196 fluxes[:, i] = self.measureExposure(f, positions, title="Fringe frame")
198 expFringes = self.measureExposure(exposure, positions, title="Science")
199 solution, rms = self.solve(expFringes, fluxes)
200 self.subtract(exposure, fringes, solution)
201 if display:
202 afwDisplay.Display(frame=getFrame()).mtv(exposure, title="Fringe subtracted")
203 return solution, rms
205 @timeMethod
206 def runDataRef(self, exposure, dataRef, assembler=None):
207 """Remove fringes from the provided science exposure.
209 Retrieve fringes from butler dataRef provided and remove from
210 provided science exposure. Fringes are only subtracted if the
211 science exposure has a filter listed in the configuration.
213 Parameters
214 ----------
215 exposure : `lsst.afw.image.Exposure`
216 Science exposure from which to remove fringes.
217 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
218 Butler reference to the exposure. Used to find
219 appropriate fringe data.
220 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
221 An instance of AssembleCcdTask (for assembling fringe
222 frames).
224 Returns
225 -------
226 solution : `np.array`
227 Fringe solution amplitudes for each input fringe frame.
228 rms : `float`
229 RMS error for the fit solution for this exposure.
230 """
231 if not self.checkFilter(exposure):
232 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
233 return
234 fringeStruct = self.readFringes(dataRef, assembler=assembler)
235 return self.run(exposure, **fringeStruct.getDict())
237 def checkFilter(self, exposure):
238 """Check whether we should fringe-subtract the science exposure.
240 Parameters
241 ----------
242 exposure : `lsst.afw.image.Exposure`
243 Exposure to check the filter of.
245 Returns
246 -------
247 needsFringe : `bool`
248 If True, then the exposure has a filter listed in the
249 configuration, and should have the fringe applied.
250 """
251 return exposure.getFilter().getName() in self.config.filters
253 def removePedestal(self, fringe):
254 """Remove pedestal from fringe exposure.
256 Parameters
257 ----------
258 fringe : `lsst.afw.image.Exposure`
259 Fringe data to subtract the pedestal value from.
260 """
261 stats = afwMath.StatisticsControl()
262 stats.setNumSigmaClip(self.config.stats.clip)
263 stats.setNumIter(self.config.stats.iterations)
264 mi = fringe.getMaskedImage()
265 pedestal = afwMath.makeStatistics(mi, afwMath.MEDIAN, stats).getValue()
266 self.log.info("Removing fringe pedestal: %f", pedestal)
267 mi -= pedestal
269 def generatePositions(self, exposure, rng):
270 """Generate a random distribution of positions for measuring fringe amplitudes.
272 Parameters
273 ----------
274 exposure : `lsst.afw.image.Exposure`
275 Exposure to measure the positions on.
276 rng : `numpy.random.RandomState`
277 Random number generator to use.
279 Returns
280 -------
281 positions : `numpy.array`
282 Two-dimensional array containing the positions to sample
283 for fringe amplitudes.
284 """
285 start = self.config.large
286 num = self.config.num
287 width = exposure.getWidth() - self.config.large
288 height = exposure.getHeight() - self.config.large
289 return numpy.array([rng.randint(start, width, size=num),
290 rng.randint(start, height, size=num)]).swapaxes(0, 1)
292 @timeMethod
293 def measureExposure(self, exposure, positions, title="Fringe"):
294 """Measure fringe amplitudes for an exposure
296 The fringe amplitudes are measured as the statistic within a square
297 aperture. The statistic within a larger aperture are subtracted so
298 as to remove the background.
300 Parameters
301 ----------
302 exposure : `lsst.afw.image.Exposure`
303 Exposure to measure the positions on.
304 positions : `numpy.array`
305 Two-dimensional array containing the positions to sample
306 for fringe amplitudes.
307 title : `str`, optional
308 Title used for debug out plots.
310 Returns
311 -------
312 fringes : `numpy.array`
313 Array of measured exposure values at each of the positions
314 supplied.
315 """
316 stats = afwMath.StatisticsControl()
317 stats.setNumSigmaClip(self.config.stats.clip)
318 stats.setNumIter(self.config.stats.iterations)
319 stats.setAndMask(exposure.getMaskedImage().getMask().getPlaneBitMask(self.config.stats.badMaskPlanes))
321 num = self.config.num
322 fringes = numpy.ndarray(num)
324 for i in range(num):
325 x, y = positions[i]
326 small = measure(exposure.getMaskedImage(), x, y, self.config.small, self.config.stats.stat, stats)
327 large = measure(exposure.getMaskedImage(), x, y, self.config.large, self.config.stats.stat, stats)
328 fringes[i] = small - large
330 import lsstDebug
331 display = lsstDebug.Info(__name__).display
332 if display:
333 disp = afwDisplay.Display(frame=getFrame())
334 disp.mtv(exposure, title=title)
335 if False:
336 with disp.Buffering():
337 for x, y in positions:
338 corners = numpy.array([[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]) + [[x, y]]
339 disp.line(corners*self.config.small, ctype=afwDisplay.GREEN)
340 disp.line(corners*self.config.large, ctype=afwDisplay.BLUE)
342 return fringes
344 @timeMethod
345 def solve(self, science, fringes):
346 """Solve for the scale factors with iterative clipping.
348 Parameters
349 ----------
350 science : `numpy.array`
351 Array of measured science image values at each of the
352 positions supplied.
353 fringes : `numpy.array`
354 Array of measured fringe values at each of the positions
355 supplied.
357 Returns
358 -------
359 solution : `np.array`
360 Fringe solution amplitudes for each input fringe frame.
361 rms : `float`
362 RMS error for the fit solution for this exposure.
363 """
364 import lsstDebug
365 doPlot = lsstDebug.Info(__name__).plot
367 origNum = len(science)
369 def emptyResult(msg=""):
370 """Generate an empty result for return to the user
372 There are no good pixels; doesn't matter what we return.
373 """
374 self.log.warn("Unable to solve for fringes: no good pixels%s", msg)
375 out = [0]
376 if len(fringes) > 1:
377 out = out*len(fringes)
378 return numpy.array(out), numpy.nan
380 good = numpy.where(numpy.logical_and(numpy.isfinite(science), numpy.any(numpy.isfinite(fringes), 1)))
381 science = science[good]
382 fringes = fringes[good]
383 oldNum = len(science)
384 if oldNum == 0:
385 return emptyResult()
387 # Up-front rejection to get rid of extreme, potentially troublesome values
388 # (e.g., fringe apertures that fall on objects).
389 good = select(science, self.config.clip)
390 for ff in range(fringes.shape[1]):
391 good &= select(fringes[:, ff], self.config.clip)
392 science = science[good]
393 fringes = fringes[good]
394 oldNum = len(science)
395 if oldNum == 0:
396 return emptyResult(" after initial rejection")
398 for i in range(self.config.iterations):
399 solution = self._solve(science, fringes)
400 resid = science - numpy.sum(solution*fringes, 1)
401 rms = stdev(resid)
402 good = numpy.logical_not(abs(resid) > self.config.clip*rms)
403 self.log.debug("Iteration %d: RMS=%f numGood=%d", i, rms, good.sum())
404 self.log.debug("Solution %d: %s", i, solution)
405 newNum = good.sum()
406 if newNum == 0:
407 return emptyResult(" after %d rejection iterations" % i)
409 if doPlot:
410 import matplotlib.pyplot as plot
411 for j in range(fringes.shape[1]):
412 fig = plot.figure(j)
413 fig.clf()
414 try:
415 fig.canvas._tkcanvas._root().lift() # == Tk's raise
416 except Exception:
417 pass
418 ax = fig.add_subplot(1, 1, 1)
419 adjust = science.copy()
420 others = set(range(fringes.shape[1]))
421 others.discard(j)
422 for k in others:
423 adjust -= solution[k]*fringes[:, k]
424 ax.plot(fringes[:, j], adjust, 'r.')
425 xmin = fringes[:, j].min()
426 xmax = fringes[:, j].max()
427 ymin = solution[j]*xmin
428 ymax = solution[j]*xmax
429 ax.plot([xmin, xmax], [ymin, ymax], 'b-')
430 ax.set_title("Fringe %d: %f" % (j, solution[j]))
431 ax.set_xlabel("Fringe amplitude")
432 ax.set_ylabel("Science amplitude")
433 ax.set_autoscale_on(False)
434 ax.set_xbound(lower=xmin, upper=xmax)
435 ax.set_ybound(lower=ymin, upper=ymax)
436 fig.show()
437 while True:
438 ans = input("Enter or c to continue [chp]").lower()
439 if ans in ("", "c",):
440 break
441 if ans in ("p",):
442 import pdb
443 pdb.set_trace()
444 elif ans in ("h", ):
445 print("h[elp] c[ontinue] p[db]")
447 if newNum == oldNum:
448 # Not gaining
449 break
450 oldNum = newNum
451 good = numpy.where(good)
452 science = science[good]
453 fringes = fringes[good]
455 # Final solution without rejection
456 solution = self._solve(science, fringes)
457 self.log.info("Fringe solution: %s RMS: %f Good: %d/%d", solution, rms, len(science), origNum)
458 return solution, rms
460 def _solve(self, science, fringes):
461 """Solve for the scale factors.
463 Parameters
464 ----------
465 science : `numpy.array`
466 Array of measured science image values at each of the
467 positions supplied.
468 fringes : `numpy.array`
469 Array of measured fringe values at each of the positions
470 supplied.
472 Returns
473 -------
474 solution : `np.array`
475 Fringe solution amplitudes for each input fringe frame.
476 """
477 return afwMath.LeastSquares.fromDesignMatrix(fringes, science,
478 afwMath.LeastSquares.DIRECT_SVD).getSolution()
480 def subtract(self, science, fringes, solution):
481 """Subtract the fringes.
483 Parameters
484 ----------
485 science : `lsst.afw.image.Exposure`
486 Science exposure from which to remove fringes.
487 fringes : `lsst.afw.image.Exposure` or `list` thereof
488 Calibration fringe files containing master fringe frames.
489 solution : `np.array`
490 Fringe solution amplitudes for each input fringe frame.
492 Raises
493 ------
494 RuntimeError :
495 Raised if the number of fringe frames does not match the
496 number of measured amplitudes.
497 """
498 if len(solution) != len(fringes):
499 raise RuntimeError("Number of fringe frames (%s) != number of scale factors (%s)" %
500 (len(fringes), len(solution)))
502 for s, f in zip(solution, fringes):
503 science.getMaskedImage().scaledMinus(s, f.getMaskedImage())
506def measure(mi, x, y, size, statistic, stats):
507 """Measure a statistic within an aperture
509 @param mi MaskedImage to measure
510 @param x, y Center for aperture
511 @param size Size of aperture
512 @param statistic Statistic to measure
513 @param stats StatisticsControl object
514 @return Value of statistic within aperture
515 """
516 bbox = lsst.geom.Box2I(lsst.geom.Point2I(int(x) - size, int(y - size)),
517 lsst.geom.Extent2I(2*size, 2*size))
518 subImage = mi.Factory(mi, bbox, afwImage.LOCAL)
519 return afwMath.makeStatistics(subImage, statistic, stats).getValue()
522def stdev(vector):
523 """Calculate a robust standard deviation of an array of values
525 @param vector Array of values
526 @return Standard deviation
527 """
528 q1, q3 = numpy.percentile(vector, (25, 75))
529 return 0.74*(q3 - q1)
532def select(vector, clip):
533 """Select values within 'clip' standard deviations of the median
535 Returns a boolean array.
536 """
537 q1, q2, q3 = numpy.percentile(vector, (25, 50, 75))
538 return numpy.abs(vector - q2) < clip*0.74*(q3 - q1)