Coverage for python/lsst/ip/isr/fringe.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy
24import lsst.geom
25import lsst.afw.image as afwImage
26import lsst.afw.math as afwMath
27import lsst.afw.display as afwDisplay
29from lsst.pipe.base import Task, Struct, timeMethod
30from lsst.pex.config import Config, Field, ListField, ConfigField
32afwDisplay.setDefaultMaskTransparency(75)
35def getFrame():
36 """Produce a new frame number each time"""
37 getFrame.frame += 1
38 return getFrame.frame
41getFrame.frame = 0
44class FringeStatisticsConfig(Config):
45 """Options for measuring fringes on an exposure"""
46 badMaskPlanes = ListField(dtype=str, default=["SAT"], doc="Ignore pixels with these masks")
47 stat = Field(dtype=int, default=int(afwMath.MEDIAN), doc="Statistic to use")
48 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold")
49 iterations = Field(dtype=int, default=3, doc="Number of fitting iterations")
50 rngSeedOffset = Field(dtype=int, default=0,
51 doc="Offset to the random number generator seed (full seed includes exposure ID)")
54class FringeConfig(Config):
55 """Fringe subtraction options"""
56 filters = ListField(dtype=str, default=[], doc="Only fringe-subtract these filters")
57 useFilterAliases = Field(dtype=bool, default=False, doc="Search filter aliases during check.")
58 num = Field(dtype=int, default=30000, doc="Number of fringe measurements")
59 small = Field(dtype=int, default=3, doc="Half-size of small (fringe) measurements (pixels)")
60 large = Field(dtype=int, default=30, doc="Half-size of large (background) measurements (pixels)")
61 iterations = Field(dtype=int, default=20, doc="Number of fitting iterations")
62 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold")
63 stats = ConfigField(dtype=FringeStatisticsConfig, doc="Statistics for measuring fringes")
64 pedestal = Field(dtype=bool, default=False, doc="Remove fringe pedestal?")
67class FringeTask(Task):
68 """Task to remove fringes from a science exposure
70 We measure fringe amplitudes at random positions on the science exposure
71 and at the same positions on the (potentially multiple) fringe frames
72 and solve for the scales simultaneously.
73 """
74 ConfigClass = FringeConfig
75 _DefaultName = 'isrFringe'
77 def readFringes(self, dataRef, assembler=None):
78 """Read the fringe frame(s), and pack data into a Struct
80 The current implementation assumes only a single fringe frame and
81 will have to be updated to support multi-mode fringe subtraction.
83 This implementation could be optimised by persisting the fringe
84 positions and fluxes.
86 Parameters
87 ----------
88 dataRef : `daf.butler.butlerSubset.ButlerDataRef`
89 Butler reference for the exposure that will have fringing
90 removed.
91 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
92 An instance of AssembleCcdTask (for assembling fringe
93 frames).
95 Returns
96 -------
97 fringeData : `pipeBase.Struct`
98 Struct containing fringe data:
99 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
100 Calibration fringe files containing master fringe frames.
101 - ``seed`` : `int`, optional
102 Seed for random number generation.
103 """
104 try:
105 fringe = dataRef.get("fringe", immediate=True)
106 except Exception as e:
107 raise RuntimeError("Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e))
109 return self.loadFringes(fringe, assembler)
111 def loadFringes(self, fringeExp, expId=0, assembler=None):
112 """Pack the fringe data into a Struct.
114 This method moves the struct parsing code into a butler
115 generation agnostic handler.
117 Parameters
118 ----------
119 fringeExp : `lsst.afw.exposure.Exposure`
120 The exposure containing the fringe data.
121 expId : `int`, optional
122 Exposure id to be fringe corrected, used to set RNG seed.
123 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
124 An instance of AssembleCcdTask (for assembling fringe
125 frames).
127 Returns
128 -------
129 fringeData : `pipeBase.Struct`
130 Struct containing fringe data:
131 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
132 Calibration fringe files containing master fringe frames.
133 - ``seed`` : `int`, optional
134 Seed for random number generation.
135 """
136 if assembler is not None:
137 fringeExp = assembler.assembleCcd(fringeExp)
139 if expId is None:
140 seed = self.config.stats.rngSeedOffset
141 else:
142 print(f"{self.config.stats.rngSeedOffset} {expId}")
143 seed = self.config.stats.rngSeedOffset + expId
145 # Seed for numpy.random.RandomState must be convertable to a 32 bit unsigned integer
146 seed %= 2**32
148 return Struct(fringes=fringeExp,
149 seed=seed)
151 @timeMethod
152 def run(self, exposure, fringes, seed=None):
153 """Remove fringes from the provided science exposure.
155 Primary method of FringeTask. Fringes are only subtracted if the
156 science exposure has a filter listed in the configuration.
158 Parameters
159 ----------
160 exposure : `lsst.afw.image.Exposure`
161 Science exposure from which to remove fringes.
162 fringes : `lsst.afw.image.Exposure` or `list` thereof
163 Calibration fringe files containing master fringe frames.
164 seed : `int`, optional
165 Seed for random number generation.
167 Returns
168 -------
169 solution : `np.array`
170 Fringe solution amplitudes for each input fringe frame.
171 rms : `float`
172 RMS error for the fit solution for this exposure.
173 """
174 import lsstDebug
175 display = lsstDebug.Info(__name__).display
177 if not self.checkFilter(exposure):
178 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
179 return
181 if seed is None:
182 seed = self.config.stats.rngSeedOffset
183 rng = numpy.random.RandomState(seed=seed)
185 if not hasattr(fringes, '__iter__'):
186 fringes = [fringes]
188 mask = exposure.getMaskedImage().getMask()
189 for fringe in fringes:
190 fringe.getMaskedImage().getMask().__ior__(mask)
191 if self.config.pedestal:
192 self.removePedestal(fringe)
194 positions = self.generatePositions(fringes[0], rng)
195 fluxes = numpy.ndarray([self.config.num, len(fringes)])
196 for i, f in enumerate(fringes):
197 fluxes[:, i] = self.measureExposure(f, positions, title="Fringe frame")
199 expFringes = self.measureExposure(exposure, positions, title="Science")
200 solution, rms = self.solve(expFringes, fluxes)
201 self.subtract(exposure, fringes, solution)
202 if display:
203 afwDisplay.Display(frame=getFrame()).mtv(exposure, title="Fringe subtracted")
204 return solution, rms
206 @timeMethod
207 def runDataRef(self, exposure, dataRef, assembler=None):
208 """Remove fringes from the provided science exposure.
210 Retrieve fringes from butler dataRef provided and remove from
211 provided science exposure. Fringes are only subtracted if the
212 science exposure has a filter listed in the configuration.
214 Parameters
215 ----------
216 exposure : `lsst.afw.image.Exposure`
217 Science exposure from which to remove fringes.
218 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
219 Butler reference to the exposure. Used to find
220 appropriate fringe data.
221 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
222 An instance of AssembleCcdTask (for assembling fringe
223 frames).
225 Returns
226 -------
227 solution : `np.array`
228 Fringe solution amplitudes for each input fringe frame.
229 rms : `float`
230 RMS error for the fit solution for this exposure.
231 """
232 if not self.checkFilter(exposure):
233 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
234 return
235 fringeStruct = self.readFringes(dataRef, assembler=assembler)
236 return self.run(exposure, **fringeStruct.getDict())
238 def checkFilter(self, exposure):
239 """Check whether we should fringe-subtract the science exposure.
241 Parameters
242 ----------
243 exposure : `lsst.afw.image.Exposure`
244 Exposure to check the filter of.
246 Returns
247 -------
248 needsFringe : `bool`
249 If True, then the exposure has a filter listed in the
250 configuration, and should have the fringe applied.
251 """
252 filterObj = afwImage.Filter(exposure.getFilter().getId())
253 if self.config.useFilterAliases:
254 filterNameSet = set(filterObj.getAliases() + [filterObj.getName()])
255 else:
256 filterNameSet = set([filterObj.getName(), ])
257 return bool(len(filterNameSet.intersection(self.config.filters)))
259 def removePedestal(self, fringe):
260 """Remove pedestal from fringe exposure.
262 Parameters
263 ----------
264 fringe : `lsst.afw.image.Exposure`
265 Fringe data to subtract the pedestal value from.
266 """
267 stats = afwMath.StatisticsControl()
268 stats.setNumSigmaClip(self.config.stats.clip)
269 stats.setNumIter(self.config.stats.iterations)
270 mi = fringe.getMaskedImage()
271 pedestal = afwMath.makeStatistics(mi, afwMath.MEDIAN, stats).getValue()
272 self.log.info("Removing fringe pedestal: %f", pedestal)
273 mi -= pedestal
275 def generatePositions(self, exposure, rng):
276 """Generate a random distribution of positions for measuring fringe amplitudes.
278 Parameters
279 ----------
280 exposure : `lsst.afw.image.Exposure`
281 Exposure to measure the positions on.
282 rng : `numpy.random.RandomState`
283 Random number generator to use.
285 Returns
286 -------
287 positions : `numpy.array`
288 Two-dimensional array containing the positions to sample
289 for fringe amplitudes.
290 """
291 start = self.config.large
292 num = self.config.num
293 width = exposure.getWidth() - self.config.large
294 height = exposure.getHeight() - self.config.large
295 return numpy.array([rng.randint(start, width, size=num),
296 rng.randint(start, height, size=num)]).swapaxes(0, 1)
298 @timeMethod
299 def measureExposure(self, exposure, positions, title="Fringe"):
300 """Measure fringe amplitudes for an exposure
302 The fringe amplitudes are measured as the statistic within a square
303 aperture. The statistic within a larger aperture are subtracted so
304 as to remove the background.
306 Parameters
307 ----------
308 exposure : `lsst.afw.image.Exposure`
309 Exposure to measure the positions on.
310 positions : `numpy.array`
311 Two-dimensional array containing the positions to sample
312 for fringe amplitudes.
313 title : `str`, optional
314 Title used for debug out plots.
316 Returns
317 -------
318 fringes : `numpy.array`
319 Array of measured exposure values at each of the positions
320 supplied.
321 """
322 stats = afwMath.StatisticsControl()
323 stats.setNumSigmaClip(self.config.stats.clip)
324 stats.setNumIter(self.config.stats.iterations)
325 stats.setAndMask(exposure.getMaskedImage().getMask().getPlaneBitMask(self.config.stats.badMaskPlanes))
327 num = self.config.num
328 fringes = numpy.ndarray(num)
330 for i in range(num):
331 x, y = positions[i]
332 small = measure(exposure.getMaskedImage(), x, y, self.config.small, self.config.stats.stat, stats)
333 large = measure(exposure.getMaskedImage(), x, y, self.config.large, self.config.stats.stat, stats)
334 fringes[i] = small - large
336 import lsstDebug
337 display = lsstDebug.Info(__name__).display
338 if display:
339 disp = afwDisplay.Display(frame=getFrame())
340 disp.mtv(exposure, title=title)
341 if False:
342 with disp.Buffering():
343 for x, y in positions:
344 corners = numpy.array([[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]) + [[x, y]]
345 disp.line(corners*self.config.small, ctype=afwDisplay.GREEN)
346 disp.line(corners*self.config.large, ctype=afwDisplay.BLUE)
348 return fringes
350 @timeMethod
351 def solve(self, science, fringes):
352 """Solve for the scale factors with iterative clipping.
354 Parameters
355 ----------
356 science : `numpy.array`
357 Array of measured science image values at each of the
358 positions supplied.
359 fringes : `numpy.array`
360 Array of measured fringe values at each of the positions
361 supplied.
363 Returns
364 -------
365 solution : `np.array`
366 Fringe solution amplitudes for each input fringe frame.
367 rms : `float`
368 RMS error for the fit solution for this exposure.
369 """
370 import lsstDebug
371 doPlot = lsstDebug.Info(__name__).plot
373 origNum = len(science)
375 def emptyResult(msg=""):
376 """Generate an empty result for return to the user
378 There are no good pixels; doesn't matter what we return.
379 """
380 self.log.warn("Unable to solve for fringes: no good pixels%s", msg)
381 out = [0]
382 if len(fringes) > 1:
383 out = out*len(fringes)
384 return numpy.array(out), numpy.nan
386 good = numpy.where(numpy.logical_and(numpy.isfinite(science), numpy.any(numpy.isfinite(fringes), 1)))
387 science = science[good]
388 fringes = fringes[good]
389 oldNum = len(science)
390 if oldNum == 0:
391 return emptyResult()
393 # Up-front rejection to get rid of extreme, potentially troublesome values
394 # (e.g., fringe apertures that fall on objects).
395 good = select(science, self.config.clip)
396 for ff in range(fringes.shape[1]):
397 good &= select(fringes[:, ff], self.config.clip)
398 science = science[good]
399 fringes = fringes[good]
400 oldNum = len(science)
401 if oldNum == 0:
402 return emptyResult(" after initial rejection")
404 for i in range(self.config.iterations):
405 solution = self._solve(science, fringes)
406 resid = science - numpy.sum(solution*fringes, 1)
407 rms = stdev(resid)
408 good = numpy.logical_not(abs(resid) > self.config.clip*rms)
409 self.log.debug("Iteration %d: RMS=%f numGood=%d", i, rms, good.sum())
410 self.log.debug("Solution %d: %s", i, solution)
411 newNum = good.sum()
412 if newNum == 0:
413 return emptyResult(" after %d rejection iterations" % i)
415 if doPlot:
416 import matplotlib.pyplot as plot
417 for j in range(fringes.shape[1]):
418 fig = plot.figure(j)
419 fig.clf()
420 try:
421 fig.canvas._tkcanvas._root().lift() # == Tk's raise
422 except Exception:
423 pass
424 ax = fig.add_subplot(1, 1, 1)
425 adjust = science.copy()
426 others = set(range(fringes.shape[1]))
427 others.discard(j)
428 for k in others:
429 adjust -= solution[k]*fringes[:, k]
430 ax.plot(fringes[:, j], adjust, 'r.')
431 xmin = fringes[:, j].min()
432 xmax = fringes[:, j].max()
433 ymin = solution[j]*xmin
434 ymax = solution[j]*xmax
435 ax.plot([xmin, xmax], [ymin, ymax], 'b-')
436 ax.set_title("Fringe %d: %f" % (j, solution[j]))
437 ax.set_xlabel("Fringe amplitude")
438 ax.set_ylabel("Science amplitude")
439 ax.set_autoscale_on(False)
440 ax.set_xbound(lower=xmin, upper=xmax)
441 ax.set_ybound(lower=ymin, upper=ymax)
442 fig.show()
443 while True:
444 ans = input("Enter or c to continue [chp]").lower()
445 if ans in ("", "c",):
446 break
447 if ans in ("p",):
448 import pdb
449 pdb.set_trace()
450 elif ans in ("h", ):
451 print("h[elp] c[ontinue] p[db]")
453 if newNum == oldNum:
454 # Not gaining
455 break
456 oldNum = newNum
457 good = numpy.where(good)
458 science = science[good]
459 fringes = fringes[good]
461 # Final solution without rejection
462 solution = self._solve(science, fringes)
463 self.log.info("Fringe solution: %s RMS: %f Good: %d/%d", solution, rms, len(science), origNum)
464 return solution, rms
466 def _solve(self, science, fringes):
467 """Solve for the scale factors.
469 Parameters
470 ----------
471 science : `numpy.array`
472 Array of measured science image values at each of the
473 positions supplied.
474 fringes : `numpy.array`
475 Array of measured fringe values at each of the positions
476 supplied.
478 Returns
479 -------
480 solution : `np.array`
481 Fringe solution amplitudes for each input fringe frame.
482 """
483 return afwMath.LeastSquares.fromDesignMatrix(fringes, science,
484 afwMath.LeastSquares.DIRECT_SVD).getSolution()
486 def subtract(self, science, fringes, solution):
487 """Subtract the fringes.
489 Parameters
490 ----------
491 science : `lsst.afw.image.Exposure`
492 Science exposure from which to remove fringes.
493 fringes : `lsst.afw.image.Exposure` or `list` thereof
494 Calibration fringe files containing master fringe frames.
495 solution : `np.array`
496 Fringe solution amplitudes for each input fringe frame.
498 Raises
499 ------
500 RuntimeError :
501 Raised if the number of fringe frames does not match the
502 number of measured amplitudes.
503 """
504 if len(solution) != len(fringes):
505 raise RuntimeError("Number of fringe frames (%s) != number of scale factors (%s)" %
506 (len(fringes), len(solution)))
508 for s, f in zip(solution, fringes):
509 science.getMaskedImage().scaledMinus(s, f.getMaskedImage())
512def measure(mi, x, y, size, statistic, stats):
513 """Measure a statistic within an aperture
515 @param mi MaskedImage to measure
516 @param x, y Center for aperture
517 @param size Size of aperture
518 @param statistic Statistic to measure
519 @param stats StatisticsControl object
520 @return Value of statistic within aperture
521 """
522 bbox = lsst.geom.Box2I(lsst.geom.Point2I(int(x) - size, int(y - size)),
523 lsst.geom.Extent2I(2*size, 2*size))
524 subImage = mi.Factory(mi, bbox, afwImage.LOCAL)
525 return afwMath.makeStatistics(subImage, statistic, stats).getValue()
528def stdev(vector):
529 """Calculate a robust standard deviation of an array of values
531 @param vector Array of values
532 @return Standard deviation
533 """
534 q1, q3 = numpy.percentile(vector, (25, 75))
535 return 0.74*(q3 - q1)
538def select(vector, clip):
539 """Select values within 'clip' standard deviations of the median
541 Returns a boolean array.
542 """
543 q1, q2, q3 = numpy.percentile(vector, (25, 50, 75))
544 return numpy.abs(vector - q2) < clip*0.74*(q3 - q1)