32 afwDisplay.setDefaultMaskTransparency(75)
36 """Produce a new frame number each time"""
45 """Options for measuring fringes on an exposure"""
46 badMaskPlanes = ListField(dtype=str, default=[
"SAT"], doc=
"Ignore pixels with these masks")
47 stat = Field(dtype=int, default=int(afwMath.MEDIAN), doc=
"Statistic to use")
48 clip = Field(dtype=float, default=3.0, doc=
"Sigma clip threshold")
49 iterations = Field(dtype=int, default=3, doc=
"Number of fitting iterations")
50 rngSeedOffset = Field(dtype=int, default=0,
51 doc=
"Offset to the random number generator seed (full seed includes exposure ID)")
55 """Fringe subtraction options"""
57 filters = ListField(dtype=str, default=[], doc=
"Only fringe-subtract these filters")
59 useFilterAliases = Field(dtype=bool, default=
False, doc=
"Search filter aliases during check.",
60 deprecated=(
"Removed with no replacement (FilterLabel has no aliases)."
61 "Will be removed after v22."))
62 num = Field(dtype=int, default=30000, doc=
"Number of fringe measurements")
63 small = Field(dtype=int, default=3, doc=
"Half-size of small (fringe) measurements (pixels)")
64 large = Field(dtype=int, default=30, doc=
"Half-size of large (background) measurements (pixels)")
65 iterations = Field(dtype=int, default=20, doc=
"Number of fitting iterations")
66 clip = Field(dtype=float, default=3.0, doc=
"Sigma clip threshold")
67 stats = ConfigField(dtype=FringeStatisticsConfig, doc=
"Statistics for measuring fringes")
68 pedestal = Field(dtype=bool, default=
False, doc=
"Remove fringe pedestal?")
72 """Task to remove fringes from a science exposure
74 We measure fringe amplitudes at random positions on the science exposure
75 and at the same positions on the (potentially multiple) fringe frames
76 and solve for the scales simultaneously.
78 ConfigClass = FringeConfig
79 _DefaultName =
'isrFringe'
82 """Read the fringe frame(s), and pack data into a Struct
84 The current implementation assumes only a single fringe frame and
85 will have to be updated to support multi-mode fringe subtraction.
87 This implementation could be optimised by persisting the fringe
92 dataRef : `daf.butler.butlerSubset.ButlerDataRef`
93 Butler reference for the exposure that will have fringing
95 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
96 An instance of AssembleCcdTask (for assembling fringe
101 fringeData : `pipeBase.Struct`
102 Struct containing fringe data:
103 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
104 Calibration fringe files containing master fringe frames.
105 - ``seed`` : `int`, optional
106 Seed for random number generation.
109 fringe = dataRef.get(
"fringe", immediate=
True)
110 except Exception
as e:
111 raise RuntimeError(
"Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e))
113 return self.
loadFringesloadFringes(fringe, assembler)
116 """Pack the fringe data into a Struct.
118 This method moves the struct parsing code into a butler
119 generation agnostic handler.
123 fringeExp : `lsst.afw.exposure.Exposure`
124 The exposure containing the fringe data.
125 expId : `int`, optional
126 Exposure id to be fringe corrected, used to set RNG seed.
127 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
128 An instance of AssembleCcdTask (for assembling fringe
133 fringeData : `pipeBase.Struct`
134 Struct containing fringe data:
135 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof
136 Calibration fringe files containing master fringe frames.
137 - ``seed`` : `int`, optional
138 Seed for random number generation.
140 if assembler
is not None:
141 fringeExp = assembler.assembleCcd(fringeExp)
144 seed = self.config.stats.rngSeedOffset
146 print(f
"{self.config.stats.rngSeedOffset} {expId}")
147 seed = self.config.stats.rngSeedOffset + expId
152 return Struct(fringes=fringeExp,
156 def run(self, exposure, fringes, seed=None):
157 """Remove fringes from the provided science exposure.
159 Primary method of FringeTask. Fringes are only subtracted if the
160 science exposure has a filter listed in the configuration.
164 exposure : `lsst.afw.image.Exposure`
165 Science exposure from which to remove fringes.
166 fringes : `lsst.afw.image.Exposure` or `list` thereof
167 Calibration fringe files containing master fringe frames.
168 seed : `int`, optional
169 Seed for random number generation.
173 solution : `np.array`
174 Fringe solution amplitudes for each input fringe frame.
176 RMS error for the fit solution for this exposure.
182 self.log.info(
"Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
186 seed = self.config.stats.rngSeedOffset
187 rng = numpy.random.RandomState(seed=seed)
189 if not hasattr(fringes,
'__iter__'):
192 mask = exposure.getMaskedImage().getMask()
193 for fringe
in fringes:
194 fringe.getMaskedImage().getMask().__ior__(mask)
195 if self.config.pedestal:
199 fluxes = numpy.ndarray([self.config.num, len(fringes)])
200 for i, f
in enumerate(fringes):
201 fluxes[:, i] = self.
measureExposuremeasureExposure(f, positions, title=
"Fringe frame")
203 expFringes = self.
measureExposuremeasureExposure(exposure, positions, title=
"Science")
204 solution, rms = self.
solvesolve(expFringes, fluxes)
205 self.
subtractsubtract(exposure, fringes, solution)
207 afwDisplay.Display(frame=
getFrame()).mtv(exposure, title=
"Fringe subtracted")
212 """Remove fringes from the provided science exposure.
214 Retrieve fringes from butler dataRef provided and remove from
215 provided science exposure. Fringes are only subtracted if the
216 science exposure has a filter listed in the configuration.
220 exposure : `lsst.afw.image.Exposure`
221 Science exposure from which to remove fringes.
222 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
223 Butler reference to the exposure. Used to find
224 appropriate fringe data.
225 assembler : `lsst.ip.isr.AssembleCcdTask`, optional
226 An instance of AssembleCcdTask (for assembling fringe
231 solution : `np.array`
232 Fringe solution amplitudes for each input fringe frame.
234 RMS error for the fit solution for this exposure.
237 self.log.info(
"Filter not found in FringeTaskConfig.filters. Skipping fringe correction.")
239 fringeStruct = self.
readFringesreadFringes(dataRef, assembler=assembler)
240 return self.
runrun(exposure, **fringeStruct.getDict())
243 """Check whether we should fringe-subtract the science exposure.
247 exposure : `lsst.afw.image.Exposure`
248 Exposure to check the filter of.
253 If True, then the exposure has a filter listed in the
254 configuration, and should have the fringe applied.
256 filterObj = afwImage.Filter(exposure.getFilter().getId())
258 if self.config.useFilterAliases:
259 filterNameSet = set(filterObj.getAliases() + [filterObj.getName()])
261 filterNameSet = set([filterObj.getName(), ])
262 return bool(len(filterNameSet.intersection(self.config.filters)))
265 """Remove pedestal from fringe exposure.
269 fringe : `lsst.afw.image.Exposure`
270 Fringe data to subtract the pedestal value from.
272 stats = afwMath.StatisticsControl()
273 stats.setNumSigmaClip(self.config.stats.clip)
274 stats.setNumIter(self.config.stats.iterations)
275 mi = fringe.getMaskedImage()
276 pedestal = afwMath.makeStatistics(mi, afwMath.MEDIAN, stats).getValue()
277 self.log.info(
"Removing fringe pedestal: %f", pedestal)
281 """Generate a random distribution of positions for measuring fringe amplitudes.
285 exposure : `lsst.afw.image.Exposure`
286 Exposure to measure the positions on.
287 rng : `numpy.random.RandomState`
288 Random number generator to use.
292 positions : `numpy.array`
293 Two-dimensional array containing the positions to sample
294 for fringe amplitudes.
296 start = self.config.large
297 num = self.config.num
298 width = exposure.getWidth() - self.config.large
299 height = exposure.getHeight() - self.config.large
300 return numpy.array([rng.randint(start, width, size=num),
301 rng.randint(start, height, size=num)]).swapaxes(0, 1)
305 """Measure fringe amplitudes for an exposure
307 The fringe amplitudes are measured as the statistic within a square
308 aperture. The statistic within a larger aperture are subtracted so
309 as to remove the background.
313 exposure : `lsst.afw.image.Exposure`
314 Exposure to measure the positions on.
315 positions : `numpy.array`
316 Two-dimensional array containing the positions to sample
317 for fringe amplitudes.
318 title : `str`, optional
319 Title used for debug out plots.
323 fringes : `numpy.array`
324 Array of measured exposure values at each of the positions
327 stats = afwMath.StatisticsControl()
328 stats.setNumSigmaClip(self.config.stats.clip)
329 stats.setNumIter(self.config.stats.iterations)
330 stats.setAndMask(exposure.getMaskedImage().getMask().getPlaneBitMask(self.config.stats.badMaskPlanes))
332 num = self.config.num
333 fringes = numpy.ndarray(num)
337 small =
measure(exposure.getMaskedImage(), x, y, self.config.small, self.config.stats.stat, stats)
338 large =
measure(exposure.getMaskedImage(), x, y, self.config.large, self.config.stats.stat, stats)
339 fringes[i] = small - large
344 disp = afwDisplay.Display(frame=
getFrame())
345 disp.mtv(exposure, title=title)
347 with disp.Buffering():
348 for x, y
in positions:
349 corners = numpy.array([[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]) + [[x, y]]
350 disp.line(corners*self.config.small, ctype=afwDisplay.GREEN)
351 disp.line(corners*self.config.large, ctype=afwDisplay.BLUE)
357 """Solve for the scale factors with iterative clipping.
361 science : `numpy.array`
362 Array of measured science image values at each of the
364 fringes : `numpy.array`
365 Array of measured fringe values at each of the positions
370 solution : `np.array`
371 Fringe solution amplitudes for each input fringe frame.
373 RMS error for the fit solution for this exposure.
378 origNum = len(science)
380 def emptyResult(msg=""):
381 """Generate an empty result for return to the user
383 There are no good pixels; doesn't matter what we return.
385 self.log.warn(
"Unable to solve for fringes: no good pixels%s", msg)
388 out = out*len(fringes)
389 return numpy.array(out), numpy.nan
391 good = numpy.where(numpy.logical_and(numpy.isfinite(science), numpy.any(numpy.isfinite(fringes), 1)))
392 science = science[good]
393 fringes = fringes[good]
394 oldNum = len(science)
400 good =
select(science, self.config.clip)
401 for ff
in range(fringes.shape[1]):
402 good &=
select(fringes[:, ff], self.config.clip)
403 science = science[good]
404 fringes = fringes[good]
405 oldNum = len(science)
407 return emptyResult(
" after initial rejection")
409 for i
in range(self.config.iterations):
410 solution = self.
_solve_solve(science, fringes)
411 resid = science - numpy.sum(solution*fringes, 1)
413 good = numpy.logical_not(abs(resid) > self.config.clip*rms)
414 self.log.debug(
"Iteration %d: RMS=%f numGood=%d", i, rms, good.sum())
415 self.log.debug(
"Solution %d: %s", i, solution)
418 return emptyResult(
" after %d rejection iterations" % i)
421 import matplotlib.pyplot
as plot
422 for j
in range(fringes.shape[1]):
426 fig.canvas._tkcanvas._root().lift()
429 ax = fig.add_subplot(1, 1, 1)
430 adjust = science.copy()
431 others = set(range(fringes.shape[1]))
434 adjust -= solution[k]*fringes[:, k]
435 ax.plot(fringes[:, j], adjust,
'r.')
436 xmin = fringes[:, j].min()
437 xmax = fringes[:, j].max()
438 ymin = solution[j]*xmin
439 ymax = solution[j]*xmax
440 ax.plot([xmin, xmax], [ymin, ymax],
'b-')
441 ax.set_title(
"Fringe %d: %f" % (j, solution[j]))
442 ax.set_xlabel(
"Fringe amplitude")
443 ax.set_ylabel(
"Science amplitude")
444 ax.set_autoscale_on(
False)
445 ax.set_xbound(lower=xmin, upper=xmax)
446 ax.set_ybound(lower=ymin, upper=ymax)
449 ans = input(
"Enter or c to continue [chp]").lower()
450 if ans
in (
"",
"c",):
456 print(
"h[elp] c[ontinue] p[db]")
462 good = numpy.where(good)
463 science = science[good]
464 fringes = fringes[good]
467 solution = self.
_solve_solve(science, fringes)
468 self.log.info(
"Fringe solution: %s RMS: %f Good: %d/%d", solution, rms, len(science), origNum)
471 def _solve(self, science, fringes):
472 """Solve for the scale factors.
476 science : `numpy.array`
477 Array of measured science image values at each of the
479 fringes : `numpy.array`
480 Array of measured fringe values at each of the positions
485 solution : `np.array`
486 Fringe solution amplitudes for each input fringe frame.
488 return afwMath.LeastSquares.fromDesignMatrix(fringes, science,
489 afwMath.LeastSquares.DIRECT_SVD).getSolution()
492 """Subtract the fringes.
496 science : `lsst.afw.image.Exposure`
497 Science exposure from which to remove fringes.
498 fringes : `lsst.afw.image.Exposure` or `list` thereof
499 Calibration fringe files containing master fringe frames.
500 solution : `np.array`
501 Fringe solution amplitudes for each input fringe frame.
506 Raised if the number of fringe frames does not match the
507 number of measured amplitudes.
509 if len(solution) != len(fringes):
510 raise RuntimeError(
"Number of fringe frames (%s) != number of scale factors (%s)" %
511 (len(fringes), len(solution)))
513 for s, f
in zip(solution, fringes):
514 science.getMaskedImage().scaledMinus(s, f.getMaskedImage())
517 def measure(mi, x, y, size, statistic, stats):
518 """Measure a statistic within an aperture
520 @param mi MaskedImage to measure
521 @param x, y Center for aperture
522 @param size Size of aperture
523 @param statistic Statistic to measure
524 @param stats StatisticsControl object
525 @return Value of statistic within aperture
529 subImage = mi.Factory(mi, bbox, afwImage.LOCAL)
530 return afwMath.makeStatistics(subImage, statistic, stats).getValue()
534 """Calculate a robust standard deviation of an array of values
536 @param vector Array of values
537 @return Standard deviation
539 q1, q3 = numpy.percentile(vector, (25, 75))
540 return 0.74*(q3 - q1)
544 """Select values within 'clip' standard deviations of the median
546 Returns a boolean array.
548 q1, q2, q3 = numpy.percentile(vector, (25, 50, 75))
549 return numpy.abs(vector - q2) < clip*0.74*(q3 - q1)
def solve(self, science, fringes)
def readFringes(self, dataRef, assembler=None)
def measureExposure(self, exposure, positions, title="Fringe")
def loadFringes(self, fringeExp, expId=0, assembler=None)
def subtract(self, science, fringes, solution)
def removePedestal(self, fringe)
def runDataRef(self, exposure, dataRef, assembler=None)
def generatePositions(self, exposure, rng)
def run(self, exposure, fringes, seed=None)
def _solve(self, science, fringes)
def checkFilter(self, exposure)
def measure(mi, x, y, size, statistic, stats)