lsst.cp.pipe  20.0.0-31-g42917e2+deb446c958
cpFlatNormTask.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import numpy as np
22 from collections import defaultdict
23 
24 import lsst.afw.math as afwMath
25 import lsst.daf.base as dafBase
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
29 from lsst.cp.pipe.cpCombine import VignetteExposure
30 from lsst.cp.pipe.utils import ddict2dict
31 
32 from ._lookupStaticCalibration import lookupStaticCalibration
33 
34 __all__ = ["CpFlatMeasureTask", "CpFlatMeasureTaskConfig",
35  "CpFlatNormalizationTask", "CpFlatNormalizationTaskConfig"]
36 
37 
38 class CpFlatMeasureConnections(pipeBase.PipelineTaskConnections,
39  dimensions=("instrument", "exposure", "detector")):
40  inputExp = cT.Input(
41  name="postISRCCD",
42  doc="Input exposure to measure statistics from.",
43  storageClass="Exposure",
44  dimensions=("instrument", "exposure", "detector"),
45  )
46  outputStats = cT.Output(
47  name="flatStats",
48  doc="Output statistics to write.",
49  storageClass="PropertyList",
50  dimensions=("instrument", "exposure", "detector"),
51  )
52 
53 
54 class CpFlatMeasureTaskConfig(pipeBase.PipelineTaskConfig,
55  pipelineConnections=CpFlatMeasureConnections):
56  maskNameList = pexConfig.ListField(
57  dtype=str,
58  doc="Mask list to exclude from statistics calculations.",
59  default=['DETECTED', 'BAD', 'NO_DATA'],
60  )
61  doVignette = pexConfig.Field(
62  dtype=bool,
63  doc="Mask vignetted regions?",
64  default=True,
65  )
66  numSigmaClip = pexConfig.Field(
67  dtype=float,
68  doc="Rejection threshold (sigma) for statistics clipping.",
69  default=3.0,
70  )
71  clipMaxIter = pexConfig.Field(
72  dtype=int,
73  doc="Max number of clipping iterations to apply.",
74  default=3,
75  )
76 
77 
78 class CpFlatMeasureTask(pipeBase.PipelineTask,
79  pipeBase.CmdLineTask):
80  """Apply extra masking and measure image statistics.
81  """
82  ConfigClass = CpFlatMeasureTaskConfig
83  _DefaultName = "cpFlatMeasure"
84 
85  def run(self, inputExp):
86  """Mask ISR processed FLAT exposures to ensure consistent statistics.
87 
88  Parameters
89  ----------
90  inputExp : `lsst.afw.image.Exposure`
91  Post-ISR processed exposure to measure.
92 
93  Returns
94  -------
95  outputStats : `lsst.daf.base.PropertyList`
96  List containing the statistics.
97  """
98  if self.config.doVignette:
99  VignetteExposure(inputExp, doUpdateMask=True, maskPlane='BAD',
100  doSetValue=False, log=self.log)
101  mask = inputExp.getMask()
102  maskVal = mask.getPlaneBitMask(self.config.maskNameList)
103  statsControl = afwMath.StatisticsControl(self.config.numSigmaClip,
104  self.config.clipMaxIter,
105  maskVal)
106  statsControl.setAndMask(maskVal)
107 
108  outputStats = dafBase.PropertyList()
109 
110  # Detector level:
111  stats = afwMath.makeStatistics(inputExp.getMaskedImage(),
112  afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.NPOINT,
113  statsControl)
114  outputStats['DETECTOR_MEDIAN'] = stats.getValue(afwMath.MEANCLIP)
115  outputStats['DETECTOR_SIGMA'] = stats.getValue(afwMath.STDEVCLIP)
116  outputStats['DETECTOR_N'] = stats.getValue(afwMath.NPOINT)
117  self.log.info("Stats: median=%f sigma=%f n=%d",
118  outputStats['DETECTOR_MEDIAN'],
119  outputStats['DETECTOR_SIGMA'],
120  outputStats['DETECTOR_N'])
121 
122  # AMP LEVEL:
123  for ampIdx, amp in enumerate(inputExp.getDetector()):
124  ampName = amp.getName()
125  ampExp = inputExp.Factory(inputExp, amp.getBBox())
126  stats = afwMath.makeStatistics(ampExp.getMaskedImage(),
127  afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.NPOINT,
128  statsControl)
129  outputStats[f'AMP_NAME_{ampIdx}'] = ampName
130  outputStats[f'AMP_MEDIAN_{ampIdx}'] = stats.getValue(afwMath.MEANCLIP)
131  outputStats[f'AMP_SIGMA_{ampIdx}'] = stats.getValue(afwMath.STDEVCLIP)
132  outputStats[f'AMP_N_{ampIdx}'] = stats.getValue(afwMath.NPOINT)
133 
134  return pipeBase.Struct(
135  outputStats=outputStats
136  )
137 
138 
139 class CpFlatNormalizationConnections(pipeBase.PipelineTaskConnections,
140  dimensions=("instrument", "physical_filter")):
141  inputMDs = cT.Input(
142  name="cpFlatProc_metadata",
143  doc="Input metadata for each visit/detector in input set.",
144  storageClass="PropertyList",
145  dimensions=("instrument", "physical_filter", "detector", "exposure"),
146  multiple=True,
147  )
148  camera = cT.PrerequisiteInput(
149  name="camera",
150  doc="Input camera to use for gain lookup.",
151  storageClass="Camera",
152  dimensions=("instrument",),
153  lookupFunction=lookupStaticCalibration,
154  isCalibration=True,
155  )
156 
157  outputScales = cT.Output(
158  name="cpFlatNormScales",
159  doc="Output combined proposed calibration.",
160  storageClass="StructuredDataDict",
161  dimensions=("instrument", "physical_filter"),
162  )
163 
164 
165 class CpFlatNormalizationTaskConfig(pipeBase.PipelineTaskConfig,
166  pipelineConnections=CpFlatNormalizationConnections):
167  level = pexConfig.ChoiceField(
168  dtype=str,
169  doc="Which level to apply normalizations.",
170  default='DETECTOR',
171  allowed={
172  'DETECTOR': "Correct using full detector statistics.",
173  'AMP': "Correct using individual amplifiers.",
174  },
175  )
176  scaleMaxIter = pexConfig.Field(
177  dtype=int,
178  doc="Max number of iterations to use in scale solver.",
179  default=10,
180  )
181 
182 
183 class CpFlatNormalizationTask(pipeBase.PipelineTask,
184  pipeBase.CmdLineTask):
185  """Rescale merged flat frames to remove unequal screen illumination.
186  """
187  ConfigClass = CpFlatNormalizationTaskConfig
188  _DefaultName = "cpFlatNorm"
189 
190  def runQuantum(self, butlerQC, inputRefs, outputRefs):
191  inputs = butlerQC.get(inputRefs)
192 
193  # Use the dimensions of the inputs for generating
194  # output scales.
195  dimensions = [exp.dataId.byName() for exp in inputRefs.inputMDs]
196  inputs['inputDims'] = dimensions
197 
198  outputs = self.run(**inputs)
199  butlerQC.put(outputs, outputRefs)
200 
201  def run(self, inputMDs, inputDims, camera):
202  """Normalize FLAT exposures to a consistent level.
203 
204  Parameters
205  ----------
206  inputMDs : `list` [`lsst.daf.base.PropertyList`]
207  Amplifier-level metadata used to construct scales.
208  inputDims : `list` [`dict`]
209  List of dictionaries of input data dimensions/values.
210  Each list entry should contain:
211 
212  ``"exposure"``
213  exposure id value (`int`)
214  ``"detector"``
215  detector id value (`int`)
216 
217  Returns
218  -------
219  outputScales : `dict` [`dict` [`dict` [`float`]]]
220  Dictionary of scales, indexed by detector (`int`),
221  amplifier (`int`), and exposure (`int`).
222 
223  Raises
224  ------
225  KeyError
226  Raised if the input dimensions do not contain detector and
227  exposure, or if the metadata does not contain the expected
228  statistic entry.
229  """
230  expSet = sorted(set([d['exposure'] for d in inputDims]))
231  detSet = sorted(set([d['detector'] for d in inputDims]))
232 
233  expMap = {exposureId: idx for idx, exposureId in enumerate(expSet)}
234  detMap = {detectorId: idx for idx, detectorId in enumerate(detSet)}
235 
236  nExp = len(expSet)
237  nDet = len(detSet)
238  if self.config.level == 'DETECTOR':
239  bgMatrix = np.zeros((nDet, nExp))
240  bgCounts = np.ones((nDet, nExp))
241  elif self.config.level == 'AMP':
242  nAmp = len(camera[detSet[0]])
243  bgMatrix = np.zeros((nDet * nAmp, nExp))
244  bgCounts = np.ones((nDet * nAmp, nExp))
245 
246  for inMetadata, inDimensions in zip(inputMDs, inputDims):
247  try:
248  exposureId = inDimensions['exposure']
249  detectorId = inDimensions['detector']
250  except Exception as e:
251  raise KeyError("Cannot find expected dimensions in %s" % (inDimensions, )) from e
252 
253  if self.config.level == 'DETECTOR':
254  detIdx = detMap[detectorId]
255  expIdx = expMap[exposureId]
256  try:
257  value = inMetadata.get('DETECTOR_MEDIAN')
258  count = inMetadata.get('DETECTOR_N')
259  except Exception as e:
260  raise KeyError("Cannot read expected metadata string.") from e
261 
262  if np.isfinite(value):
263  bgMatrix[detIdx][expIdx] = value
264  bgCounts[detIdx][expIdx] = count
265  else:
266  bgMatrix[detIdx][expIdx] = np.nan
267  bgCounts[detIdx][expIdx] = 1
268 
269  elif self.config.level == 'AMP':
270  detector = camera[detectorId]
271  nAmp = len(detector)
272 
273  detIdx = detMap[detectorId] * nAmp
274  expIdx = expMap[exposureId]
275 
276  for ampIdx, amp in enumerate(detector):
277  try:
278  value = inMetadata.get(f'AMP_MEDIAN_{ampIdx}')
279  count = inMetadata.get(f'AMP_N_{ampIdx}')
280  except Exception as e:
281  raise KeyError("cannot read expected metadata string.") from e
282 
283  detAmpIdx = detIdx + ampIdx
284  if np.isfinite(value):
285  bgMatrix[detAmpIdx][expIdx] = value
286  bgCounts[detAmpIdx][expIdx] = count
287  else:
288  bgMatrix[detAmpIdx][expIdx] = np.nan
289  bgMatrix[detAmpIdx][expIdx] = 1
290 
291  scaleResult = self.measureScales(bgMatrix, bgCounts, iterations=self.config.scaleMaxIter)
292  expScales = scaleResult.expScales
293  detScales = scaleResult.detScales
294 
295  outputScales = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(float))))
296 
297  # Note that the enumerated "detId"/"expId" here index the
298  # "detScales" and "expScales" arrays.
299  if self.config.level == 'DETECTOR':
300  for detIdx, det in enumerate(detSet):
301  for amp in camera[det]:
302  for expIdx, exp in enumerate(expSet):
303  outputScales['expScale'][det][amp.getName()][exp] = expScales[expIdx].tolist()
304  outputScales['detScale'][det] = detScales[detIdx].tolist()
305  elif self.config.level == 'AMP':
306  for detIdx, det in enumerate(detSet):
307  for ampIdx, amp in enumerate(camera[det]):
308  for expIdx, exp in enumerate(expSet):
309  outputScales['expScale'][det][amp.getName()][exp] = expScales[expIdx].tolist()
310  detAmpIdx = detIdx + ampIdx
311  outputScales['detScale'][det][amp.getName()] = detScales[detAmpIdx].tolist()
312 
313  return pipeBase.Struct(
314  outputScales=ddict2dict(outputScales),
315  )
316 
317  def measureScales(self, bgMatrix, bgCounts=None, iterations=10):
318  """Convert backgrounds to exposure and detector components.
319 
320  Parameters
321  ----------
322  bgMatrix : `np.ndarray`, (nDetectors, nExposures)
323  Input backgrounds indexed by exposure (axis=0) and
324  detector (axis=1).
325  bgCounts : `np.ndarray`, (nDetectors, nExposures), optional
326  Input pixel counts used to in measuring bgMatrix, indexed
327  identically.
328  iterations : `int`, optional
329  Number of iterations to use in decomposition.
330 
331  Returns
332  -------
333  scaleResult : `lsst.pipe.base.Struct`
334  Result struct containing fields:
335 
336  ``vectorE``
337  Output E vector of exposure level scalings
338  (`np.array`, (nExposures)).
339  ``vectorG``
340  Output G vector of detector level scalings
341  (`np.array`, (nExposures)).
342  ``bgModel``
343  Expected model bgMatrix values, calculated from E and G
344  (`np.ndarray`, (nDetectors, nExposures)).
345 
346  Notes
347  -----
348 
349  The set of background measurements B[exposure, detector] of
350  flat frame data should be defined by a "Cartesian" product of
351  two vectors, E[exposure] and G[detector]. The E vector
352  represents the total flux incident on the focal plane. In a
353  perfect camera, this is simply the sum along the columns of B
354  (np.sum(B, axis=0)).
355 
356  However, this simple model ignores differences in detector
357  gains, the vignetting of the detectors, and the illumination
358  pattern of the source lamp. The G vector describes these
359  detector dependent differences, which should be identical over
360  different exposures. For a perfect lamp of unit total
361  intensity, this is simply the sum along the rows of B
362  (np.sum(B, axis=1)). This algorithm divides G by the total
363  flux level, to provide the relative (not absolute) scales
364  between detectors.
365 
366  The algorithm here, from pipe_drivers/constructCalibs.py and
367  from there from Eugene Magnier/PanSTARRS [1]_, attempts to
368  iteratively solve this decomposition from initial "perfect" E
369  and G vectors. The operation is performed in log space to
370  reduce the multiply and divides to linear additions and
371  subtractions.
372 
373  References
374  ----------
375  .. [1] https://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/browser/trunk/psModules/src/detrend/pmFlatNormalize.c # noqa: E501
376 
377  """
378  numExps = bgMatrix.shape[1]
379  numChips = bgMatrix.shape[0]
380  if bgCounts is None:
381  bgCounts = np.ones_like(bgMatrix)
382 
383  logMeas = np.log(bgMatrix)
384  logMeas = np.ma.masked_array(logMeas, ~np.isfinite(logMeas))
385  logG = np.zeros(numChips)
386  logE = np.array([np.average(logMeas[:, iexp] - logG,
387  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
388 
389  for iter in range(iterations):
390  logG = np.array([np.average(logMeas[ichip, :] - logE,
391  weights=bgCounts[ichip, :]) for ichip in range(numChips)])
392 
393  bad = np.isnan(logG)
394  if np.any(bad):
395  logG[bad] = logG[~bad].mean()
396 
397  logE = np.array([np.average(logMeas[:, iexp] - logG,
398  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
399  fluxLevel = np.average(np.exp(logG), weights=np.sum(bgCounts, axis=1))
400 
401  logG -= np.log(fluxLevel)
402  self.log.debug(f"ITER {iter}: Flux: {fluxLevel}")
403  self.log.debug(f"Exps: {np.exp(logE)}")
404  self.log.debug(f"{np.mean(logG)}")
405 
406  logE = np.array([np.average(logMeas[:, iexp] - logG,
407  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
408 
409  bgModel = np.exp(logE[np.newaxis, :] - logG[:, np.newaxis])
410  return pipeBase.Struct(
411  expScales=np.exp(logE),
412  detScales=np.exp(logG),
413  bgModel=bgModel,
414  )
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTask.run
def run(self, inputExp)
Definition: cpFlatNormTask.py:85
lsst.cp.pipe.cpCombine.VignetteExposure
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)
Definition: cpCombine.py:528
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: cpFlatNormTask.py:190
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTaskConfig
Definition: cpFlatNormTask.py:55
lsst.cp.pipe.cpCombine
Definition: cpCombine.py:1
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.run
def run(self, inputMDs, inputDims, camera)
Definition: cpFlatNormTask.py:201
lsst.cp.pipe.utils
Definition: utils.py:1
lsst::pex::config
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationConnections
Definition: cpFlatNormTask.py:140
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTaskConfig
Definition: cpFlatNormTask.py:166
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTask
Definition: cpFlatNormTask.py:79
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask
Definition: cpFlatNormTask.py:184
lsst::daf::base
lsst::afw::math
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.measureScales
def measureScales(self, bgMatrix, bgCounts=None, iterations=10)
Definition: cpFlatNormTask.py:317
lsst::pipe::base
lsst::pipe::base::connectionTypes
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureConnections
Definition: cpFlatNormTask.py:39
lsst.cp.pipe.utils.ddict2dict
def ddict2dict(d)
Definition: utils.py:637