lsst.cp.pipe  20.0.0-8-gea2affd+05e11544cc
cpFlatNormTask.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import numpy as np
22 from collections import defaultdict
23 
24 import lsst.afw.math as afwMath
25 import lsst.daf.base as dafBase
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
29 from lsst.cp.pipe.cpCombine import VignetteExposure
30 
31 __all__ = ["CpFlatMeasureTask", "CpFlatMeasureTaskConfig",
32  "CpFlatNormalizationTask", "CpFlatNormalizationTaskConfig"]
33 
34 
35 class CpFlatMeasureConnections(pipeBase.PipelineTaskConnections,
36  dimensions=("instrument", "exposure", "detector")):
37  inputExp = cT.Input(
38  name="postISRCCD",
39  doc="Input exposure to measure statistics from.",
40  storageClass="ExposureF",
41  dimensions=("instrument", "exposure", "detector"),
42  )
43  outputStats = cT.Output(
44  name="flatStats",
45  doc="Output statistics to write.",
46  storageClass="PropertyList",
47  dimensions=("instrument", "exposure", "detector"),
48  )
49 
50 
51 class CpFlatMeasureTaskConfig(pipeBase.PipelineTaskConfig,
52  pipelineConnections=CpFlatMeasureConnections):
53  maskNameList = pexConfig.ListField(
54  dtype=str,
55  doc="Mask list to exclude from statistics calculations.",
56  default=['DETECTED', 'BAD', 'NO_DATA'],
57  )
58  doVignette = pexConfig.Field(
59  dtype=bool,
60  doc="Mask vignetted regions?",
61  default=True,
62  )
63  numSigmaClip = pexConfig.Field(
64  dtype=float,
65  doc="Rejection threshold (sigma) for statistics clipping.",
66  default=3.0,
67  )
68  clipMaxIter = pexConfig.Field(
69  dtype=int,
70  doc="Max number of clipping iterations to apply.",
71  default=3,
72  )
73 
74 
75 class CpFlatMeasureTask(pipeBase.PipelineTask,
76  pipeBase.CmdLineTask):
77  """Apply extra masking and measure image statistics.
78  """
79  ConfigClass = CpFlatMeasureTaskConfig
80  _DefaultName = "cpFlatMeasure"
81 
82  def run(self, inputExp):
83  """Mask ISR processed FLAT exposures to ensure consistent statistics.
84 
85  Parameters
86  ----------
87  inputExp : `lsst.afw.image.Exposure`
88  Post-ISR processed exposure to measure.
89 
90  Returns
91  -------
92  outputStats : `lsst.daf.base.PropertyList`
93  List containing the statistics.
94  """
95  if self.config.doVignette:
96  VignetteExposure(inputExp, doUpdateMask=True, maskPlane='BAD',
97  doSetValue=False, log=self.log)
98  mask = inputExp.getMask()
99  maskVal = mask.getPlaneBitMask(self.config.maskNameList)
100  statsControl = afwMath.StatisticsControl(self.config.numSigmaClip,
101  self.config.clipMaxIter,
102  maskVal)
103  statsControl.setAndMask(maskVal)
104 
105  outputStats = dafBase.PropertyList()
106 
107  # Detector level:
108  stats = afwMath.makeStatistics(inputExp.getMaskedImage(),
109  afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.NPOINT,
110  statsControl)
111  outputStats['DETECTOR_MEDIAN'] = stats.getValue(afwMath.MEANCLIP)
112  outputStats['DETECTOR_SIGMA'] = stats.getValue(afwMath.STDEVCLIP)
113  outputStats['DETECTOR_N'] = stats.getValue(afwMath.NPOINT)
114  self.log.info("Stats: median=%f sigma=%f n=%d",
115  outputStats['DETECTOR_MEDIAN'],
116  outputStats['DETECTOR_SIGMA'],
117  outputStats['DETECTOR_N'])
118 
119  # AMP LEVEL:
120  for ampIdx, amp in enumerate(inputExp.getDetector()):
121  ampName = amp.getName()
122  ampExp = inputExp.Factory(inputExp, amp.getBBox())
123  stats = afwMath.makeStatistics(ampExp.getMaskedImage(),
124  afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.NPOINT,
125  statsControl)
126  outputStats[f'AMP_NAME_{ampIdx}'] = ampName
127  outputStats[f'AMP_MEDIAN_{ampIdx}'] = stats.getValue(afwMath.MEANCLIP)
128  outputStats[f'AMP_SIGMA_{ampIdx}'] = stats.getValue(afwMath.STDEVCLIP)
129  outputStats[f'AMP_N_{ampIdx}'] = stats.getValue(afwMath.NPOINT)
130 
131  return pipeBase.Struct(
132  outputStats=outputStats
133  )
134 
135 
136 class CpFlatNormalizationConnections(pipeBase.PipelineTaskConnections,
137  dimensions=("instrument", "physical_filter")):
138  inputMDs = cT.Input(
139  name="cpFlatProc_metadata",
140  doc="Input metadata for each visit/detector in input set.",
141  storageClass="PropertyList",
142  dimensions=("instrument", "physical_filter", "detector", "exposure"),
143  multiple=True,
144  )
145  camera = cT.PrerequisiteInput(
146  name="camera",
147  doc="Input camera to use for gain lookup.",
148  storageClass="Camera",
149  dimensions=("instrument", "calibration_label"),
150  )
151 
152  outputScales = cT.Output(
153  name="cpFlatNormScales",
154  doc="Output combined proposed calibration.",
155  storageClass="StructuredDataDict",
156  dimensions=("instrument", "physical_filter"),
157  )
158 
159 
160 class CpFlatNormalizationTaskConfig(pipeBase.PipelineTaskConfig,
161  pipelineConnections=CpFlatNormalizationConnections):
162  level = pexConfig.ChoiceField(
163  dtype=str,
164  doc="Which level to apply normalizations.",
165  default='DETECTOR',
166  allowed={
167  'DETECTOR': "Correct using full detector statistics.",
168  'AMP': "Correct using individual amplifiers.",
169  },
170  )
171  scaleMaxIter = pexConfig.Field(
172  dtype=int,
173  doc="Max number of iterations to use in scale solver.",
174  default=10,
175  )
176 
177 
178 class CpFlatNormalizationTask(pipeBase.PipelineTask,
179  pipeBase.CmdLineTask):
180  """Rescale merged flat frames to remove unequal screen illumination.
181  """
182  ConfigClass = CpFlatNormalizationTaskConfig
183  _DefaultName = "cpFlatNorm"
184 
185  def runQuantum(self, butlerQC, inputRefs, outputRefs):
186  inputs = butlerQC.get(inputRefs)
187 
188  # Use the dimensions of the inputs for generating
189  # output scales.
190  dimensions = [exp.dataId.byName() for exp in inputRefs.inputMDs]
191  inputs['inputDims'] = dimensions
192 
193  outputs = self.run(**inputs)
194  butlerQC.put(outputs, outputRefs)
195 
196  def run(self, inputMDs, inputDims, camera):
197  """Normalize FLAT exposures to a consistent level.
198 
199  Parameters
200  ----------
201  inputMDs : `list` [`lsst.daf.base.PropertyList`]
202  Amplifier-level metadata used to construct scales.
203  inputDims : `list` [`dict`]
204  List of dictionaries of input data dimensions/values.
205  Each list entry should contain:
206 
207  ``"exposure"``
208  exposure id value (`int`)
209  ``"detector"``
210  detector id value (`int`)
211 
212  Returns
213  -------
214  outputScales : `dict` [`dict` [`dict` [`float`]]]
215  Dictionary of scales, indexed by detector (`int`),
216  amplifier (`int`), and exposure (`int`).
217 
218  Raises
219  ------
220  KeyError
221  Raised if the input dimensions do not contain detector and
222  exposure, or if the metadata does not contain the expected
223  statistic entry.
224  """
225  expSet = sorted(set([d['exposure'] for d in inputDims]))
226  detSet = sorted(set([d['detector'] for d in inputDims]))
227 
228  expMap = {exposureId: idx for idx, exposureId in enumerate(expSet)}
229  detMap = {detectorId: idx for idx, detectorId in enumerate(detSet)}
230 
231  nExp = len(expSet)
232  nDet = len(detSet)
233  if self.config.level == 'DETECTOR':
234  bgMatrix = np.zeros((nDet, nExp))
235  bgCounts = np.ones((nDet, nExp))
236  elif self.config.level == 'AMP':
237  nAmp = len(camera[0])
238  bgMatrix = np.zeros((nDet * nAmp, nExp))
239  bgCounts = np.ones((nDet * nAmp, nExp))
240 
241  for inMetadata, inDimensions in zip(inputMDs, inputDims):
242  try:
243  exposureId = inDimensions['exposure']
244  detectorId = inDimensions['detector']
245  except Exception as e:
246  raise KeyError("Cannot find expected dimensions in %s" % (inDimensions, )) from e
247 
248  if self.config.level == 'DETECTOR':
249  detId = detMap[detectorId]
250  expId = expMap[exposureId]
251  try:
252  value = inMetadata.get('DETECTOR_MEDIAN')
253  count = inMetadata.get('DETECTOR_N')
254  except Exception as e:
255  raise KeyError("Cannot read expected metadata string.") from e
256 
257  if np.isfinite(value):
258  bgMatrix[detId][expId] = value
259  bgCounts[detId][expId] = count
260  else:
261  bgMatrix[detId][expId] = np.nan
262  bgCounts[detId][expId] = 1
263 
264  elif self.config.level == 'AMP':
265  detector = camera[detectorId]
266  nAmp = len(detector)
267 
268  detId = detMap[detectorId] * nAmp
269  expId = expMap[exposureId]
270 
271  for ampIdx, amp in enumerate(detector):
272  try:
273  value = inMetadata.get(f'AMP_MEDIAN_{ampIdx}')
274  count = inMetadata.get(f'AMP_N_{ampIdx}')
275  except Exception as e:
276  raise KeyError("cannot read expected metadata string.") from e
277 
278  detAmpId = detId + ampIdx
279  if np.isfinite(value):
280  bgMatrix[detAmpId][expId] = value
281  bgCounts[detAmpId][expId] = count
282  else:
283  bgMatrix[detAmpId][expId] = np.nan
284  bgMatrix[detAmpId][expId] = 1
285 
286  scaleResult = self.measureScales(bgMatrix, bgCounts, iterations=self.config.scaleMaxIter)
287  expScales = scaleResult.expScales
288  detScales = scaleResult.detScales
289 
290  outputScales = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(float))))
291 
292  if self.config.level == 'DETECTOR':
293  for detId, det in enumerate(detSet):
294  for amp in camera[detId]:
295  for expId, exp in enumerate(expSet):
296  outputScales['expScale'][det][amp.getName()][exp] = expScales[expId].tolist()
297  outputScales['detScale'][det] = detScales[detId].tolist()
298  elif self.config.level == 'AMP':
299  for detId, det in enumerate(detSet):
300  for ampIdx, amp in enumerate(camera[detId]):
301  for expId, exp in enumerate(expSet):
302  outputScales['expScale'][det][amp.getName()][exp] = expScales[expId].tolist()
303  detAmpId = detId + ampIdx
304  outputScales['detScale'][det][amp.getName()] = detScales[detAmpId].tolist()
305 
306  return pipeBase.Struct(
307  outputScales=outputScales,
308  )
309 
310  def measureScales(self, bgMatrix, bgCounts=None, iterations=10):
311  """Convert backgrounds to exposure and detector components.
312 
313  Parameters
314  ----------
315  bgMatrix : `np.ndarray`, (nDetectors, nExposures)
316  Input backgrounds indexed by exposure (axis=0) and
317  detector (axis=1).
318  bgCounts : `np.ndarray`, (nDetectors, nExposures), optional
319  Input pixel counts used to in measuring bgMatrix, indexed
320  identically.
321  iterations : `int`, optional
322  Number of iterations to use in decomposition.
323 
324  Returns
325  -------
326  scaleResult : `lsst.pipe.base.Struct`
327  Result struct containing fields:
328 
329  ``vectorE``
330  Output E vector of exposure level scalings
331  (`np.array`, (nExposures)).
332  ``vectorG``
333  Output G vector of detector level scalings
334  (`np.array`, (nExposures)).
335  ``bgModel``
336  Expected model bgMatrix values, calculated from E and G
337  (`np.ndarray`, (nDetectors, nExposures)).
338 
339  Notes
340  -----
341 
342  The set of background measurements B[exposure, detector] of
343  flat frame data should be defined by a "Cartesian" product of
344  two vectors, E[exposure] and G[detector]. The E vector
345  represents the total flux incident on the focal plane. In a
346  perfect camera, this is simply the sum along the columns of B
347  (np.sum(B, axis=0)).
348 
349  However, this simple model ignores differences in detector
350  gains, the vignetting of the detectors, and the illumination
351  pattern of the source lamp. The G vector describes these
352  detector dependent differences, which should be identical over
353  different exposures. For a perfect lamp of unit total
354  intensity, this is simply the sum along the rows of B
355  (np.sum(B, axis=1)). This algorithm divides G by the total
356  flux level, to provide the relative (not absolute) scales
357  between detectors.
358 
359  The algorithm here, from pipe_drivers/constructCalibs.py and
360  from there from Eugene Magnier/PanSTARRS [1]_, attempts to
361  iteratively solve this decomposition from initial "perfect" E
362  and G vectors. The operation is performed in log space to
363  reduce the multiply and divides to linear additions and
364  subtractions.
365 
366  References
367  ----------
368  .. [1] https://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/browser/trunk/psModules/src/detrend/pmFlatNormalize.c # noqa: E501
369 
370  """
371  numExps = bgMatrix.shape[1]
372  numChips = bgMatrix.shape[0]
373  if bgCounts is None:
374  bgCounts = np.ones_like(bgMatrix)
375 
376  logMeas = np.log(bgMatrix)
377  logMeas = np.ma.masked_array(logMeas, ~np.isfinite(logMeas))
378  logG = np.zeros(numChips)
379  logE = np.array([np.average(logMeas[:, iexp] - logG,
380  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
381 
382  for iter in range(iterations):
383  logG = np.array([np.average(logMeas[ichip, :] - logE,
384  weights=bgCounts[ichip, :]) for ichip in range(numChips)])
385 
386  bad = np.isnan(logG)
387  if np.any(bad):
388  logG[bad] = logG[~bad].mean()
389 
390  logE = np.array([np.average(logMeas[:, iexp] - logG,
391  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
392  fluxLevel = np.average(np.exp(logG), weights=np.sum(bgCounts, axis=1))
393 
394  logG -= np.log(fluxLevel)
395  self.log.debug(f"ITER {iter}: Flux: {fluxLevel}")
396  self.log.debug(f"Exps: {np.exp(logE)}")
397  self.log.debug(f"{np.mean(logG)}")
398 
399  logE = np.array([np.average(logMeas[:, iexp] - logG,
400  weights=bgCounts[:, iexp]) for iexp in range(numExps)])
401 
402  bgModel = np.exp(logE[np.newaxis, :] - logG[:, np.newaxis])
403  return pipeBase.Struct(
404  expScales=np.exp(logE),
405  detScales=np.exp(logG),
406  bgModel=bgModel,
407  )
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTask.run
def run(self, inputExp)
Definition: cpFlatNormTask.py:82
lsst.cp.pipe.cpCombine.VignetteExposure
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)
Definition: cpCombine.py:523
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: cpFlatNormTask.py:185
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTaskConfig
Definition: cpFlatNormTask.py:52
lsst.cp.pipe.cpCombine
Definition: cpCombine.py:1
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.run
def run(self, inputMDs, inputDims, camera)
Definition: cpFlatNormTask.py:196
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationConnections
Definition: cpFlatNormTask.py:137
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTaskConfig
Definition: cpFlatNormTask.py:161
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureTask
Definition: cpFlatNormTask.py:76
lsst::pex::config
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask
Definition: cpFlatNormTask.py:179
lsst::daf::base
lsst::afw::math
lsst.cp.pipe.cpFlatNormTask.CpFlatNormalizationTask.measureScales
def measureScales(self, bgMatrix, bgCounts=None, iterations=10)
Definition: cpFlatNormTask.py:310
lsst::pipe::base
lsst::pipe::base::connectionTypes
lsst.cp.pipe.cpFlatNormTask.CpFlatMeasureConnections
Definition: cpFlatNormTask.py:36