lsst.cp.pipe  20.0.0-21-g1f5da01+74cacf985d
linearity.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 import numpy as np
23 
24 import lsst.pipe.base as pipeBase
26 import lsst.pex.config as pexConfig
27 
28 from lsstDebug import getDebugFrame
29 from lsst.ip.isr import (Linearizer, IsrProvenance)
30 
31 from .utils import (fitLeastSq, funcPolynomial)
32 
33 
34 __all__ = ["LinearitySolveTask", "LinearitySolveConfig"]
35 
36 
37 class LinearitySolveConnections(pipeBase.PipelineTaskConnections,
38  dimensions=("instrument", "detector")):
39 
40  inputPtc = cT.Input(
41  name="inputPtc",
42  doc="Input PTC dataset.",
43  storageClass="StructuredDataDict",
44  dimensions=("instrument", "detector"),
45  multiple=False,
46  )
47  camera = cT.Input(
48  name="camera",
49  doc="Camera Geometry definition.",
50  storageClass="Camera",
51  dimensions=("instrument", ),
52  )
53  outputLinearizer = cT.Output(
54  name="linearity",
55  doc="Output linearity measurements.",
56  storageClass="Linearizer",
57  dimensions=("instrument", "detector"),
58  isCalibration=True,
59  )
60 
61 
62 class LinearitySolveConfig(pipeBase.PipelineTaskConfig,
63  pipelineConnections=LinearitySolveConnections):
64  """Configuration for solving the linearity from PTC dataset.
65  """
66  linearityType = pexConfig.ChoiceField(
67  dtype=str,
68  doc="Type of linearizer to construct.",
69  default="Polynomial",
70  allowed={
71  "LookupTable": "Create a lookup table solution.",
72  "Polynomial": "Create an arbitrary polynomial solution.",
73  "Squared": "Create a single order squared solution.",
74  "None": "Create a dummy solution.",
75  }
76  )
77  polynomialOrder = pexConfig.Field(
78  dtype=int,
79  doc="Degree of polynomial to fit.",
80  default=3,
81  )
82  maxLookupTableAdu = pexConfig.Field(
83  dtype=int,
84  doc="Maximum DN value for a LookupTable linearizer.",
85  default=2**18,
86  )
87 
88 
89 class LinearitySolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
90  """Fit the linearity from the PTC dataset.
91  """
92  ConfigClass = LinearitySolveConfig
93  _DefaultName = 'cpLinearitySolve'
94 
95  def runQuantum(self, butlerQC, inputRefs, outputRefs):
96  """Ensure that the input and output dimensions are passed along.
97 
98  Parameters
99  ----------
100  butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
101  Butler to operate on.
102  inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
103  Input data refs to load.
104  ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
105  Output data refs to persist.
106  """
107  inputs = butlerQC.get(inputRefs)
108 
109  # Use the dimensions to set calib/provenance information.
110  inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputPtc]
111 
112  outputs = self.run(**inputs)
113  butlerQC.put(outputs, outputRefs)
114 
115  def run(self, inputPtc, camera=None, inputDims=None):
116  """Fit non-linearity to PTC data, returning the correct Linearizer
117  object.
118 
119  Parameters
120  ----------
121  inputPtc : `lsst.cp.pipe.PtcDataset`
122  Pre-measured PTC dataset.
123  camera : `lsst.afw.cameraGeom.Camera`, optional
124  Camera geometry.
125  inputDims : `lsst.daf.butler.DataCoordinate` or `dict`, optional
126  DataIds to use to populate the output calibration.
127 
128  Returns
129  -------
130  results : `lsst.pipe.base.Struct`
131  The results struct containing:
132 
133  ``outputLinearizer`` : `lsst.ip.isr.Linearizer`
134  Final linearizer calibration.
135  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
136  Provenance data for the new calibration.
137 
138  Notes
139  -----
140  This task currently fits only polynomial-defined corrections,
141  where the correction coefficients are defined such that:
142  corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
143  These `c_i` are defined in terms of the direct polynomial fit:
144  meanVector ~ P(x=timeVector) = sum_j k_j x^j
145  such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
146  Eq. 37 of 2003.05978). The `config.polynomialOrder` defines
147  the maximum order of x^j to fit. As k_0 and k_1 are
148  degenerate with bias level and gain, they are not included in
149  the non-linearity correction.
150 
151  """
152  if camera:
153  detector = camera[inputDims['detector']]
154 
155  if self.config.linearityType == 'LookupTable':
156  table = np.zeros((len(detector), self.config.maxLookupTableAdu), dtype=np.float32)
157  tableIndex = 0
158  else:
159  table = None
160  tableIndex = None # This will fail if we increment it.
161 
162  # Initialize the linearizer.
163  linearizer = Linearizer(detector=detector, table=table, log=self.log)
164 
165  for i, amp in enumerate(detector):
166  ampName = amp.getName()
167  if (len(inputPtc.expIdMask[ampName]) == 0):
168  self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.")
169  mask = np.repeat(True, len(inputPtc.rawExpTimes[ampName]))
170  else:
171  mask = inputPtc.expIdMask[ampName]
172 
173  timeVector = np.array(inputPtc.rawExpTimes[ampName])[mask]
174  meanVector = np.array(inputPtc.rawMeans[ampName])[mask]
175 
176  if self.config.linearityType in ['Polynomial', 'Squared', 'LookupTable']:
177  polyFit = np.zeros(self.config.polynomialOrder + 1)
178  polyFit[1] = 1.0
179  polyFit, polyFitErr, chiSq = fitLeastSq(polyFit, timeVector, meanVector, funcPolynomial)
180 
181  # Truncate the polynomial fit
182  k1 = polyFit[1]
183  linearityFit = [-coeff/(k1**order) for order, coeff in enumerate(polyFit)]
184  significant = np.where(np.abs(linearityFit) > 1e-10, True, False)
185  self.log.info(f"Significant polynomial fits: {significant}")
186 
187  if self.config.linearityType == 'Squared':
188  linearityFit = [linearityFit[2]]
189  elif self.config.linearityType == 'LookupTable':
190  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
191  tMax = (self.config.maxLookupTableAdu - polyFit[0])/polyFit[1]
192  timeRange = np.linspace(0, tMax, self.config.maxLookupTableAdu)
193  signalIdeal = polyFit[0] + polyFit[1]*timeRange
194  signalUncorrected = funcPolynomial(polyFit, timeRange)
195  lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
196 
197  linearizer.tableData[tableIndex, :] = lookupTableRow
198  linearityFit = [tableIndex, 0]
199  tableIndex += 1
200  else:
201  polyFit = [0.0]
202  polyFitErr = [0.0]
203  chiSq = np.nan
204  linearityFit = [0.0]
205 
206  linearizer.linearityType[ampName] = 'self.config.linearityType'
207  linearizer.linearityCoeffs[ampName] = linearityFit
208  linearizer.linearityBBox[ampName] = amp.getBBox()
209  linearizer.fitParams[ampName] = polyFit
210  linearizer.fitParamsErr[ampName] = polyFitErr
211  linearizer.fitChiSq[ampName] = chiSq
212  self.debugFit('solution', timeVector, meanVector, linearizer, ampName)
213 
214  linearizer.validate()
215  linearizer.updateMetadata(setDate=True)
216  provenance = IsrProvenance(calibType='linearizer')
217 
218  return pipeBase.Struct(
219  outputLinearizer=linearizer,
220  outputProvenance=provenance,
221  )
222 
223  def debugFit(self, stepname, timeVector, meanVector, linearizer, ampName):
224  """Debug method for linearity fitting.
225 
226  Parameters
227  ----------
228  stepname : `str`
229  A label to use to check if we care to debug at a given
230  line of code.
231  timeVector : `numpy.array`
232  The values to use as the independent variable in the
233  linearity fit.
234  meanVector : `numpy.array`
235  The values to use as the dependent variable in the
236  linearity fit.
237  linearizer : `lsst.ip.isr.Linearizer`
238  The linearity correction to compare.
239  ampName : `str`
240  Amplifier name to lookup linearity correction values.
241  """
242  frame = getDebugFrame(self._display, stepname)
243  if frame:
244  import matplotlib.pyplot as plot
245  figure = plot.figure(1)
246  figure.clear()
247 
248  axes = figure.add_axes((min(timeVector), min(meanVector),
249  max(timeVector), max(meanVector)))
250  axes.plot(timeVector, meanVector, 'k+')
251 
252  axes.plot(timeVector,
253  np.polynomial.polynomial.polyval(linearizer.fitParams[ampName],
254  timeVector), 'r')
255  plot.xlabel("Exposure Time")
256  plot.ylabel("Mean Flux")
257  plot.title(f"Linearity {ampName} {linearizer.linearityType[ampName]}"
258  f" chi={linearizer.fitChiSq[ampName]}")
259  figure.show()
260 
261  prompt = "Press enter to continue: "
262  while True:
263  ans = input(prompt).lower()
264  if ans in ("", "c",):
265  break
266  plot.close()
lsst.cp.pipe.linearity.LinearitySolveTask.debugFit
def debugFit(self, stepname, timeVector, meanVector, linearizer, ampName)
Definition: linearity.py:223
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:423
lsst.cp.pipe.linearity.LinearitySolveTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: linearity.py:95
lsst.cp.pipe.linearity.LinearitySolveTask
Definition: linearity.py:89
lsst::pex::config
lsst.cp.pipe.linearity.LinearitySolveConnections
Definition: linearity.py:38
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:285
lsst.cp.pipe.linearity.LinearitySolveTask.run
def run(self, inputPtc, camera=None, inputDims=None)
Definition: linearity.py:115
lsst.cp.pipe.linearity.LinearitySolveConfig
Definition: linearity.py:63
lsst::ip::isr
lsst::pipe::base
lsst::pipe::base::connectionTypes