lsst.cp.pipe  20.0.0-14-gbeb4550+636d787cb5
linearity.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 import numpy as np
23 
24 import lsst.pipe.base as pipeBase
26 import lsst.pex.config as pexConfig
27 
28 from lsstDebug import getDebugFrame
29 from lsst.ip.isr import (Linearizer, IsrProvenance)
30 
31 from .utils import (fitLeastSq, funcPolynomial)
32 
33 
34 __all__ = ["LinearitySolveTask", "LinearitySolveConfig"]
35 
36 
37 class LinearitySolveConnections(pipeBase.PipelineTaskConnections,
38  dimensions=("instrument", "detector")):
39 
40  inputPtc = cT.Input(
41  name="inputPtc",
42  doc="Input PTC dataset.",
43  storageClass="StructuredDataDict",
44  dimensions=("instrument", "detector"),
45  multiple=False,
46  )
47  camera = cT.Input(
48  name="camera",
49  doc="Camera Geometry definition.",
50  storageClass="Camera",
51  dimensions=("instrument", ),
52  )
53  outputLinearizer = cT.Output(
54  name="linearity",
55  doc="Output linearity measurements.",
56  storageClass="Linearizer",
57  dimensions=("instrument", "detector"),
58  )
59 
60 
61 class LinearitySolveConfig(pipeBase.PipelineTaskConfig,
62  pipelineConnections=LinearitySolveConnections):
63  """Configuration for solving the linearity from PTC dataset.
64  """
65  linearityType = pexConfig.ChoiceField(
66  dtype=str,
67  doc="Type of linearizer to construct.",
68  default="Polynomial",
69  allowed={
70  "LookupTable": "Create a lookup table solution.",
71  "Polynomial": "Create an arbitrary polynomial solution.",
72  "Squared": "Create a single order squared solution.",
73  "None": "Create a dummy solution.",
74  }
75  )
76  polynomialOrder = pexConfig.Field(
77  dtype=int,
78  doc="Degree of polynomial to fit.",
79  default=3,
80  )
81  maxLookupTableAdu = pexConfig.Field(
82  dtype=int,
83  doc="Maximum DN value for a LookupTable linearizer.",
84  default=2**18,
85  )
86 
87 
88 class LinearitySolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
89  """Fit the linearity from the PTC dataset.
90  """
91  ConfigClass = LinearitySolveConfig
92  _DefaultName = 'cpLinearitySolve'
93 
94  def runQuantum(self, butlerQC, inputRefs, outputRefs):
95  """Ensure that the input and output dimensions are passed along.
96 
97  Parameters
98  ----------
99  butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
100  Butler to operate on.
101  inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
102  Input data refs to load.
103  ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
104  Output data refs to persist.
105  """
106  inputs = butlerQC.get(inputRefs)
107 
108  # Use the dimensions to set calib/provenance information.
109  inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputPtc]
110 
111  outputs = self.run(**inputs)
112  butlerQC.put(outputs, outputRefs)
113 
114  def run(self, inputPtc, camera=None, inputDims=None):
115  """Fit non-linearity to PTC data, returning the correct Linearizer
116  object.
117 
118  Parameters
119  ----------
120  inputPtc : `lsst.cp.pipe.PtcDataset`
121  Pre-measured PTC dataset.
122  camera : `lsst.afw.cameraGeom.Camera`, optional
123  Camera geometry.
124  inputDims : `lsst.daf.butler.DataCoordinate` or `dict`, optional
125  DataIds to use to populate the output calibration.
126 
127  Returns
128  -------
129  results : `lsst.pipe.base.Struct`
130  The results struct containing:
131 
132  ``outputLinearizer`` : `lsst.ip.isr.Linearizer`
133  Final linearizer calibration.
134  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
135  Provenance data for the new calibration.
136 
137  Notes
138  -----
139  This task currently fits only polynomial-defined corrections,
140  where the correction coefficients are defined such that:
141  corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
142  These `c_i` are defined in terms of the direct polynomial fit:
143  meanVector ~ P(x=timeVector) = sum_j k_j x^j
144  such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
145  Eq. 37 of 2003.05978). The `config.polynomialOrder` defines
146  the maximum order of x^j to fit. As k_0 and k_1 are
147  degenerate with bias level and gain, they are not included in
148  the non-linearity correction.
149 
150  """
151  if camera:
152  detector = camera[inputDims['detector']]
153 
154  if self.config.linearityType == 'LookupTable':
155  table = np.zeros((len(detector), self.config.maxLookupTableAdu), dtype=np.float32)
156  tableIndex = 0
157  else:
158  table = None
159  tableIndex = None # This will fail if we increment it.
160 
161  # Initialize the linearizer.
162  linearizer = Linearizer(detector=detector, table=table, log=self.log)
163 
164  for i, amp in enumerate(detector):
165  ampName = amp.getName()
166  if (len(inputPtc.visitMask[ampName]) == 0):
167  self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.")
168  mask = np.repeat(True, len(inputPtc.rawExpTimes[ampName]))
169  else:
170  mask = inputPtc.visitMask[ampName]
171 
172  timeVector = np.array(inputPtc.rawExpTimes[ampName])[mask]
173  meanVector = np.array(inputPtc.rawMeans[ampName])[mask]
174 
175  if self.config.linearityType in ['Polynomial', 'Squared', 'LookupTable']:
176  polyFit = np.zeros(self.config.polynomialOrder + 1)
177  polyFit[1] = 1.0
178  polyFit, polyFitErr, chiSq = fitLeastSq(polyFit, timeVector, meanVector, funcPolynomial)
179 
180  # Truncate the polynomial fit
181  k1 = polyFit[1]
182  linearityFit = [-coeff/(k1**order) for order, coeff in enumerate(polyFit)]
183  significant = np.where(np.abs(linearityFit) > 1e-10, True, False)
184  self.log.info(f"Significant polynomial fits: {significant}")
185 
186  if self.config.linearityType == 'Squared':
187  linearityFit = [linearityFit[2]]
188  elif self.config.linearityType == 'LookupTable':
189  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
190  tMax = (self.config.maxLookupTableAdu - polyFit[0])/polyFit[1]
191  timeRange = np.linspace(0, tMax, self.config.maxLookupTableAdu)
192  signalIdeal = polyFit[0] + polyFit[1]*timeRange
193  signalUncorrected = funcPolynomial(polyFit, timeRange)
194  lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
195 
196  linearizer.tableData[tableIndex, :] = lookupTableRow
197  linearityFit = [tableIndex, 0]
198  tableIndex += 1
199  else:
200  polyFit = [0.0]
201  polyFitErr = [0.0]
202  chiSq = np.nan
203  linearityFit = [0.0]
204 
205  linearizer.linearityType[ampName] = 'self.config.linearityType'
206  linearizer.linearityCoeffs[ampName] = linearityFit
207  linearizer.linearityBBox[ampName] = amp.getBBox()
208  linearizer.fitParams[ampName] = polyFit
209  linearizer.fitParamsErr[ampName] = polyFitErr
210  linearizer.fitChiSq[ampName] = chiSq
211  self.debugFit('solution', timeVector, meanVector, linearizer, ampName)
212 
213  linearizer.validate()
214  linearizer.updateMetadata(setDate=True)
215  provenance = IsrProvenance(calibType='linearizer')
216 
217  return pipeBase.Struct(
218  outputLinearizer=linearizer,
219  outputProvenance=provenance,
220  )
221 
222  def debugFit(self, stepname, timeVector, meanVector, linearizer, ampName):
223  """Debug method for linearity fitting.
224 
225  Parameters
226  ----------
227  stepname : `str`
228  A label to use to check if we care to debug at a given
229  line of code.
230  timeVector : `numpy.array`
231  The values to use as the independent variable in the
232  linearity fit.
233  meanVector : `numpy.array`
234  The values to use as the dependent variable in the
235  linearity fit.
236  linearizer : `lsst.ip.isr.Linearizer`
237  The linearity correction to compare.
238  ampName : `str`
239  Amplifier name to lookup linearity correction values.
240  """
241  frame = getDebugFrame(self._display, stepname)
242  if frame:
243  import matplotlib.pyplot as plot
244  figure = plot.figure(1)
245  figure.clear()
246 
247  axes = figure.add_axes((min(timeVector), min(meanVector),
248  max(timeVector), max(meanVector)))
249  axes.plot(timeVector, meanVector, 'k+')
250 
251  axes.plot(timeVector,
252  np.polynomial.polynomial.polyval(linearizer.fitParams[ampName],
253  timeVector), 'r')
254  plot.xlabel("Exposure Time")
255  plot.ylabel("Mean Flux")
256  plot.title(f"Linearity {ampName} {linearizer.linearityType[ampName]}"
257  f" chi={linearizer.fitChiSq[ampName]}")
258  figure.show()
259 
260  prompt = "Press enter to continue: "
261  while True:
262  ans = input(prompt).lower()
263  if ans in ("", "c",):
264  break
265  plot.close()
lsst.cp.pipe.linearity.LinearitySolveTask.debugFit
def debugFit(self, stepname, timeVector, meanVector, linearizer, ampName)
Definition: linearity.py:222
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:423
lsst.cp.pipe.linearity.LinearitySolveTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: linearity.py:94
lsst.cp.pipe.linearity.LinearitySolveTask
Definition: linearity.py:88
lsst.cp.pipe.linearity.LinearitySolveConnections
Definition: linearity.py:38
lsst::pex::config
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:285
lsst.cp.pipe.linearity.LinearitySolveTask.run
def run(self, inputPtc, camera=None, inputDims=None)
Definition: linearity.py:114
lsst.cp.pipe.linearity.LinearitySolveConfig
Definition: linearity.py:62
lsst::ip::isr
lsst::pipe::base
lsst::pipe::base::connectionTypes