Coverage for python/lsst/cp/pipe/linearity.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.image as afwImage
25import lsst.afw.math as afwMath
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.pex.config as pexConfig
30from lsstDebug import getDebugFrame
31from lsst.ip.isr import (Linearizer, IsrProvenance)
33from .utils import (funcPolynomial, irlsFit)
36__all__ = ["LinearitySolveTask", "LinearitySolveConfig", "MeasureLinearityTask"]
39class LinearitySolveConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "detector")):
41 inputPtc = cT.Input(
42 name="inputPtc",
43 doc="Input PTC dataset.",
44 storageClass="StructuredDataDict",
45 dimensions=("instrument", "detector"),
46 multiple=False,
47 )
48 camera = cT.Input(
49 name="camera",
50 doc="Camera Geometry definition.",
51 storageClass="Camera",
52 dimensions=("instrument", ),
53 )
54 outputLinearizer = cT.Output(
55 name="linearity",
56 doc="Output linearity measurements.",
57 storageClass="Linearizer",
58 dimensions=("instrument", "detector"),
59 isCalibration=True,
60 )
63class LinearitySolveConfig(pipeBase.PipelineTaskConfig,
64 pipelineConnections=LinearitySolveConnections):
65 """Configuration for solving the linearity from PTC dataset.
66 """
67 linearityType = pexConfig.ChoiceField(
68 dtype=str,
69 doc="Type of linearizer to construct.",
70 default="Squared",
71 allowed={
72 "LookupTable": "Create a lookup table solution.",
73 "Polynomial": "Create an arbitrary polynomial solution.",
74 "Squared": "Create a single order squared solution.",
75 "Spline": "Create a spline based solution.",
76 "None": "Create a dummy solution.",
77 }
78 )
79 polynomialOrder = pexConfig.Field(
80 dtype=int,
81 doc="Degree of polynomial to fit.",
82 default=3,
83 )
84 splineKnots = pexConfig.Field(
85 dtype=int,
86 doc="Number of spline knots to use in fit.",
87 default=10,
88 )
89 maxLookupTableAdu = pexConfig.Field(
90 dtype=int,
91 doc="Maximum DN value for a LookupTable linearizer.",
92 default=2**18,
93 )
94 maxLinearAdu = pexConfig.Field(
95 dtype=float,
96 doc="Maximum DN value to use to estimate linear term.",
97 default=20000.0,
98 )
99 minLinearAdu = pexConfig.Field(
100 dtype=float,
101 doc="Minimum DN value to use to estimate linear term.",
102 default=2000.0,
103 )
104 nSigmaClipLinear = pexConfig.Field(
105 dtype=float,
106 doc="Maximum deviation from linear solution for Poissonian noise.",
107 default=5.0,
108 )
111class LinearitySolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
112 """Fit the linearity from the PTC dataset.
113 """
114 ConfigClass = LinearitySolveConfig
115 _DefaultName = 'cpLinearitySolve'
117 def runQuantum(self, butlerQC, inputRefs, outputRefs):
118 """Ensure that the input and output dimensions are passed along.
120 Parameters
121 ----------
122 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
123 Butler to operate on.
124 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
125 Input data refs to load.
126 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
127 Output data refs to persist.
128 """
129 inputs = butlerQC.get(inputRefs)
131 # Use the dimensions to set calib/provenance information.
132 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputPtc]
134 outputs = self.run(**inputs)
135 butlerQC.put(outputs, outputRefs)
137 def run(self, inputPtc, camera, inputDims):
138 """Fit non-linearity to PTC data, returning the correct Linearizer
139 object.
141 Parameters
142 ----------
143 inputPtc : `lsst.cp.pipe.PtcDataset`
144 Pre-measured PTC dataset.
145 camera : `lsst.afw.cameraGeom.Camera`
146 Camera geometry.
147 inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
148 DataIds to use to populate the output calibration.
150 Returns
151 -------
152 results : `lsst.pipe.base.Struct`
153 The results struct containing:
155 ``outputLinearizer`` : `lsst.ip.isr.Linearizer`
156 Final linearizer calibration.
157 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
158 Provenance data for the new calibration.
160 Notes
161 -----
162 This task currently fits only polynomial-defined corrections,
163 where the correction coefficients are defined such that:
164 corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
165 These `c_i` are defined in terms of the direct polynomial fit:
166 meanVector ~ P(x=timeVector) = sum_j k_j x^j
167 such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
168 Eq. 37 of 2003.05978). The `config.polynomialOrder` or
169 `config.splineKnots` define the maximum order of x^j to fit.
170 As k_0 and k_1 are degenerate with bias level and gain, they
171 are not included in the non-linearity correction.
172 """
173 detector = camera[inputDims['detector']]
174 if self.config.linearityType == 'LookupTable':
175 table = np.zeros((len(detector), self.config.maxLookupTableAdu), dtype=np.float32)
176 tableIndex = 0
177 else:
178 table = None
179 tableIndex = None # This will fail if we increment it.
181 if self.config.linearityType == 'Spline':
182 fitOrder = self.config.splineKnots
183 else:
184 fitOrder = self.config.polynomialOrder
186 # Initialize the linearizer.
187 linearizer = Linearizer(detector=detector, table=table, log=self.log)
189 for i, amp in enumerate(detector):
190 ampName = amp.getName()
191 if (len(inputPtc.expIdMask[ampName]) == 0):
192 self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.")
193 mask = np.repeat(True, len(inputPtc.expIdMask[ampName]))
194 else:
195 mask = inputPtc.expIdMask[ampName]
197 inputAbscissa = np.array(inputPtc.rawExpTimes[ampName])[mask]
198 inputOrdinate = np.array(inputPtc.rawMeans[ampName])[mask]
200 # Determine proxy-to-linear-flux transformation
201 fluxMask = inputOrdinate < self.config.maxLinearAdu
202 lowMask = inputOrdinate > self.config.minLinearAdu
203 fluxMask = fluxMask & lowMask
204 linearAbscissa = inputAbscissa[fluxMask]
205 linearOrdinate = inputOrdinate[fluxMask]
207 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 100.0], linearAbscissa,
208 linearOrdinate, funcPolynomial)
209 # Convert this proxy-to-flux fit into an expected linear flux
210 linearOrdinate = linearFit[0] + linearFit[1] * inputAbscissa
212 # Exclude low end outliers
213 threshold = self.config.nSigmaClipLinear * np.sqrt(linearOrdinate)
214 fluxMask = np.abs(inputOrdinate - linearOrdinate) < threshold
215 linearOrdinate = linearOrdinate[fluxMask]
216 fitOrdinate = inputOrdinate[fluxMask]
217 self.debugFit('linearFit', inputAbscissa, inputOrdinate, linearOrdinate, fluxMask, ampName)
218 # Do fits
219 if self.config.linearityType in ['Polynomial', 'Squared', 'LookupTable']:
220 polyFit = np.zeros(fitOrder + 1)
221 polyFit[1] = 1.0
222 polyFit, polyFitErr, chiSq, weights = irlsFit(polyFit, linearOrdinate,
223 fitOrdinate, funcPolynomial)
225 # Truncate the polynomial fit
226 k1 = polyFit[1]
227 linearityFit = [-coeff/(k1**order) for order, coeff in enumerate(polyFit)]
228 significant = np.where(np.abs(linearityFit) > 1e-10, True, False)
229 self.log.info(f"Significant polynomial fits: {significant}")
231 modelOrdinate = funcPolynomial(polyFit, linearAbscissa)
232 self.debugFit('polyFit', linearAbscissa, fitOrdinate, modelOrdinate, None, ampName)
234 if self.config.linearityType == 'Squared':
235 linearityFit = [linearityFit[2]]
236 elif self.config.linearityType == 'LookupTable':
237 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
238 tMax = (self.config.maxLookupTableAdu - polyFit[0])/polyFit[1]
239 timeRange = np.linspace(0, tMax, self.config.maxLookupTableAdu)
240 signalIdeal = polyFit[0] + polyFit[1]*timeRange
241 signalUncorrected = funcPolynomial(polyFit, timeRange)
242 lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has correction
244 linearizer.tableData[tableIndex, :] = lookupTableRow
245 linearityFit = [tableIndex, 0]
246 tableIndex += 1
247 elif self.config.linearityType in ['Spline']:
248 # See discussion in `lsst.ip.isr.linearize.py` before modifying.
249 numPerBin, binEdges = np.histogram(linearOrdinate, bins=fitOrder)
250 with np.errstate(invalid="ignore"):
251 # Algorithm note: With the counts of points per
252 # bin above, the next histogram calculates the
253 # values to put in each bin by weighting each
254 # point by the correction value.
255 values = np.histogram(linearOrdinate, bins=fitOrder,
256 weights=(inputOrdinate[fluxMask] - linearOrdinate))[0]/numPerBin
258 # After this is done, the binCenters are
259 # calculated by weighting by the value we're
260 # binning over. This ensures that widely
261 # spaced/poorly sampled data aren't assigned to
262 # the midpoint of the bin (as could be done using
263 # the binEdges above), but to the weighted mean of
264 # the inputs. Note that both histograms are
265 # scaled by the count per bin to normalize what
266 # the histogram returns (a sum of the points
267 # inside) into an average.
268 binCenters = np.histogram(linearOrdinate, bins=fitOrder,
269 weights=linearOrdinate)[0]/numPerBin
270 values = values[numPerBin > 0]
271 binCenters = binCenters[numPerBin > 0]
273 self.debugFit('splineFit', binCenters, np.abs(values), values, None, ampName)
274 interp = afwMath.makeInterpolate(binCenters.tolist(), values.tolist(),
275 afwMath.stringToInterpStyle("AKIMA_SPLINE"))
276 modelOrdinate = linearOrdinate + interp.interpolate(linearOrdinate)
277 self.debugFit('splineFit', linearOrdinate, fitOrdinate, modelOrdinate, None, ampName)
279 # If we exclude a lot of points, we may end up with
280 # less than fitOrder points. Pad out the low-flux end
281 # to ensure equal lengths.
282 if len(binCenters) != fitOrder:
283 padN = fitOrder - len(binCenters)
284 binCenters = np.pad(binCenters, (padN, 0), 'linear_ramp',
285 end_values=(binCenters.min() - 1.0, ))
286 # This stores the correction, which is zero at low values.
287 values = np.pad(values, (padN, 0))
289 # Pack the spline into a single array.
290 linearityFit = np.concatenate((binCenters.tolist(), values.tolist())).tolist()
291 polyFit = [0.0]
292 polyFitErr = [0.0]
293 chiSq = np.nan
294 else:
295 polyFit = [0.0]
296 polyFitErr = [0.0]
297 chiSq = np.nan
298 linearityFit = [0.0]
300 linearizer.linearityType[ampName] = self.config.linearityType
301 linearizer.linearityCoeffs[ampName] = np.array(linearityFit)
302 linearizer.linearityBBox[ampName] = amp.getBBox()
303 linearizer.fitParams[ampName] = np.array(polyFit)
304 linearizer.fitParamsErr[ampName] = np.array(polyFitErr)
305 linearizer.fitChiSq[ampName] = chiSq
307 image = afwImage.ImageF(len(inputOrdinate), 1)
308 image.getArray()[:, :] = inputOrdinate
309 linearizeFunction = linearizer.getLinearityTypeByName(linearizer.linearityType[ampName])
310 linearizeFunction()(image,
311 **{'coeffs': linearizer.linearityCoeffs[ampName],
312 'table': linearizer.tableData,
313 'log': linearizer.log})
314 linearizeModel = image.getArray()[0, :]
316 self.debugFit('solution', inputOrdinate[fluxMask], linearOrdinate,
317 linearizeModel[fluxMask], None, ampName)
319 linearizer.hasLinearity = True
320 linearizer.validate()
321 linearizer.updateMetadata(camera=camera, detector=detector, filterName='NONE')
322 linearizer.updateMetadata(setDate=True, setCalibId=True)
323 provenance = IsrProvenance(calibType='linearizer')
325 return pipeBase.Struct(
326 outputLinearizer=linearizer,
327 outputProvenance=provenance,
328 )
330 def debugFit(self, stepname, xVector, yVector, yModel, mask, ampName):
331 """Debug method for linearity fitting.
333 Parameters
334 ----------
335 stepname : `str`
336 A label to use to check if we care to debug at a given
337 line of code.
338 xVector : `numpy.array`
339 The values to use as the independent variable in the
340 linearity fit.
341 yVector : `numpy.array`
342 The values to use as the dependent variable in the
343 linearity fit.
344 yModel : `numpy.array`
345 The values to use as the linearized result.
346 mask : `numpy.array` [ `bool` ], optional
347 A mask to indicate which entries of ``xVector`` and
348 ``yVector`` to keep.
349 ampName : `str`
350 Amplifier name to lookup linearity correction values.
352 """
353 frame = getDebugFrame(self._display, stepname)
354 if frame:
355 import matplotlib.pyplot as plt
356 fig, axs = plt.subplots(2)
358 if mask is None:
359 mask = np.ones_like(xVector, dtype=bool)
361 fig.suptitle(f"{stepname} {ampName} {self.config.linearityType}")
362 if stepname == 'linearFit':
363 axs[0].set_xlabel("Input Abscissa (time or mondiode)")
364 axs[0].set_ylabel("Input Ordinate (flux)")
365 axs[1].set_xlabel("Linear Ordinate (linear flux)")
366 axs[1].set_ylabel("Flux Difference: (input - linear)")
367 elif stepname in ('polyFit', 'splineFit'):
368 axs[0].set_xlabel("Linear Abscissa (linear flux)")
369 axs[0].set_ylabel("Input Ordinate (flux)")
370 axs[1].set_xlabel("Linear Ordinate (linear flux)")
371 axs[1].set_ylabel("Flux Difference: (input - full model fit)")
372 elif stepname == 'solution':
373 axs[0].set_xlabel("Input Abscissa (time or mondiode)")
374 axs[0].set_ylabel("Linear Ordinate (linear flux)")
375 axs[1].set_xlabel("Model flux (linear flux)")
376 axs[1].set_ylabel("Flux Difference: (linear - model)")
378 axs[0].set_yscale('log')
379 axs[0].set_xscale('log')
380 axs[0].scatter(xVector, yVector)
381 axs[0].scatter(xVector[~mask], yVector[~mask], c='red', marker='x')
382 axs[1].set_xscale('log')
384 axs[1].scatter(yModel, yVector[mask] - yModel)
385 fig.show()
387 prompt = "Press Enter or c to continue [chpx]..."
388 while True:
389 ans = input(prompt).lower()
390 if ans in ("", " ", "c",):
391 break
392 elif ans in ("p", ):
393 import pdb
394 pdb.set_trace()
395 elif ans in ("h", ):
396 print("[h]elp [c]ontinue [p]db")
397 elif ans in ('x', ):
398 exit()
399 plt.close()
402class MeasureLinearityConfig(pexConfig.Config):
403 solver = pexConfig.ConfigurableField(
404 target=LinearitySolveTask,
405 doc="Task to convert PTC data to linearity solutions.",
406 )
409class MeasureLinearityTask(pipeBase.CmdLineTask):
410 """Stand alone Gen2 linearity measurement.
412 This class wraps the Gen3 linearity task to allow it to be run as
413 a Gen2 CmdLineTask.
414 """
415 ConfigClass = MeasureLinearityConfig
416 _DefaultName = "measureLinearity"
418 def __init__(self, **kwargs):
419 super().__init__(**kwargs)
420 self.makeSubtask("solver")
422 def runDataRef(self, dataRef):
423 """Run new linearity code for gen2.
425 Parameters
426 ----------
427 dataRef : `lsst.daf.persistence.ButlerDataRef`
428 Input dataref for the photon transfer curve data.
430 Returns
431 -------
432 results : `lsst.pipe.base.Struct`
433 The results struct containing:
435 ``outputLinearizer`` : `lsst.ip.isr.Linearizer`
436 Final linearizer calibration.
437 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
438 Provenance data for the new calibration.
439 """
440 ptc = dataRef.get('photonTransferCurveDataset')
441 camera = dataRef.get('camera')
442 inputDims = dataRef.dataId # This is the closest gen2 has.
443 linearityResults = self.solver.run(ptc, camera=camera, inputDims=inputDims)
445 inputDims['calibDate'] = linearityResults.outputLinearizer.getMetadata().get('CALIBDATE')
446 butler = dataRef.getButler()
447 butler.put(linearityResults.outputLinearizer, "linearizer", inputDims)
448 return linearityResults