Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.afw.image as afwImage 

25import lsst.afw.math as afwMath 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.pex.config as pexConfig 

29 

30from lsstDebug import getDebugFrame 

31from lsst.ip.isr import (Linearizer, IsrProvenance) 

32 

33from .utils import (funcPolynomial, irlsFit) 

34 

35 

36__all__ = ["LinearitySolveTask", "LinearitySolveConfig", "MeasureLinearityTask"] 

37 

38 

39class LinearitySolveConnections(pipeBase.PipelineTaskConnections, 

40 dimensions=("instrument", "detector")): 

41 inputPtc = cT.Input( 

42 name="inputPtc", 

43 doc="Input PTC dataset.", 

44 storageClass="StructuredDataDict", 

45 dimensions=("instrument", "detector"), 

46 multiple=False, 

47 ) 

48 camera = cT.Input( 

49 name="camera", 

50 doc="Camera Geometry definition.", 

51 storageClass="Camera", 

52 dimensions=("instrument", ), 

53 ) 

54 outputLinearizer = cT.Output( 

55 name="linearity", 

56 doc="Output linearity measurements.", 

57 storageClass="Linearizer", 

58 dimensions=("instrument", "detector"), 

59 isCalibration=True, 

60 ) 

61 

62 

63class LinearitySolveConfig(pipeBase.PipelineTaskConfig, 

64 pipelineConnections=LinearitySolveConnections): 

65 """Configuration for solving the linearity from PTC dataset. 

66 """ 

67 linearityType = pexConfig.ChoiceField( 

68 dtype=str, 

69 doc="Type of linearizer to construct.", 

70 default="Squared", 

71 allowed={ 

72 "LookupTable": "Create a lookup table solution.", 

73 "Polynomial": "Create an arbitrary polynomial solution.", 

74 "Squared": "Create a single order squared solution.", 

75 "Spline": "Create a spline based solution.", 

76 "None": "Create a dummy solution.", 

77 } 

78 ) 

79 polynomialOrder = pexConfig.Field( 

80 dtype=int, 

81 doc="Degree of polynomial to fit.", 

82 default=3, 

83 ) 

84 splineKnots = pexConfig.Field( 

85 dtype=int, 

86 doc="Number of spline knots to use in fit.", 

87 default=10, 

88 ) 

89 maxLookupTableAdu = pexConfig.Field( 

90 dtype=int, 

91 doc="Maximum DN value for a LookupTable linearizer.", 

92 default=2**18, 

93 ) 

94 maxLinearAdu = pexConfig.Field( 

95 dtype=float, 

96 doc="Maximum DN value to use to estimate linear term.", 

97 default=20000.0, 

98 ) 

99 minLinearAdu = pexConfig.Field( 

100 dtype=float, 

101 doc="Minimum DN value to use to estimate linear term.", 

102 default=2000.0, 

103 ) 

104 nSigmaClipLinear = pexConfig.Field( 

105 dtype=float, 

106 doc="Maximum deviation from linear solution for Poissonian noise.", 

107 default=5.0, 

108 ) 

109 

110 

111class LinearitySolveTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

112 """Fit the linearity from the PTC dataset. 

113 """ 

114 ConfigClass = LinearitySolveConfig 

115 _DefaultName = 'cpLinearitySolve' 

116 

117 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

118 """Ensure that the input and output dimensions are passed along. 

119 

120 Parameters 

121 ---------- 

122 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

123 Butler to operate on. 

124 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

125 Input data refs to load. 

126 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

127 Output data refs to persist. 

128 """ 

129 inputs = butlerQC.get(inputRefs) 

130 

131 # Use the dimensions to set calib/provenance information. 

132 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputPtc] 

133 

134 outputs = self.run(**inputs) 

135 butlerQC.put(outputs, outputRefs) 

136 

137 def run(self, inputPtc, camera, inputDims): 

138 """Fit non-linearity to PTC data, returning the correct Linearizer 

139 object. 

140 

141 Parameters 

142 ---------- 

143 inputPtc : `lsst.cp.pipe.PtcDataset` 

144 Pre-measured PTC dataset. 

145 camera : `lsst.afw.cameraGeom.Camera` 

146 Camera geometry. 

147 inputDims : `lsst.daf.butler.DataCoordinate` or `dict` 

148 DataIds to use to populate the output calibration. 

149 

150 Returns 

151 ------- 

152 results : `lsst.pipe.base.Struct` 

153 The results struct containing: 

154 

155 ``outputLinearizer`` : `lsst.ip.isr.Linearizer` 

156 Final linearizer calibration. 

157 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

158 Provenance data for the new calibration. 

159 

160 Notes 

161 ----- 

162 This task currently fits only polynomial-defined corrections, 

163 where the correction coefficients are defined such that: 

164 corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i) 

165 These `c_i` are defined in terms of the direct polynomial fit: 

166 meanVector ~ P(x=timeVector) = sum_j k_j x^j 

167 such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f., 

168 Eq. 37 of 2003.05978). The `config.polynomialOrder` or 

169 `config.splineKnots` define the maximum order of x^j to fit. 

170 As k_0 and k_1 are degenerate with bias level and gain, they 

171 are not included in the non-linearity correction. 

172 """ 

173 detector = camera[inputDims['detector']] 

174 if self.config.linearityType == 'LookupTable': 

175 table = np.zeros((len(detector), self.config.maxLookupTableAdu), dtype=np.float32) 

176 tableIndex = 0 

177 else: 

178 table = None 

179 tableIndex = None # This will fail if we increment it. 

180 

181 if self.config.linearityType == 'Spline': 

182 fitOrder = self.config.splineKnots 

183 else: 

184 fitOrder = self.config.polynomialOrder 

185 

186 # Initialize the linearizer. 

187 linearizer = Linearizer(detector=detector, table=table, log=self.log) 

188 

189 for i, amp in enumerate(detector): 

190 ampName = amp.getName() 

191 if (len(inputPtc.expIdMask[ampName]) == 0): 

192 self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.") 

193 mask = np.repeat(True, len(inputPtc.expIdMask[ampName])) 

194 else: 

195 mask = inputPtc.expIdMask[ampName] 

196 

197 inputAbscissa = np.array(inputPtc.rawExpTimes[ampName])[mask] 

198 inputOrdinate = np.array(inputPtc.rawMeans[ampName])[mask] 

199 

200 # Determine proxy-to-linear-flux transformation 

201 fluxMask = inputOrdinate < self.config.maxLinearAdu 

202 lowMask = inputOrdinate > self.config.minLinearAdu 

203 fluxMask = fluxMask & lowMask 

204 linearAbscissa = inputAbscissa[fluxMask] 

205 linearOrdinate = inputOrdinate[fluxMask] 

206 

207 linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 100.0], linearAbscissa, 

208 linearOrdinate, funcPolynomial) 

209 # Convert this proxy-to-flux fit into an expected linear flux 

210 linearOrdinate = linearFit[0] + linearFit[1] * inputAbscissa 

211 

212 # Exclude low end outliers 

213 threshold = self.config.nSigmaClipLinear * np.sqrt(linearOrdinate) 

214 fluxMask = np.abs(inputOrdinate - linearOrdinate) < threshold 

215 linearOrdinate = linearOrdinate[fluxMask] 

216 fitOrdinate = inputOrdinate[fluxMask] 

217 self.debugFit('linearFit', inputAbscissa, inputOrdinate, linearOrdinate, fluxMask, ampName) 

218 # Do fits 

219 if self.config.linearityType in ['Polynomial', 'Squared', 'LookupTable']: 

220 polyFit = np.zeros(fitOrder + 1) 

221 polyFit[1] = 1.0 

222 polyFit, polyFitErr, chiSq, weights = irlsFit(polyFit, linearOrdinate, 

223 fitOrdinate, funcPolynomial) 

224 

225 # Truncate the polynomial fit 

226 k1 = polyFit[1] 

227 linearityFit = [-coeff/(k1**order) for order, coeff in enumerate(polyFit)] 

228 significant = np.where(np.abs(linearityFit) > 1e-10, True, False) 

229 self.log.info(f"Significant polynomial fits: {significant}") 

230 

231 modelOrdinate = funcPolynomial(polyFit, linearAbscissa) 

232 self.debugFit('polyFit', linearAbscissa, fitOrdinate, modelOrdinate, None, ampName) 

233 

234 if self.config.linearityType == 'Squared': 

235 linearityFit = [linearityFit[2]] 

236 elif self.config.linearityType == 'LookupTable': 

237 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN 

238 tMax = (self.config.maxLookupTableAdu - polyFit[0])/polyFit[1] 

239 timeRange = np.linspace(0, tMax, self.config.maxLookupTableAdu) 

240 signalIdeal = polyFit[0] + polyFit[1]*timeRange 

241 signalUncorrected = funcPolynomial(polyFit, timeRange) 

242 lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has correction 

243 

244 linearizer.tableData[tableIndex, :] = lookupTableRow 

245 linearityFit = [tableIndex, 0] 

246 tableIndex += 1 

247 elif self.config.linearityType in ['Spline']: 

248 # See discussion in `lsst.ip.isr.linearize.py` before modifying. 

249 numPerBin, binEdges = np.histogram(linearOrdinate, bins=fitOrder) 

250 with np.errstate(invalid="ignore"): 

251 # Algorithm note: With the counts of points per 

252 # bin above, the next histogram calculates the 

253 # values to put in each bin by weighting each 

254 # point by the correction value. 

255 values = np.histogram(linearOrdinate, bins=fitOrder, 

256 weights=(inputOrdinate[fluxMask] - linearOrdinate))[0]/numPerBin 

257 

258 # After this is done, the binCenters are 

259 # calculated by weighting by the value we're 

260 # binning over. This ensures that widely 

261 # spaced/poorly sampled data aren't assigned to 

262 # the midpoint of the bin (as could be done using 

263 # the binEdges above), but to the weighted mean of 

264 # the inputs. Note that both histograms are 

265 # scaled by the count per bin to normalize what 

266 # the histogram returns (a sum of the points 

267 # inside) into an average. 

268 binCenters = np.histogram(linearOrdinate, bins=fitOrder, 

269 weights=linearOrdinate)[0]/numPerBin 

270 values = values[numPerBin > 0] 

271 binCenters = binCenters[numPerBin > 0] 

272 

273 self.debugFit('splineFit', binCenters, np.abs(values), values, None, ampName) 

274 interp = afwMath.makeInterpolate(binCenters.tolist(), values.tolist(), 

275 afwMath.stringToInterpStyle("AKIMA_SPLINE")) 

276 modelOrdinate = linearOrdinate + interp.interpolate(linearOrdinate) 

277 self.debugFit('splineFit', linearOrdinate, fitOrdinate, modelOrdinate, None, ampName) 

278 

279 # If we exclude a lot of points, we may end up with 

280 # less than fitOrder points. Pad out the low-flux end 

281 # to ensure equal lengths. 

282 if len(binCenters) != fitOrder: 

283 padN = fitOrder - len(binCenters) 

284 binCenters = np.pad(binCenters, (padN, 0), 'linear_ramp', 

285 end_values=(binCenters.min() - 1.0, )) 

286 # This stores the correction, which is zero at low values. 

287 values = np.pad(values, (padN, 0)) 

288 

289 # Pack the spline into a single array. 

290 linearityFit = np.concatenate((binCenters.tolist(), values.tolist())).tolist() 

291 polyFit = [0.0] 

292 polyFitErr = [0.0] 

293 chiSq = np.nan 

294 else: 

295 polyFit = [0.0] 

296 polyFitErr = [0.0] 

297 chiSq = np.nan 

298 linearityFit = [0.0] 

299 

300 linearizer.linearityType[ampName] = self.config.linearityType 

301 linearizer.linearityCoeffs[ampName] = np.array(linearityFit) 

302 linearizer.linearityBBox[ampName] = amp.getBBox() 

303 linearizer.fitParams[ampName] = np.array(polyFit) 

304 linearizer.fitParamsErr[ampName] = np.array(polyFitErr) 

305 linearizer.fitChiSq[ampName] = chiSq 

306 

307 image = afwImage.ImageF(len(inputOrdinate), 1) 

308 image.getArray()[:, :] = inputOrdinate 

309 linearizeFunction = linearizer.getLinearityTypeByName(linearizer.linearityType[ampName]) 

310 linearizeFunction()(image, 

311 **{'coeffs': linearizer.linearityCoeffs[ampName], 

312 'table': linearizer.tableData, 

313 'log': linearizer.log}) 

314 linearizeModel = image.getArray()[0, :] 

315 

316 self.debugFit('solution', inputOrdinate[fluxMask], linearOrdinate, 

317 linearizeModel[fluxMask], None, ampName) 

318 

319 linearizer.hasLinearity = True 

320 linearizer.validate() 

321 linearizer.updateMetadata(camera=camera, detector=detector, filterName='NONE') 

322 linearizer.updateMetadata(setDate=True, setCalibId=True) 

323 provenance = IsrProvenance(calibType='linearizer') 

324 

325 return pipeBase.Struct( 

326 outputLinearizer=linearizer, 

327 outputProvenance=provenance, 

328 ) 

329 

330 def debugFit(self, stepname, xVector, yVector, yModel, mask, ampName): 

331 """Debug method for linearity fitting. 

332 

333 Parameters 

334 ---------- 

335 stepname : `str` 

336 A label to use to check if we care to debug at a given 

337 line of code. 

338 xVector : `numpy.array` 

339 The values to use as the independent variable in the 

340 linearity fit. 

341 yVector : `numpy.array` 

342 The values to use as the dependent variable in the 

343 linearity fit. 

344 yModel : `numpy.array` 

345 The values to use as the linearized result. 

346 mask : `numpy.array` [ `bool` ], optional 

347 A mask to indicate which entries of ``xVector`` and 

348 ``yVector`` to keep. 

349 ampName : `str` 

350 Amplifier name to lookup linearity correction values. 

351 

352 """ 

353 frame = getDebugFrame(self._display, stepname) 

354 if frame: 

355 import matplotlib.pyplot as plt 

356 fig, axs = plt.subplots(2) 

357 

358 if mask is None: 

359 mask = np.ones_like(xVector, dtype=bool) 

360 

361 fig.suptitle(f"{stepname} {ampName} {self.config.linearityType}") 

362 if stepname == 'linearFit': 

363 axs[0].set_xlabel("Input Abscissa (time or mondiode)") 

364 axs[0].set_ylabel("Input Ordinate (flux)") 

365 axs[1].set_xlabel("Linear Ordinate (linear flux)") 

366 axs[1].set_ylabel("Flux Difference: (input - linear)") 

367 elif stepname in ('polyFit', 'splineFit'): 

368 axs[0].set_xlabel("Linear Abscissa (linear flux)") 

369 axs[0].set_ylabel("Input Ordinate (flux)") 

370 axs[1].set_xlabel("Linear Ordinate (linear flux)") 

371 axs[1].set_ylabel("Flux Difference: (input - full model fit)") 

372 elif stepname == 'solution': 

373 axs[0].set_xlabel("Input Abscissa (time or mondiode)") 

374 axs[0].set_ylabel("Linear Ordinate (linear flux)") 

375 axs[1].set_xlabel("Model flux (linear flux)") 

376 axs[1].set_ylabel("Flux Difference: (linear - model)") 

377 

378 axs[0].set_yscale('log') 

379 axs[0].set_xscale('log') 

380 axs[0].scatter(xVector, yVector) 

381 axs[0].scatter(xVector[~mask], yVector[~mask], c='red', marker='x') 

382 axs[1].set_xscale('log') 

383 

384 axs[1].scatter(yModel, yVector[mask] - yModel) 

385 fig.show() 

386 

387 prompt = "Press Enter or c to continue [chpx]..." 

388 while True: 

389 ans = input(prompt).lower() 

390 if ans in ("", " ", "c",): 

391 break 

392 elif ans in ("p", ): 

393 import pdb 

394 pdb.set_trace() 

395 elif ans in ("h", ): 

396 print("[h]elp [c]ontinue [p]db") 

397 elif ans in ('x', ): 

398 exit() 

399 plt.close() 

400 

401 

402class MeasureLinearityConfig(pexConfig.Config): 

403 solver = pexConfig.ConfigurableField( 

404 target=LinearitySolveTask, 

405 doc="Task to convert PTC data to linearity solutions.", 

406 ) 

407 

408 

409class MeasureLinearityTask(pipeBase.CmdLineTask): 

410 """Stand alone Gen2 linearity measurement. 

411 

412 This class wraps the Gen3 linearity task to allow it to be run as 

413 a Gen2 CmdLineTask. 

414 """ 

415 ConfigClass = MeasureLinearityConfig 

416 _DefaultName = "measureLinearity" 

417 

418 def __init__(self, **kwargs): 

419 super().__init__(**kwargs) 

420 self.makeSubtask("solver") 

421 

422 def runDataRef(self, dataRef): 

423 """Run new linearity code for gen2. 

424 

425 Parameters 

426 ---------- 

427 dataRef : `lsst.daf.persistence.ButlerDataRef` 

428 Input dataref for the photon transfer curve data. 

429 

430 Returns 

431 ------- 

432 results : `lsst.pipe.base.Struct` 

433 The results struct containing: 

434 

435 ``outputLinearizer`` : `lsst.ip.isr.Linearizer` 

436 Final linearizer calibration. 

437 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

438 Provenance data for the new calibration. 

439 """ 

440 ptc = dataRef.get('photonTransferCurveDataset') 

441 camera = dataRef.get('camera') 

442 inputDims = dataRef.dataId # This is the closest gen2 has. 

443 linearityResults = self.solver.run(ptc, camera=camera, inputDims=inputDims) 

444 

445 inputDims['calibDate'] = linearityResults.outputLinearizer.getMetadata().get('CALIBDATE') 

446 butler = dataRef.getButler() 

447 butler.put(linearityResults.outputLinearizer, "linearizer", inputDims) 

448 return linearityResults