Coverage for python/lsst/ip/isr/brighterFatterKernel.py: 7%
212 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-18 09:34 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Brighter Fatter Kernel calibration definition."""
25__all__ = ['BrighterFatterKernel']
28import numpy as np
29from astropy.table import Table
30import lsst.afw.math as afwMath
31from . import IsrCalib
34class BrighterFatterKernel(IsrCalib):
35 """Calibration of brighter-fatter kernels for an instrument.
37 ampKernels are the kernels for each amplifier in a detector, as
38 generated by having level == 'AMP'
40 detectorKernel is the kernel generated for a detector as a
41 whole, as generated by having level == 'DETECTOR'
43 makeDetectorKernelFromAmpwiseKernels is a method to generate the
44 kernel for a detector, constructed by averaging together the
45 ampwise kernels in the detector. The existing application code is
46 only defined for kernels with level == 'DETECTOR', so this method
47 is used if the supplied kernel was built with level == 'AMP'.
49 Parameters
50 ----------
51 level : `str`
52 Level the kernels will be generated for.
53 log : `logging.Logger`, optional
54 Log to write messages to.
55 **kwargs :
56 Parameters to pass to parent constructor.
58 Notes
59 -----
60 TODO: DM-35260
61 Document what is stored in the BFK calibration.
63 Version 1.1 adds the `expIdMask` property, and substitutes
64 `means` and `variances` for `rawMeans` and `rawVariances`
65 from the PTC dataset.
66 """
67 _OBSTYPE = 'bfk'
68 _SCHEMA = 'Brighter-fatter kernel'
69 _VERSION = 1.1
71 def __init__(self, camera=None, level=None, **kwargs):
72 self.level = level
74 # Things inherited from the PTC
75 self.expIdMask = dict()
76 self.rawMeans = dict()
77 self.rawVariances = dict()
78 self.rawXcorrs = dict()
79 self.badAmps = list()
80 self.shape = (17, 17)
81 self.gain = dict()
82 self.noise = dict()
84 # Things calculated from the PTC
85 self.meanXcorrs = dict()
86 self.valid = dict()
88 # Things that are used downstream
89 self.ampKernels = dict()
90 self.detKernels = dict()
92 super().__init__(**kwargs)
94 if camera:
95 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None))
97 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs',
98 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid',
99 'ampKernels', 'detKernels'])
101 def updateMetadata(self, setDate=False, **kwargs):
102 """Update calibration metadata.
104 This calls the base class's method after ensuring the required
105 calibration keywords will be saved.
107 Parameters
108 ----------
109 setDate : `bool`, optional
110 Update the CALIBDATE fields in the metadata to the current
111 time. Defaults to False.
112 kwargs :
113 Other keyword parameters to set in the metadata.
114 """
115 kwargs['LEVEL'] = self.level
116 kwargs['KERNEL_DX'] = self.shape[0]
117 kwargs['KERNEL_DY'] = self.shape[1]
119 super().updateMetadata(setDate=setDate, **kwargs)
121 def initFromCamera(self, camera, detectorId=None):
122 """Initialize kernel structure from camera.
124 Parameters
125 ----------
126 camera : `lsst.afw.cameraGeom.Camera`
127 Camera to use to define geometry.
128 detectorId : `int`, optional
129 Index of the detector to generate.
131 Returns
132 -------
133 calib : `lsst.ip.isr.BrighterFatterKernel`
134 The initialized calibration.
136 Raises
137 ------
138 RuntimeError :
139 Raised if no detectorId is supplied for a calibration with
140 level='AMP'.
141 """
142 self._instrument = camera.getName()
144 if detectorId is not None:
145 detector = camera[detectorId]
146 self._detectorId = detectorId
147 self._detectorName = detector.getName()
148 self._detectorSerial = detector.getSerial()
150 if self.level == 'AMP':
151 if detectorId is None:
152 raise RuntimeError("A detectorId must be supplied if level='AMP'.")
154 self.badAmps = []
156 for amp in detector:
157 ampName = amp.getName()
158 self.expIdMask[ampName] = []
159 self.rawMeans[ampName] = []
160 self.rawVariances[ampName] = []
161 self.rawXcorrs[ampName] = []
162 self.gain[ampName] = amp.getGain()
163 self.noise[ampName] = amp.getReadNoise()
164 self.meanXcorrs[ampName] = []
165 self.ampKernels[ampName] = []
166 self.valid[ampName] = []
167 elif self.level == 'DETECTOR':
168 if detectorId is None:
169 for det in camera:
170 detName = det.getName()
171 self.detKernels[detName] = []
172 else:
173 self.detKernels[self._detectorName] = []
175 return self
177 def getLengths(self):
178 """Return the set of lengths needed for reshaping components.
180 Returns
181 -------
182 kernelLength : `int`
183 Product of the elements of self.shape.
184 smallLength : `int`
185 Size of an untiled covariance.
186 nObs : `int`
187 Number of observation pairs used in the kernel.
188 """
189 kernelLength = self.shape[0] * self.shape[1]
190 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4)
191 if self.level == 'AMP':
192 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans])
193 if len(nObservations) != 1:
194 raise RuntimeError("Inconsistent number of observations found.")
195 nObs = nObservations.pop()
196 else:
197 nObs = 0
199 return (kernelLength, smallLength, nObs)
201 @classmethod
202 def fromDict(cls, dictionary):
203 """Construct a calibration from a dictionary of properties.
205 Parameters
206 ----------
207 dictionary : `dict`
208 Dictionary of properties.
210 Returns
211 -------
212 calib : `lsst.ip.isr.BrighterFatterKernel
213 Constructed calibration.
215 Raises
216 ------
217 RuntimeError :
218 Raised if the supplied dictionary is for a different
219 calibration.
220 Raised if the version of the supplied dictionary is 1.0.
221 """
222 calib = cls()
224 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']):
225 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, "
226 f"found {found}")
227 calib.setMetadata(dictionary['metadata'])
228 calib.calibInfoFromDict(dictionary)
230 calib.level = dictionary['metadata'].get('LEVEL', 'AMP')
231 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0),
232 dictionary['metadata'].get('KERNEL_DY', 0))
234 calibVersion = dictionary['metadata']['bfk_VERSION']
235 if calibVersion == 1.0:
236 calib.log.warning("Old Version of brighter-fatter kernel found. Current version: "
237 f"{calib._VERSION}. The new attribute 'expIdMask' will be "
238 "populated with 'True' values, and the new attributes 'rawMeans'"
239 "and 'rawVariances' will be populated with the masked 'means'."
240 "and 'variances' values."
241 )
242 # use 'means', because 'expIdMask' does not exist.
243 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in
244 dictionary['means']}
245 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']}
246 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in
247 dictionary['variances']}
248 elif calibVersion == 1.1:
249 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']}
250 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']}
251 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in
252 dictionary['rawVariances']}
253 else:
254 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
256 # Lengths for reshape:
257 _, smallLength, nObs = calib.getLengths()
258 smallShapeSide = int(np.sqrt(smallLength))
260 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs,
261 smallShapeSide,
262 smallShapeSide))
263 for amp in dictionary['rawXcorrs']}
265 calib.gain = dictionary['gain']
266 calib.noise = dictionary['noise']
268 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape)
269 for amp in dictionary['rawXcorrs']}
270 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape)
271 for amp in dictionary['ampKernels']}
272 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()}
273 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False]
275 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape)
276 for det in dictionary['detKernels']}
278 calib.updateMetadata()
279 return calib
281 def toDict(self):
282 """Return a dictionary containing the calibration properties.
284 The dictionary should be able to be round-tripped through
285 `fromDict`.
287 Returns
288 -------
289 dictionary : `dict`
290 Dictionary of properties.
291 """
292 self.updateMetadata()
294 outDict = {}
295 metadata = self.getMetadata()
296 outDict['metadata'] = metadata
298 # Lengths for ravel:
299 kernelLength, smallLength, nObs = self.getLengths()
301 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask}
302 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans}
303 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in
304 self.rawVariances}
305 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()
306 for amp in self.rawXcorrs}
307 outDict['badAmps'] = self.badAmps
308 outDict['gain'] = self.gain
309 outDict['noise'] = self.noise
311 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist()
312 for amp in self.meanXcorrs}
313 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist()
314 for amp in self.ampKernels}
315 outDict['valid'] = self.valid
317 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist()
318 for det in self.detKernels}
319 return outDict
321 @classmethod
322 def fromTable(cls, tableList):
323 """Construct calibration from a list of tables.
325 This method uses the `fromDict` method to create the
326 calibration, after constructing an appropriate dictionary from
327 the input tables.
329 Parameters
330 ----------
331 tableList : `list` [`astropy.table.Table`]
332 List of tables to use to construct the brighter-fatter
333 calibration.
335 Returns
336 -------
337 calib : `lsst.ip.isr.BrighterFatterKernel`
338 The calibration defined in the tables.
339 """
340 ampTable = tableList[0]
342 metadata = ampTable.meta
343 inDict = dict()
344 inDict['metadata'] = metadata
346 amps = ampTable['AMPLIFIER']
348 # Determine version for expected values. The ``fromDict``
349 # method can unpack either, but the appropriate fields need to
350 # be supplied.
351 calibVersion = metadata['bfk_VERSION']
353 if calibVersion == 1.0:
354 # We expect to find ``means`` and ``variances`` for this
355 # case, and will construct an ``expIdMask`` from these
356 # parameters in the ``fromDict`` method.
357 rawMeanList = ampTable['MEANS']
358 rawVarianceList = ampTable['VARIANCES']
360 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
361 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
362 elif calibVersion == 1.1:
363 # This will have ``rawMeans`` and ``rawVariances``, which
364 # are filtered via the ``expIdMask`` fields.
365 expIdMaskList = ampTable['EXP_ID_MASK']
366 rawMeanList = ampTable['RAW_MEANS']
367 rawVarianceList = ampTable['RAW_VARIANCES']
369 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)}
370 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
371 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
372 else:
373 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
375 rawXcorrs = ampTable['RAW_XCORRS']
376 gainList = ampTable['GAIN']
377 noiseList = ampTable['NOISE']
379 meanXcorrs = ampTable['MEAN_XCORRS']
380 ampKernels = ampTable['KERNEL']
381 validList = ampTable['VALID']
383 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)}
384 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)}
385 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)}
386 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)}
387 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)}
388 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)}
390 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False]
392 if len(tableList) > 1:
393 detTable = tableList[1]
394 inDict['detKernels'] = {det: kernel for det, kernel
395 in zip(detTable['DETECTOR'], detTable['KERNEL'])}
396 else:
397 inDict['detKernels'] = {}
399 return cls.fromDict(inDict)
401 def toTable(self):
402 """Construct a list of tables containing the information in this
403 calibration.
405 The list of tables should create an identical calibration
406 after being passed to this class's fromTable method.
408 Returns
409 -------
410 tableList : `list` [`lsst.afw.table.Table`]
411 List of tables containing the crosstalk calibration
412 information.
414 """
415 tableList = []
416 self.updateMetadata()
418 # Lengths
419 kernelLength, smallLength, nObs = self.getLengths()
421 ampList = []
422 expIdMaskList = []
423 rawMeanList = []
424 rawVarianceList = []
425 rawXcorrs = []
426 gainList = []
427 noiseList = []
429 meanXcorrsList = []
430 kernelList = []
431 validList = []
433 if self.level == 'AMP':
434 for amp in self.rawMeans.keys():
435 ampList.append(amp)
436 expIdMaskList.append(self.expIdMask[amp])
437 rawMeanList.append(self.rawMeans[amp])
438 rawVarianceList.append(self.rawVariances[amp])
439 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist())
440 gainList.append(self.gain[amp])
441 noiseList.append(self.noise[amp])
443 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist())
444 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist())
445 validList.append(int(self.valid[amp] and not (amp in self.badAmps)))
447 ampTable = Table({'AMPLIFIER': ampList,
448 'EXP_ID_MASK': expIdMaskList,
449 'RAW_MEANS': rawMeanList,
450 'RAW_VARIANCES': rawVarianceList,
451 'RAW_XCORRS': rawXcorrs,
452 'GAIN': gainList,
453 'NOISE': noiseList,
454 'MEAN_XCORRS': meanXcorrsList,
455 'KERNEL': kernelList,
456 'VALID': validList,
457 })
459 ampTable.meta = self.getMetadata().toDict()
460 tableList.append(ampTable)
462 if len(self.detKernels):
463 detList = []
464 kernelList = []
465 for det in self.detKernels.keys():
466 detList.append(det)
467 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist())
469 detTable = Table({'DETECTOR': detList,
470 'KERNEL': kernelList})
471 detTable.meta = self.getMetadata().toDict()
472 tableList.append(detTable)
474 return tableList
476 # Implementation methods
477 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]):
478 """Average the amplifier level kernels to create a detector level
479 kernel.
480 """
481 inKernels = np.array([self.ampKernels[amp] for amp in
482 self.ampKernels if amp not in ampsToExclude])
483 averagingList = np.transpose(inKernels)
484 avgKernel = np.zeros_like(inKernels[0])
485 sctrl = afwMath.StatisticsControl()
486 sctrl.setNumSigmaClip(5.0)
487 for i in range(np.shape(avgKernel)[0]):
488 for j in range(np.shape(avgKernel)[1]):
489 avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j],
490 afwMath.MEANCLIP, sctrl).getValue()
492 self.detKernels[detectorName] = avgKernel
494 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName):
495 self.detKernel[detectorName] = self.ampKernel[ampName]