Coverage for python/lsst/ip/isr/brighterFatterKernel.py: 7%
212 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-14 10:14 +0000
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-14 10:14 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Brighter Fatter Kernel calibration definition."""
25__all__ = ['BrighterFatterKernel']
28import numpy as np
29from astropy.table import Table
30import lsst.afw.math as afwMath
31from . import IsrCalib
34class BrighterFatterKernel(IsrCalib):
35 """Calibration of brighter-fatter kernels for an instrument.
37 ampKernels are the kernels for each amplifier in a detector, as
38 generated by having ``level == 'AMP'``.
40 detectorKernel is the kernel generated for a detector as a
41 whole, as generated by having ``level == 'DETECTOR'``.
43 makeDetectorKernelFromAmpwiseKernels is a method to generate the
44 kernel for a detector, constructed by averaging together the
45 ampwise kernels in the detector. The existing application code is
46 only defined for kernels with ``level == 'DETECTOR'``, so this method
47 is used if the supplied kernel was built with ``level == 'AMP'``.
49 Parameters
50 ----------
51 camera : `lsst.afw.cameraGeom.Camera`
52 Camera describing detector geometry.
53 level : `str`
54 Level the kernels will be generated for.
55 log : `logging.Logger`, optional
56 Log to write messages to.
57 **kwargs :
58 Parameters to pass to parent constructor.
60 Notes
61 -----
62 Version 1.1 adds the `expIdMask` property, and substitutes
63 `means` and `variances` for `rawMeans` and `rawVariances`
64 from the PTC dataset.
66 expIdMask : `dict`, [`str`,`numpy.ndarray`]
67 Dictionary keyed by amp names containing the mask produced after
68 outlier rejection.
69 rawMeans : `dict`, [`str`, `numpy.ndarray`]
70 Dictionary keyed by amp names containing the unmasked average of the
71 means of the exposures in each flat pair.
72 rawVariances : `dict`, [`str`, `numpy.ndarray`]
73 Dictionary keyed by amp names containing the variance of the
74 difference image of the exposures in each flat pair.
75 Corresponds to rawVars of PTC.
76 rawXcorrs : `dict`, [`str`, `numpy.ndarray`]
77 Dictionary keyed by amp names containing an array of measured
78 covariances per mean flux.
79 Corresponds to covariances of PTC.
80 badAmps : `list`
81 List of bad amplifiers names.
82 shape : `tuple`
83 Tuple of the shape of the BFK kernels.
84 gain : `dict`, [`str`,`float`]
85 Dictionary keyed by amp names containing the fitted gains.
86 noise : `dict`, [`str`,`float`]
87 Dictionary keyed by amp names containing the fitted noise.
88 meanXcorrs : `dict`, [`str`,`numpy.ndarray`]
89 Dictionary keyed by amp names containing the averaged
90 cross-correlations.
91 valid : `dict`, [`str`,`bool`]
92 Dictionary keyed by amp names containing validity of data.
93 ampKernels : `dict`, [`str`, `numpy.ndarray`]
94 Dictionary keyed by amp names containing the BF kernels.
95 detKernels : `dict`
96 Dictionary keyed by detector names containing the BF kernels.
97 """
98 _OBSTYPE = 'bfk'
99 _SCHEMA = 'Brighter-fatter kernel'
100 _VERSION = 1.1
102 def __init__(self, camera=None, level=None, **kwargs):
103 self.level = level
105 # Things inherited from the PTC
106 self.expIdMask = dict()
107 self.rawMeans = dict()
108 self.rawVariances = dict()
109 self.rawXcorrs = dict()
110 self.badAmps = list()
111 self.shape = (17, 17)
112 self.gain = dict()
113 self.noise = dict()
115 # Things calculated from the PTC
116 self.meanXcorrs = dict()
117 self.valid = dict()
119 # Things that are used downstream
120 self.ampKernels = dict()
121 self.detKernels = dict()
123 super().__init__(**kwargs)
125 if camera:
126 self.initFromCamera(camera, detectorId=kwargs.get('detectorId', None))
128 self.requiredAttributes.update(['level', 'expIdMask', 'rawMeans', 'rawVariances', 'rawXcorrs',
129 'badAmps', 'gain', 'noise', 'meanXcorrs', 'valid',
130 'ampKernels', 'detKernels'])
132 def updateMetadata(self, setDate=False, **kwargs):
133 """Update calibration metadata.
135 This calls the base class's method after ensuring the required
136 calibration keywords will be saved.
138 Parameters
139 ----------
140 setDate : `bool`, optional
141 Update the CALIBDATE fields in the metadata to the current
142 time. Defaults to False.
143 kwargs :
144 Other keyword parameters to set in the metadata.
145 """
146 kwargs['LEVEL'] = self.level
147 kwargs['KERNEL_DX'] = self.shape[0]
148 kwargs['KERNEL_DY'] = self.shape[1]
150 super().updateMetadata(setDate=setDate, **kwargs)
152 def initFromCamera(self, camera, detectorId=None):
153 """Initialize kernel structure from camera.
155 Parameters
156 ----------
157 camera : `lsst.afw.cameraGeom.Camera`
158 Camera to use to define geometry.
159 detectorId : `int`, optional
160 Index of the detector to generate.
162 Returns
163 -------
164 calib : `lsst.ip.isr.BrighterFatterKernel`
165 The initialized calibration.
167 Raises
168 ------
169 RuntimeError
170 Raised if no detectorId is supplied for a calibration with
171 ``level='AMP'``.
172 """
173 self._instrument = camera.getName()
175 if detectorId is not None:
176 detector = camera[detectorId]
177 self._detectorId = detectorId
178 self._detectorName = detector.getName()
179 self._detectorSerial = detector.getSerial()
181 if self.level == 'AMP':
182 if detectorId is None:
183 raise RuntimeError("A detectorId must be supplied if level='AMP'.")
185 self.badAmps = []
187 for amp in detector:
188 ampName = amp.getName()
189 self.expIdMask[ampName] = []
190 self.rawMeans[ampName] = []
191 self.rawVariances[ampName] = []
192 self.rawXcorrs[ampName] = []
193 self.gain[ampName] = amp.getGain()
194 self.noise[ampName] = amp.getReadNoise()
195 self.meanXcorrs[ampName] = []
196 self.ampKernels[ampName] = []
197 self.valid[ampName] = []
198 elif self.level == 'DETECTOR':
199 if detectorId is None:
200 for det in camera:
201 detName = det.getName()
202 self.detKernels[detName] = []
203 else:
204 self.detKernels[self._detectorName] = []
206 return self
208 def getLengths(self):
209 """Return the set of lengths needed for reshaping components.
211 Returns
212 -------
213 kernelLength : `int`
214 Product of the elements of self.shape.
215 smallLength : `int`
216 Size of an untiled covariance.
217 nObs : `int`
218 Number of observation pairs used in the kernel.
219 """
220 kernelLength = self.shape[0] * self.shape[1]
221 smallLength = int((self.shape[0] - 1)*(self.shape[1] - 1)/4)
222 if self.level == 'AMP':
223 nObservations = set([len(self.rawMeans[amp]) for amp in self.rawMeans])
224 if len(nObservations) != 1:
225 raise RuntimeError("Inconsistent number of observations found.")
226 nObs = nObservations.pop()
227 else:
228 nObs = 0
230 return (kernelLength, smallLength, nObs)
232 @classmethod
233 def fromDict(cls, dictionary):
234 """Construct a calibration from a dictionary of properties.
236 Parameters
237 ----------
238 dictionary : `dict`
239 Dictionary of properties.
241 Returns
242 -------
243 calib : `lsst.ip.isr.BrighterFatterKernel`
244 Constructed calibration.
246 Raises
247 ------
248 RuntimeError
249 Raised if the supplied dictionary is for a different
250 calibration.
251 Raised if the version of the supplied dictionary is 1.0.
252 """
253 calib = cls()
255 if calib._OBSTYPE != (found := dictionary['metadata']['OBSTYPE']):
256 raise RuntimeError(f"Incorrect brighter-fatter kernel supplied. Expected {calib._OBSTYPE}, "
257 f"found {found}")
258 calib.setMetadata(dictionary['metadata'])
259 calib.calibInfoFromDict(dictionary)
261 calib.level = dictionary['metadata'].get('LEVEL', 'AMP')
262 calib.shape = (dictionary['metadata'].get('KERNEL_DX', 0),
263 dictionary['metadata'].get('KERNEL_DY', 0))
265 calibVersion = dictionary['metadata']['bfk_VERSION']
266 if calibVersion == 1.0:
267 calib.log.warning("Old Version of brighter-fatter kernel found. Current version: "
268 f"{calib._VERSION}. The new attribute 'expIdMask' will be "
269 "populated with 'True' values, and the new attributes 'rawMeans'"
270 "and 'rawVariances' will be populated with the masked 'means'."
271 "and 'variances' values."
272 )
273 # use 'means', because 'expIdMask' does not exist.
274 calib.expIdMask = {amp: np.repeat(True, len(dictionary['means'][amp])) for amp in
275 dictionary['means']}
276 calib.rawMeans = {amp: np.array(dictionary['means'][amp]) for amp in dictionary['means']}
277 calib.rawVariances = {amp: np.array(dictionary['variances'][amp]) for amp in
278 dictionary['variances']}
279 elif calibVersion == 1.1:
280 calib.expIdMask = {amp: np.array(dictionary['expIdMask'][amp]) for amp in dictionary['expIdMask']}
281 calib.rawMeans = {amp: np.array(dictionary['rawMeans'][amp]) for amp in dictionary['rawMeans']}
282 calib.rawVariances = {amp: np.array(dictionary['rawVariances'][amp]) for amp in
283 dictionary['rawVariances']}
284 else:
285 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
287 # Lengths for reshape:
288 _, smallLength, nObs = calib.getLengths()
289 smallShapeSide = int(np.sqrt(smallLength))
291 calib.rawXcorrs = {amp: np.array(dictionary['rawXcorrs'][amp]).reshape((nObs,
292 smallShapeSide,
293 smallShapeSide))
294 for amp in dictionary['rawXcorrs']}
296 calib.gain = dictionary['gain']
297 calib.noise = dictionary['noise']
299 calib.meanXcorrs = {amp: np.array(dictionary['meanXcorrs'][amp]).reshape(calib.shape)
300 for amp in dictionary['rawXcorrs']}
301 calib.ampKernels = {amp: np.array(dictionary['ampKernels'][amp]).reshape(calib.shape)
302 for amp in dictionary['ampKernels']}
303 calib.valid = {amp: bool(value) for amp, value in dictionary['valid'].items()}
304 calib.badAmps = [amp for amp, valid in dictionary['valid'].items() if valid is False]
306 calib.detKernels = {det: np.array(dictionary['detKernels'][det]).reshape(calib.shape)
307 for det in dictionary['detKernels']}
309 calib.updateMetadata()
310 return calib
312 def toDict(self):
313 """Return a dictionary containing the calibration properties.
315 The dictionary should be able to be round-tripped through
316 `fromDict`.
318 Returns
319 -------
320 dictionary : `dict`
321 Dictionary of properties.
322 """
323 self.updateMetadata()
325 outDict = {}
326 metadata = self.getMetadata()
327 outDict['metadata'] = metadata
329 # Lengths for ravel:
330 kernelLength, smallLength, nObs = self.getLengths()
332 outDict['expIdMask'] = {amp: np.array(self.expIdMask[amp]).tolist() for amp in self.expIdMask}
333 outDict['rawMeans'] = {amp: np.array(self.rawMeans[amp]).tolist() for amp in self.rawMeans}
334 outDict['rawVariances'] = {amp: np.array(self.rawVariances[amp]).tolist() for amp in
335 self.rawVariances}
336 outDict['rawXcorrs'] = {amp: np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist()
337 for amp in self.rawXcorrs}
338 outDict['badAmps'] = self.badAmps
339 outDict['gain'] = self.gain
340 outDict['noise'] = self.noise
342 outDict['meanXcorrs'] = {amp: self.meanXcorrs[amp].reshape(kernelLength).tolist()
343 for amp in self.meanXcorrs}
344 outDict['ampKernels'] = {amp: self.ampKernels[amp].reshape(kernelLength).tolist()
345 for amp in self.ampKernels}
346 outDict['valid'] = self.valid
348 outDict['detKernels'] = {det: self.detKernels[det].reshape(kernelLength).tolist()
349 for det in self.detKernels}
350 return outDict
352 @classmethod
353 def fromTable(cls, tableList):
354 """Construct calibration from a list of tables.
356 This method uses the `fromDict` method to create the
357 calibration, after constructing an appropriate dictionary from
358 the input tables.
360 Parameters
361 ----------
362 tableList : `list` [`astropy.table.Table`]
363 List of tables to use to construct the brighter-fatter
364 calibration.
366 Returns
367 -------
368 calib : `lsst.ip.isr.BrighterFatterKernel`
369 The calibration defined in the tables.
370 """
371 ampTable = tableList[0]
373 metadata = ampTable.meta
374 inDict = dict()
375 inDict['metadata'] = metadata
377 amps = ampTable['AMPLIFIER']
379 # Determine version for expected values. The ``fromDict``
380 # method can unpack either, but the appropriate fields need to
381 # be supplied.
382 calibVersion = metadata['bfk_VERSION']
384 if calibVersion == 1.0:
385 # We expect to find ``means`` and ``variances`` for this
386 # case, and will construct an ``expIdMask`` from these
387 # parameters in the ``fromDict`` method.
388 rawMeanList = ampTable['MEANS']
389 rawVarianceList = ampTable['VARIANCES']
391 inDict['means'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
392 inDict['variances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
393 elif calibVersion == 1.1:
394 # This will have ``rawMeans`` and ``rawVariances``, which
395 # are filtered via the ``expIdMask`` fields.
396 expIdMaskList = ampTable['EXP_ID_MASK']
397 rawMeanList = ampTable['RAW_MEANS']
398 rawVarianceList = ampTable['RAW_VARIANCES']
400 inDict['expIdMask'] = {amp: mask for amp, mask in zip(amps, expIdMaskList)}
401 inDict['rawMeans'] = {amp: mean for amp, mean in zip(amps, rawMeanList)}
402 inDict['rawVariances'] = {amp: var for amp, var in zip(amps, rawVarianceList)}
403 else:
404 raise RuntimeError(f"Unknown version for brighter-fatter kernel: {calibVersion}")
406 rawXcorrs = ampTable['RAW_XCORRS']
407 gainList = ampTable['GAIN']
408 noiseList = ampTable['NOISE']
410 meanXcorrs = ampTable['MEAN_XCORRS']
411 ampKernels = ampTable['KERNEL']
412 validList = ampTable['VALID']
414 inDict['rawXcorrs'] = {amp: kernel for amp, kernel in zip(amps, rawXcorrs)}
415 inDict['gain'] = {amp: gain for amp, gain in zip(amps, gainList)}
416 inDict['noise'] = {amp: noise for amp, noise in zip(amps, noiseList)}
417 inDict['meanXcorrs'] = {amp: kernel for amp, kernel in zip(amps, meanXcorrs)}
418 inDict['ampKernels'] = {amp: kernel for amp, kernel in zip(amps, ampKernels)}
419 inDict['valid'] = {amp: bool(valid) for amp, valid in zip(amps, validList)}
421 inDict['badAmps'] = [amp for amp, valid in inDict['valid'].items() if valid is False]
423 if len(tableList) > 1:
424 detTable = tableList[1]
425 inDict['detKernels'] = {det: kernel for det, kernel
426 in zip(detTable['DETECTOR'], detTable['KERNEL'])}
427 else:
428 inDict['detKernels'] = {}
430 return cls.fromDict(inDict)
432 def toTable(self):
433 """Construct a list of tables containing the information in this
434 calibration.
436 The list of tables should create an identical calibration
437 after being passed to this class's fromTable method.
439 Returns
440 -------
441 tableList : `list` [`lsst.afw.table.Table`]
442 List of tables containing the crosstalk calibration
443 information.
445 """
446 tableList = []
447 self.updateMetadata()
449 # Lengths
450 kernelLength, smallLength, nObs = self.getLengths()
452 ampList = []
453 expIdMaskList = []
454 rawMeanList = []
455 rawVarianceList = []
456 rawXcorrs = []
457 gainList = []
458 noiseList = []
460 meanXcorrsList = []
461 kernelList = []
462 validList = []
464 if self.level == 'AMP':
465 for amp in self.rawMeans.keys():
466 ampList.append(amp)
467 expIdMaskList.append(self.expIdMask[amp])
468 rawMeanList.append(self.rawMeans[amp])
469 rawVarianceList.append(self.rawVariances[amp])
470 rawXcorrs.append(np.array(self.rawXcorrs[amp]).reshape(nObs*smallLength).tolist())
471 gainList.append(self.gain[amp])
472 noiseList.append(self.noise[amp])
474 meanXcorrsList.append(self.meanXcorrs[amp].reshape(kernelLength).tolist())
475 kernelList.append(self.ampKernels[amp].reshape(kernelLength).tolist())
476 validList.append(int(self.valid[amp] and not (amp in self.badAmps)))
478 ampTable = Table({'AMPLIFIER': ampList,
479 'EXP_ID_MASK': expIdMaskList,
480 'RAW_MEANS': rawMeanList,
481 'RAW_VARIANCES': rawVarianceList,
482 'RAW_XCORRS': rawXcorrs,
483 'GAIN': gainList,
484 'NOISE': noiseList,
485 'MEAN_XCORRS': meanXcorrsList,
486 'KERNEL': kernelList,
487 'VALID': validList,
488 })
490 ampTable.meta = self.getMetadata().toDict()
491 tableList.append(ampTable)
493 if len(self.detKernels):
494 detList = []
495 kernelList = []
496 for det in self.detKernels.keys():
497 detList.append(det)
498 kernelList.append(self.detKernels[det].reshape(kernelLength).tolist())
500 detTable = Table({'DETECTOR': detList,
501 'KERNEL': kernelList})
502 detTable.meta = self.getMetadata().toDict()
503 tableList.append(detTable)
505 return tableList
507 # Implementation methods
508 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]):
509 """Average the amplifier level kernels to create a detector level
510 kernel.
511 """
512 inKernels = np.array([self.ampKernels[amp] for amp in
513 self.ampKernels if amp not in ampsToExclude])
514 averagingList = np.transpose(inKernels)
515 avgKernel = np.zeros_like(inKernels[0])
516 sctrl = afwMath.StatisticsControl()
517 sctrl.setNumSigmaClip(5.0)
518 for i in range(np.shape(avgKernel)[0]):
519 for j in range(np.shape(avgKernel)[1]):
520 avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j],
521 afwMath.MEANCLIP, sctrl).getValue()
523 self.detKernels[detectorName] = avgKernel
525 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName):
526 self.detKernel[detectorName] = self.ampKernel[ampName]