Coverage for python/lsst/ip/isr/linearize.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import abc
23import numpy as np
25from astropy.table import Table
27from lsst.pipe.base import Struct
28from lsst.geom import Box2I, Point2I, Extent2I
29from .applyLookupTable import applyLookupTable
30from .calibType import IsrCalib
32__all__ = ["Linearizer",
33 "LinearizeBase", "LinearizeLookupTable", "LinearizeSquared",
34 "LinearizeProportional", "LinearizePolynomial", "LinearizeNone"]
37class Linearizer(IsrCalib):
38 """Parameter set for linearization.
40 These parameters are included in cameraGeom.Amplifier, but
41 should be accessible externally to allow for testing.
43 Parameters
44 ----------
45 table : `numpy.array`, optional
46 Lookup table; a 2-dimensional array of floats:
47 - one row for each row index (value of coef[0] in the amplifier)
48 - one column for each image value
49 To avoid copying the table the last index should vary fastest
50 (numpy default "C" order)
51 detector : `lsst.afw.cameraGeom.Detector`, optional
52 Detector object. Passed to self.fromDetector() on init.
53 log : `lsst.log.Log`, optional
54 Logger to handle messages.
55 kwargs : `dict`, optional
56 Other keyword arguments to pass to the parent init.
58 Raises
59 ------
60 RuntimeError :
61 Raised if the supplied table is not 2D, or if the table has fewer
62 columns than rows (indicating that the indices are swapped).
64 Notes
65 -----
66 The linearizer attributes stored are:
68 hasLinearity : `bool`
69 Whether a linearity correction is defined for this detector.
70 override : `bool`
71 Whether the detector parameters should be overridden.
72 ampNames : `list` [`str`]
73 List of amplifier names to correct.
74 linearityCoeffs : `dict` [`str`, `numpy.array`]
75 Coefficients to use in correction. Indexed by amplifier
76 names. The format of the array depends on the type of
77 correction to apply.
78 linearityType : `dict` [`str`, `str`]
79 Type of correction to use, indexed by amplifier names.
80 linearityBBox : `dict` [`str`, `lsst.geom.Box2I`]
81 Bounding box the correction is valid over, indexed by
82 amplifier names.
83 fitParams : `dict` [`str`, `numpy.array`], optional
84 Linearity fit parameters used to construct the correction
85 coefficients, indexed as above.
86 fitParamsErr : `dict` [`str`, `numpy.array`], optional
87 Uncertainty values of the linearity fit parameters used to
88 construct the correction coefficients, indexed as above.
89 fitChiSq : `dict` [`str`, `float`], optional
90 Chi-squared value of the linearity fit, indexed as above.
91 tableData : `numpy.array`, optional
92 Lookup table data for the linearity correction.
93 """
94 _OBSTYPE = "LINEARIZER"
95 _SCHEMA = 'Gen3 Linearizer'
96 _VERSION = 1.1
98 def __init__(self, table=None, **kwargs):
99 self.hasLinearity = False
100 self.override = False
102 self.ampNames = list()
103 self.linearityCoeffs = dict()
104 self.linearityType = dict()
105 self.linearityBBox = dict()
107 self.fitParams = dict()
108 self.fitParamsErr = dict()
109 self.fitChiSq = dict()
111 self.tableData = None
112 if table is not None:
113 if len(table.shape) != 2:
114 raise RuntimeError("table shape = %s; must have two dimensions" % (table.shape,))
115 if table.shape[1] < table.shape[0]:
116 raise RuntimeError("table shape = %s; indices are switched" % (table.shape,))
117 self.tableData = np.array(table, order="C")
119 super().__init__(**kwargs)
120 self.requiredAttributes.update(['hasLinearity', 'override',
121 'ampNames',
122 'linearityCoeffs', 'linearityType', 'linearityBBox',
123 'fitParams', 'fitParamsErr', 'fitChiSq',
124 'tableData'])
126 def updateMetadata(self, setDate=False, **kwargs):
127 """Update metadata keywords with new values.
129 This calls the base class's method after ensuring the required
130 calibration keywords will be saved.
132 Parameters
133 ----------
134 setDate : `bool`, optional
135 Update the CALIBDATE fields in the metadata to the current
136 time. Defaults to False.
137 kwargs :
138 Other keyword parameters to set in the metadata.
139 """
140 kwargs['HAS_LINEARITY'] = self.hasLinearity
141 kwargs['OVERRIDE'] = self.override
142 kwargs['HAS_TABLE'] = self.tableData is not None
144 super().updateMetadata(setDate=setDate, **kwargs)
146 def fromDetector(self, detector):
147 """Read linearity parameters from a detector.
149 Parameters
150 ----------
151 detector : `lsst.afw.cameraGeom.detector`
152 Input detector with parameters to use.
154 Returns
155 -------
156 calib : `lsst.ip.isr.Linearizer`
157 The calibration constructed from the detector.
158 """
159 self._detectorName = detector.getName()
160 self._detectorSerial = detector.getSerial()
161 self._detectorId = detector.getId()
162 self.hasLinearity = True
164 # Do not translate Threshold, Maximum, Units.
165 for amp in detector.getAmplifiers():
166 ampName = amp.getName()
167 self.ampNames.append(ampName)
168 self.linearityType[ampName] = amp.getLinearityType()
169 self.linearityCoeffs[ampName] = amp.getLinearityCoeffs()
170 self.linearityBBox[ampName] = amp.getBBox()
172 return self
174 @classmethod
175 def fromDict(cls, dictionary):
176 """Construct a calibration from a dictionary of properties
178 Parameters
179 ----------
180 dictionary : `dict`
181 Dictionary of properties
183 Returns
184 -------
185 calib : `lsst.ip.isr.Linearity`
186 Constructed calibration.
188 Raises
189 ------
190 RuntimeError
191 Raised if the supplied dictionary is for a different
192 calibration.
193 """
195 calib = cls()
197 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
198 raise RuntimeError(f"Incorrect linearity supplied. Expected {calib._OBSTYPE}, "
199 f"found {dictionary['metadata']['OBSTYPE']}")
201 calib.setMetadata(dictionary['metadata'])
203 calib.hasLinearity = dictionary.get('hasLinearity',
204 dictionary['metadata'].get('HAS_LINEARITY', False))
205 calib.override = dictionary.get('override', True)
207 if calib.hasLinearity:
208 for ampName in dictionary['amplifiers']:
209 amp = dictionary['amplifiers'][ampName]
210 calib.ampNames.append(ampName)
211 calib.linearityCoeffs[ampName] = np.array(amp.get('linearityCoeffs', None), dtype=np.float64)
212 calib.linearityType[ampName] = amp.get('linearityType', 'None')
213 calib.linearityBBox[ampName] = amp.get('linearityBBox', None)
215 calib.fitParams[ampName] = np.array(amp.get('fitParams', None), dtype=np.float64)
216 calib.fitParamsErr[ampName] = np.array(amp.get('fitParamsErr', None), dtype=np.float64)
217 calib.fitChiSq[ampName] = amp.get('fitChiSq', None)
219 calib.tableData = dictionary.get('tableData', None)
220 if calib.tableData:
221 calib.tableData = np.array(calib.tableData)
223 return calib
225 def toDict(self):
226 """Return linearity parameters as a dict.
228 Returns
229 -------
230 outDict : `dict`:
231 """
232 self.updateMetadata()
234 outDict = {'metadata': self.getMetadata(),
235 'detectorName': self._detectorName,
236 'detectorSerial': self._detectorSerial,
237 'detectorId': self._detectorId,
238 'hasTable': self.tableData is not None,
239 'amplifiers': dict(),
240 }
241 for ampName in self.linearityType:
242 outDict['amplifiers'][ampName] = {'linearityType': self.linearityType[ampName],
243 'linearityCoeffs': self.linearityCoeffs[ampName].toList(),
244 'linearityBBox': self.linearityBBox[ampName],
245 'fitParams': self.fitParams[ampName].toList(),
246 'fitParamsErr': self.fitParamsErr[ampName].toList(),
247 'fitChiSq': self.fitChiSq[ampName]}
248 if self.tableData is not None:
249 outDict['tableData'] = self.tableData.tolist()
251 return outDict
253 @classmethod
254 def fromTable(cls, tableList):
255 """Read linearity from a FITS file.
257 This method uses the `fromDict` method to create the
258 calibration, after constructing an appropriate dictionary from
259 the input tables.
261 Parameters
262 ----------
263 tableList : `list` [`astropy.table.Table`]
264 afwTable read from input file name.
266 Returns
267 -------
268 linearity : `~lsst.ip.isr.linearize.Linearizer``
269 Linearity parameters.
271 Notes
272 -----
273 The method reads a FITS file with 1 or 2 extensions. The metadata is read from the header of
274 extension 1, which must exist. Then the table is loaded, and the ['AMPLIFIER_NAME', 'TYPE',
275 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to
276 set each dictionary by looping over rows.
277 Eextension 2 is then attempted to read in the try block (which only exists for lookup tables).
278 It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row.
280 """
281 coeffTable = tableList[0]
283 metadata = coeffTable.meta
284 inDict = dict()
285 inDict['metadata'] = metadata
286 inDict['hasLinearity'] = metadata['HAS_LINEARITY']
287 inDict['amplifiers'] = dict()
289 for record in coeffTable:
290 ampName = record['AMPLIFIER_NAME']
292 fitParams = record['FIT_PARAMS'] if 'FIT_PARAMS' in record else None
293 fitParamsErr = record['FIT_PARAMS_ERR'] if 'FIT_PARAMS_ERR' in record else None
294 fitChiSq = record['RED_CHI_SQ'] if 'RED_CHI_SQ' in record else None
296 inDict['amplifiers'][ampName] = {
297 'linearityType': record['TYPE'],
298 'linearityCoeffs': record['COEFFS'],
299 'linearityBBox': Box2I(Point2I(record['BBOX_X0'], record['BBOX_Y0']),
300 Extent2I(record['BBOX_DX'], record['BBOX_DY'])),
301 'fitParams': fitParams,
302 'fitParamsErr': fitParamsErr,
303 'fitChiSq': fitChiSq,
304 }
306 if len(tableList) > 1:
307 tableData = tableList[1]
308 inDict['tableData'] = [record['LOOKUP_VALUES'] for record in tableData]
310 return cls().fromDict(inDict)
312 def toTable(self):
313 """Construct a list of tables containing the information in this calibration
315 The list of tables should create an identical calibration
316 after being passed to this class's fromTable method.
318 Returns
319 -------
320 tableList : `list` [`astropy.table.Table`]
321 List of tables containing the linearity calibration
322 information.
323 """
325 tableList = []
326 self.updateMetadata()
327 catalog = Table([{'AMPLIFIER_NAME': ampName,
328 'TYPE': self.linearityType[ampName],
329 'COEFFS': self.linearityCoeffs[ampName],
330 'BBOX_X0': self.linearityBBox[ampName].getMinX(),
331 'BBOX_Y0': self.linearityBBox[ampName].getMinY(),
332 'BBOX_DX': self.linearityBBox[ampName].getWidth(),
333 'BBOX_DY': self.linearityBBox[ampName].getHeight(),
334 'FIT_PARAMS': self.fitParams[ampName],
335 'FIT_PARAMS_ERR': self.fitParamsErr[ampName],
336 'RED_CHI_SQ': self.fitChiSq[ampName],
337 } for ampName in self.ampNames])
339 tableList.append(catalog)
341 if self.tableData:
342 catalog = Table([{'LOOKUP_VALUES': value} for value in self.tableData])
343 tableList.append(catalog)
345 return(catalog)
347 def getLinearityTypeByName(self, linearityTypeName):
348 """Determine the linearity class to use from the type name.
350 Parameters
351 ----------
352 linearityTypeName : str
353 String name of the linearity type that is needed.
355 Returns
356 -------
357 linearityType : `~lsst.ip.isr.linearize.LinearizeBase`
358 The appropriate linearity class to use. If no matching class
359 is found, `None` is returned.
360 """
361 for t in [LinearizeLookupTable,
362 LinearizeSquared,
363 LinearizePolynomial,
364 LinearizeProportional,
365 LinearizeNone]:
366 if t.LinearityType == linearityTypeName:
367 return t
368 return None
370 def validate(self, detector=None, amplifier=None):
371 """Validate linearity for a detector/amplifier.
373 Parameters
374 ----------
375 detector : `lsst.afw.cameraGeom.Detector`, optional
376 Detector to validate, along with its amplifiers.
377 amplifier : `lsst.afw.cameraGeom.Amplifier`, optional
378 Single amplifier to validate.
380 Raises
381 ------
382 RuntimeError :
383 Raised if there is a mismatch in linearity parameters, and
384 the cameraGeom parameters are not being overridden.
385 """
386 amplifiersToCheck = []
387 if detector:
388 if self._detectorName != detector.getName():
389 raise RuntimeError("Detector names don't match: %s != %s" %
390 (self._detectorName, detector.getName()))
391 if int(self._detectorId) != int(detector.getId()):
392 raise RuntimeError("Detector IDs don't match: %s != %s" %
393 (int(self._detectorId), int(detector.getId())))
394 if self._detectorSerial != detector.getSerial():
395 raise RuntimeError("Detector serial numbers don't match: %s != %s" %
396 (self._detectorSerial, detector.getSerial()))
397 if len(detector.getAmplifiers()) != len(self.linearityCoeffs.keys()):
398 raise RuntimeError("Detector number of amps = %s does not match saved value %s" %
399 (len(detector.getAmplifiers()),
400 len(self.linearityCoeffs.keys())))
401 amplifiersToCheck.extend(detector.getAmplifiers())
403 if amplifier:
404 amplifiersToCheck.extend(amplifier)
406 for amp in amplifiersToCheck:
407 ampName = amp.getName()
408 if ampName not in self.linearityCoeffs.keys():
409 raise RuntimeError("Amplifier %s is not in linearity data" %
410 (ampName, ))
411 if amp.getLinearityType() != self.linearityType[ampName]:
412 if self.override:
413 self.log.warn("Overriding amplifier defined linearityType (%s) for %s",
414 self.linearityType[ampName], ampName)
415 else:
416 raise RuntimeError("Amplifier %s type %s does not match saved value %s" %
417 (ampName, amp.getLinearityType(), self.linearityType[ampName]))
418 if (amp.getLinearityCoeffs().shape != self.linearityCoeffs[ampName].shape or not
419 np.allclose(amp.getLinearityCoeffs(), self.linearityCoeffs[ampName], equal_nan=True)):
420 if self.override:
421 self.log.warn("Overriding amplifier defined linearityCoeffs (%s) for %s",
422 self.linearityCoeffs[ampName], ampName)
423 else:
424 raise RuntimeError("Amplifier %s coeffs %s does not match saved value %s" %
425 (ampName, amp.getLinearityCoeffs(), self.linearityCoeffs[ampName]))
427 def applyLinearity(self, image, detector=None, log=None):
428 """Apply the linearity to an image.
430 If the linearity parameters are populated, use those,
431 otherwise use the values from the detector.
433 Parameters
434 ----------
435 image : `~lsst.afw.image.image`
436 Image to correct.
437 detector : `~lsst.afw.cameraGeom.detector`
438 Detector to use for linearity parameters if not already
439 populated.
440 log : `~lsst.log.Log`, optional
441 Log object to use for logging.
442 """
443 if log is None:
444 log = self.log
445 if detector and not self.hasLinearity:
446 self.fromDetector(detector)
448 self.validate(detector)
450 numAmps = 0
451 numLinearized = 0
452 numOutOfRange = 0
453 for ampName in self.linearityType.keys():
454 linearizer = self.getLinearityTypeByName(self.linearityType[ampName])
455 numAmps += 1
456 if linearizer is not None:
457 ampView = image.Factory(image, self.linearityBBox[ampName])
458 success, outOfRange = linearizer()(ampView, **{'coeffs': self.linearityCoeffs[ampName],
459 'table': self.tableData,
460 'log': self.log})
461 numOutOfRange += outOfRange
462 if success:
463 numLinearized += 1
464 elif log is not None:
465 log.warn("Amplifier %s did not linearize.",
466 ampName)
467 return Struct(
468 numAmps=numAmps,
469 numLinearized=numLinearized,
470 numOutOfRange=numOutOfRange
471 )
474class LinearizeBase(metaclass=abc.ABCMeta):
475 """Abstract base class functor for correcting non-linearity.
477 Subclasses must define __call__ and set class variable
478 LinearityType to a string that will be used for linearity type in
479 the cameraGeom.Amplifier.linearityType field.
481 All linearity corrections should be defined in terms of an
482 additive correction, such that:
484 corrected_value = uncorrected_value + f(uncorrected_value)
485 """
486 LinearityType = None # linearity type, a string used for AmpInfoCatalogs
488 @abc.abstractmethod
489 def __call__(self, image, **kwargs):
490 """Correct non-linearity.
492 Parameters
493 ----------
494 image : `lsst.afw.image.Image`
495 Image to be corrected
496 kwargs : `dict`
497 Dictionary of parameter keywords:
498 ``"coeffs"``
499 Coefficient vector (`list` or `numpy.array`).
500 ``"table"``
501 Lookup table data (`numpy.array`).
502 ``"log"``
503 Logger to handle messages (`lsst.log.Log`).
505 Returns
506 -------
507 output : `bool`
508 If true, a correction was applied successfully.
510 Raises
511 ------
512 RuntimeError:
513 Raised if the linearity type listed in the
514 detector does not match the class type.
515 """
516 pass
519class LinearizeLookupTable(LinearizeBase):
520 """Correct non-linearity with a persisted lookup table.
522 The lookup table consists of entries such that given
523 "coefficients" c0, c1:
525 for each i,j of image:
526 rowInd = int(c0)
527 colInd = int(c1 + uncorrImage[i,j])
528 corrImage[i,j] = uncorrImage[i,j] + table[rowInd, colInd]
530 - c0: row index; used to identify which row of the table to use
531 (typically one per amplifier, though one can have multiple
532 amplifiers use the same table)
533 - c1: column index offset; added to the uncorrected image value
534 before truncation; this supports tables that can handle
535 negative image values; also, if the c1 ends with .5 then
536 the nearest index is used instead of truncating to the
537 next smaller index
538 """
539 LinearityType = "LookupTable"
541 def __call__(self, image, **kwargs):
542 """Correct for non-linearity.
544 Parameters
545 ----------
546 image : `lsst.afw.image.Image`
547 Image to be corrected
548 kwargs : `dict`
549 Dictionary of parameter keywords:
550 ``"coeffs"``
551 Columnation vector (`list` or `numpy.array`).
552 ``"table"``
553 Lookup table data (`numpy.array`).
554 ``"log"``
555 Logger to handle messages (`lsst.log.Log`).
557 Returns
558 -------
559 output : `bool`
560 If true, a correction was applied successfully.
562 Raises
563 ------
564 RuntimeError:
565 Raised if the requested row index is out of the table
566 bounds.
567 """
568 numOutOfRange = 0
570 rowInd, colIndOffset = kwargs['coeffs'][0:2]
571 table = kwargs['table']
572 log = kwargs['log']
574 numTableRows = table.shape[0]
575 rowInd = int(rowInd)
576 if rowInd < 0 or rowInd > numTableRows:
577 raise RuntimeError("LinearizeLookupTable rowInd=%s not in range[0, %s)" %
578 (rowInd, numTableRows))
579 tableRow = table[rowInd, :]
580 numOutOfRange += applyLookupTable(image, tableRow, colIndOffset)
582 if numOutOfRange > 0 and log is not None:
583 log.warn("%s pixels were out of range of the linearization table",
584 numOutOfRange)
585 if numOutOfRange < image.getArray().size:
586 return True, numOutOfRange
587 else:
588 return False, numOutOfRange
591class LinearizePolynomial(LinearizeBase):
592 """Correct non-linearity with a polynomial mode.
594 corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
596 where c_i are the linearity coefficients for each amplifier.
597 Lower order coefficients are not included as they duplicate other
598 calibration parameters:
599 ``"k0"``
600 A coefficient multiplied by uncorrImage**0 is equivalent to
601 bias level. Irrelevant for correcting non-linearity.
602 ``"k1"``
603 A coefficient multiplied by uncorrImage**1 is proportional
604 to the gain. Not necessary for correcting non-linearity.
605 """
606 LinearityType = "Polynomial"
608 def __call__(self, image, **kwargs):
609 """Correct non-linearity.
611 Parameters
612 ----------
613 image : `lsst.afw.image.Image`
614 Image to be corrected
615 kwargs : `dict`
616 Dictionary of parameter keywords:
617 ``"coeffs"``
618 Coefficient vector (`list` or `numpy.array`).
619 If the order of the polynomial is n, this list
620 should have a length of n-1 ("k0" and "k1" are
621 not needed for the correction).
622 ``"log"``
623 Logger to handle messages (`lsst.log.Log`).
625 Returns
626 -------
627 output : `bool`
628 If true, a correction was applied successfully.
629 """
630 if not np.any(np.isfinite(kwargs['coeffs'])):
631 return False, 0
632 if not np.any(kwargs['coeffs']):
633 return False, 0
635 ampArray = image.getArray()
636 correction = np.zeros_like(ampArray)
637 for order, coeff in enumerate(kwargs['coeffs'], start=2):
638 correction += coeff * np.power(ampArray, order)
639 ampArray += correction
641 return True, 0
644class LinearizeSquared(LinearizeBase):
645 """Correct non-linearity with a squared model.
647 corrImage = uncorrImage + c0*uncorrImage^2
649 where c0 is linearity coefficient 0 for each amplifier.
650 """
651 LinearityType = "Squared"
653 def __call__(self, image, **kwargs):
654 """Correct for non-linearity.
656 Parameters
657 ----------
658 image : `lsst.afw.image.Image`
659 Image to be corrected
660 kwargs : `dict`
661 Dictionary of parameter keywords:
662 ``"coeffs"``
663 Coefficient vector (`list` or `numpy.array`).
664 ``"log"``
665 Logger to handle messages (`lsst.log.Log`).
667 Returns
668 -------
669 output : `bool`
670 If true, a correction was applied successfully.
671 """
673 sqCoeff = kwargs['coeffs'][0]
674 if sqCoeff != 0:
675 ampArr = image.getArray()
676 ampArr *= (1 + sqCoeff*ampArr)
677 return True, 0
678 else:
679 return False, 0
682class LinearizeProportional(LinearizeBase):
683 """Do not correct non-linearity.
684 """
685 LinearityType = "Proportional"
687 def __call__(self, image, **kwargs):
688 """Do not correct for non-linearity.
690 Parameters
691 ----------
692 image : `lsst.afw.image.Image`
693 Image to be corrected
694 kwargs : `dict`
695 Dictionary of parameter keywords:
696 ``"coeffs"``
697 Coefficient vector (`list` or `numpy.array`).
698 ``"log"``
699 Logger to handle messages (`lsst.log.Log`).
701 Returns
702 -------
703 output : `bool`
704 If true, a correction was applied successfully.
705 """
706 return True, 0
709class LinearizeNone(LinearizeBase):
710 """Do not correct non-linearity.
711 """
712 LinearityType = "None"
714 def __call__(self, image, **kwargs):
715 """Do not correct for non-linearity.
717 Parameters
718 ----------
719 image : `lsst.afw.image.Image`
720 Image to be corrected
721 kwargs : `dict`
722 Dictionary of parameter keywords:
723 ``"coeffs"``
724 Coefficient vector (`list` or `numpy.array`).
725 ``"log"``
726 Logger to handle messages (`lsst.log.Log`).
728 Returns
729 -------
730 output : `bool`
731 If true, a correction was applied successfully.
732 """
733 return True, 0