32 from lsst.geom import Box2I, Point2I, Extent2I
33 from .applyLookupTable
import applyLookupTable
35 __all__ = [
"Linearizer",
36 "LinearizeBase",
"LinearizeLookupTable",
"LinearizeSquared",
37 "LinearizeProportional",
"LinearizePolynomial",
"LinearizeNone"]
41 """Parameter set for linearization.
43 These parameters are included in cameraGeom.Amplifier, but
44 should be accessible externally to allow for testing.
48 table : `numpy.array`, optional
49 Lookup table; a 2-dimensional array of floats:
50 - one row for each row index (value of coef[0] in the amplifier)
51 - one column for each image value
52 To avoid copying the table the last index should vary fastest
53 (numpy default "C" order)
54 detector : `lsst.afw.cameraGeom.Detector`
56 override : `bool`, optional
57 Override the parameters defined in the detector/amplifier.
58 log : `lsst.log.Log`, optional
59 Logger to handle messages.
64 Raised if the supplied table is not 2D, or if the table has fewer
65 columns than rows (indicating that the indices are swapped).
68 _OBSTYPE =
"linearizer"
69 """The dataset type name used for this class"""
71 def __init__(self, table=None, detector=None, override=False, log=None):
94 if len(table.shape) != 2:
95 raise RuntimeError(
"table shape = %s; must have two dimensions" % (table.shape,))
96 if table.shape[1] < table.shape[0]:
97 raise RuntimeError(
"table shape = %s; indices are switched" % (table.shape,))
98 self.
tableData = np.array(table, order=
"C")
104 """Apply linearity, setting parameters if necessary.
108 exposure : `lsst.afw.image.Exposure`
113 output : `lsst.pipe.base.Struct`
114 Linearization results:
116 Number of amplifiers considered.
118 Number of amplifiers linearized.
122 """Read linearity parameters from a detector.
126 detector : `lsst.afw.cameraGeom.detector`
127 Input detector with parameters to use.
129 self._detectorName = detector.getName()
130 self._detectorSerial = detector.getSerial()
131 self._detectorId = detector.getId()
132 self.populated =
True
135 for amp
in detector.getAmplifiers():
136 ampName = amp.getName()
137 self.linearityCoeffs[ampName] = amp.getLinearityCoeffs()
138 self.linearityType[ampName] = amp.getLinearityType()
139 self.linearityBBox[ampName] = amp.getBBox()
142 """Read linearity parameters from a dict.
147 Dictionary containing detector and amplifier information.
149 self.
setMetadata(metadata=yamlObject.get(
'metadata',
None))
156 for ampName
in yamlObject[
'amplifiers']:
157 amp = yamlObject[
'amplifiers'][ampName]
158 self.
linearityCoeffs[ampName] = np.array(amp.get(
'linearityCoeffs',
None), dtype=np.float64)
159 self.
linearityType[ampName] = amp.get(
'linearityType',
'None')
161 self.
fitParams[ampName] = np.array(amp.get(
'linearityFitParams',
None), dtype=np.float64)
162 self.
fitParamsErr[ampName] = np.array(amp.get(
'linearityFitParamsErr',
None), dtype=np.float64)
166 self.
tableData = yamlObject.get(
'tableData',
None)
173 """Return linearity parameters as a dict.
180 now = datetime.datetime.utcnow()
188 'amplifiers': dict()}
190 outDict[
'amplifiers'][ampName] = {
'linearityType': self.
linearityType[ampName],
193 'linearityFitParams': self.
fitParams[ampName],
195 'linearityFitReducedChiSquared': (
198 outDict[
'tableData'] = self.
tableData.tolist()
204 """Read linearity from text file.
209 Name of the file containing the linearity definition.
212 linearity : `~lsst.ip.isr.linearize.Linearizer``
213 Linearity parameters.
216 with open(filename,
'r')
as f:
217 data = yaml.load(f, Loader=yaml.CLoader)
221 """Write the linearity model to a text file.
226 Name of the file to write.
231 The name of the file used to write the data.
236 Raised if filename does not end in ".yaml".
240 The file is written to YAML format and will include any metadata
241 associated with the `Linearity`.
244 if filename.lower().endswith((
".yaml")):
245 with open(filename,
'w')
as f:
246 yaml.dump(outDict, f)
248 raise RuntimeError(f
"Attempt to write to a file {filename} that does not end in '.yaml'")
254 """Read linearity from a FITS file.
258 table : `lsst.afw.table`
259 afwTable read from input file name.
260 tableExtTwo: `lsst.afw.table`, optional
261 afwTable read from second extension of input file name
265 linearity : `~lsst.ip.isr.linearize.Linearizer``
266 Linearity parameters.
270 The method reads a FITS file with 1 or 2 extensions. The metadata is read from the header of
271 extension 1, which must exist. Then the table is loaded, and the ['AMPLIFIER_NAME', 'TYPE',
272 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to
273 set each dictionary by looping over rows.
274 Eextension 2 is then attempted to read in the try block (which only exists for lookup tables).
275 It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row.
277 metadata = table.getMetadata()
278 schema = table.getSchema()
281 linDict[
'metadata'] = metadata
282 linDict[
'detectorId'] = metadata[
'DETECTOR']
283 linDict[
'detectorName'] = metadata[
'DETECTOR_NAME']
285 linDict[
'detectorSerial'] = metadata[
'DETECTOR_SERIAL']
287 linDict[
'detectorSerial'] =
'NOT SET'
288 linDict[
'amplifiers'] = dict()
291 ampNameKey = schema[
'AMPLIFIER_NAME'].asKey()
292 typeKey = schema[
'TYPE'].asKey()
293 coeffsKey = schema[
'COEFFS'].asKey()
294 x0Key = schema[
'BBOX_X0'].asKey()
295 y0Key = schema[
'BBOX_Y0'].asKey()
296 dxKey = schema[
'BBOX_DX'].asKey()
297 dyKey = schema[
'BBOX_DY'].asKey()
298 fitParamsKey = schema[
"FIT_PARAMS"].asKey()
299 fitParamsErrKey = schema[
"FIT_PARAMS_ERR"].asKey()
300 reducedChiSquaredKey = schema[
"RED_CHI_SQ"].asKey()
303 ampName = record[ampNameKey]
305 ampDict[
'linearityType'] = record[typeKey]
306 ampDict[
'linearityCoeffs'] = record[coeffsKey]
307 ampDict[
'linearityBBox'] =
Box2I(
Point2I(record[x0Key], record[y0Key]),
308 Extent2I(record[dxKey], record[dyKey]))
309 ampDict[
'linearityFitParams'] = record[fitParamsKey]
310 ampDict[
'linearityFitParamsErr'] = record[fitParamsErrKey]
311 ampDict[
'linearityFitReducedChiSquared'] = record[reducedChiSquaredKey]
313 linDict[
'amplifiers'][ampName] = ampDict
315 if tableExtTwo
is not None:
316 lookupValuesKey =
'LOOKUP_VALUES'
317 linDict[
"tableData"] = [record[lookupValuesKey]
for record
in tableExtTwo]
323 """Read linearity from a FITS file.
328 Name of the file containing the linearity definition.
331 linearity : `~lsst.ip.isr.linearize.Linearizer``
332 Linearity parameters.
336 This method and `fromTable` read a FITS file with 1 or 2 extensions. The metadata is read from the
337 header of extension 1, which must exist. Then the table is loaded, and the ['AMPLIFIER_NAME',
338 'TYPE', 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to
339 set each dictionary by looping over rows.
340 Extension 2 is then attempted to read in the try block (which only exists for lookup tables).
341 It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row.
343 table = afwTable.BaseCatalog.readFits(filename)
346 tableExtTwo = afwTable.BaseCatalog.readFits(filename, 2)
349 return cls().
fromTable(table, tableExtTwo=tableExtTwo)
352 """Produce linearity catalog
356 metadata : `lsst.daf.base.PropertyList`
361 catalog : `lsst.afw.table.BaseCatalog`
364 metadata[
"LINEARITY_SCHEMA"] =
"Linearity table"
365 metadata[
"LINEARITY_VERSION"] = 1
370 schema = afwTable.Schema()
371 names = schema.addField(
"AMPLIFIER_NAME", type=
"String", size=16, doc=
"linearity amplifier name")
372 types = schema.addField(
"TYPE", type=
"String", size=16, doc=
"linearity type names")
373 coeffs = schema.addField(
"COEFFS", type=
"ArrayD", size=length, doc=
"linearity coefficients")
374 boxX = schema.addField(
"BBOX_X0", type=
"I", doc=
"linearity bbox minimum x")
375 boxY = schema.addField(
"BBOX_Y0", type=
"I", doc=
"linearity bbox minimum y")
376 boxDx = schema.addField(
"BBOX_DX", type=
"I", doc=
"linearity bbox x dimension")
377 boxDy = schema.addField(
"BBOX_DY", type=
"I", doc=
"linearity bbox y dimension")
382 fitParams = schema.addField(
"FIT_PARAMS", type=
"ArrayD", size=lengthFitParams,
383 doc=
"parameters of linearity polynomial fit")
384 fitParamsErr = schema.addField(
"FIT_PARAMS_ERR", type=
"ArrayD", size=lengthFitParams,
385 doc=
"errors of parameters of linearity polynomial fit")
386 reducedChiSquared = schema.addField(
"RED_CHI_SQ", type=
"D",
387 doc=
"unweighted reduced chi sq. from linearity pol. fit")
389 catalog = afwTable.BaseCatalog(schema)
393 catalog[ii][names] = ampName
395 catalog[ii][coeffs] = np.array(self.
linearityCoeffs[ampName], dtype=float)
397 catalog[ii][fitParams] = np.array(self.
fitParams[ampName], dtype=float)
398 catalog[ii][fitParamsErr] = np.array(self.
fitParamsErr[ampName], dtype=float)
402 catalog[ii][boxX], catalog[ii][boxY] = bbox.getMin()
403 catalog[ii][boxDx], catalog[ii][boxDy] = bbox.getDimensions()
404 catalog.setMetadata(metadata)
409 """Produce linearity catalog from table data
413 metadata : `lsst.daf.base.PropertyList`
418 catalog : `lsst.afw.table.BaseCatalog`
422 schema = afwTable.Schema()
424 lut = schema.addField(
"LOOKUP_VALUES", type=
'ArrayF', size=dimensions[1],
425 doc=
"linearity lookup data")
426 catalog = afwTable.BaseCatalog(schema)
427 catalog.resize(dimensions[0])
429 for ii
in range(dimensions[0]):
430 catalog[ii][lut] = np.array(self.
tableData[ii], dtype=np.float32)
432 metadata[
"LINEARITY_LOOKUP"] =
True
433 catalog.setMetadata(metadata)
438 """Write the linearity model to a FITS file.
443 Name of the file to write.
447 The file is written to YAML format and will include any metadata
448 associated with the `Linearity`.
450 now = datetime.datetime.utcnow()
454 catalog.writeFits(filename)
458 catalog.writeFits(filename,
"a")
463 """Retrieve metadata associated with this `Linearizer`.
467 meta : `lsst.daf.base.PropertyList`
468 Metadata. The returned `~lsst.daf.base.PropertyList` can be
469 modified by the caller and the changes will be written to
475 """Store a copy of the supplied metadata with the `Linearizer`.
479 metadata : `lsst.daf.base.PropertyList`, optional
480 Metadata to associate with the linearizer. Will be copied and
481 overwrite existing metadata. If not supplied the existing
482 metadata will be reset.
492 def updateMetadata(self, date=None, detectorId=None, detectorName=None, instrumentName=None, calibId=None,
494 """Update metadata keywords with new values.
498 date : `datetime.datetime`, optional
499 detectorId : `int`, optional
500 detectorName: `str`, optional
501 instrumentName : `str`, optional
502 calibId: `str`, optional
503 serial: detector serial, `str`, optional
507 mdSupplemental = dict()
510 mdSupplemental[
'CALIBDATE'] = date.isoformat()
511 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat(),
512 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat(),
514 mdSupplemental[
'DETECTOR'] = f
"{detectorId}"
516 mdSupplemental[
'DETECTOR_NAME'] = detectorName
518 mdSupplemental[
'INSTRUME'] = instrumentName
520 mdSupplemental[
'CALIB_ID'] = calibId
522 mdSupplemental[
'DETECTOR_SERIAL'] = serial
524 mdOriginal.update(mdSupplemental)
527 """Determine the linearity class to use from the type name.
531 linearityTypeName : str
532 String name of the linearity type that is needed.
536 linearityType : `~lsst.ip.isr.linearize.LinearizeBase`
537 The appropriate linearity class to use. If no matching class
538 is found, `None` is returned.
540 for t
in [LinearizeLookupTable,
543 LinearizeProportional,
545 if t.LinearityType == linearityTypeName:
550 """Validate linearity for a detector/amplifier.
554 detector : `lsst.afw.cameraGeom.Detector`, optional
555 Detector to validate, along with its amplifiers.
556 amplifier : `lsst.afw.cameraGeom.Amplifier`, optional
557 Single amplifier to validate.
562 Raised if there is a mismatch in linearity parameters, and
563 the cameraGeom parameters are not being overridden.
565 amplifiersToCheck = []
568 raise RuntimeError(
"Detector names don't match: %s != %s" %
571 raise RuntimeError(
"Detector IDs don't match: %s != %s" %
574 raise RuntimeError(
"Detector serial numbers don't match: %s != %s" %
577 raise RuntimeError(
"Detector number of amps = %s does not match saved value %s" %
578 (len(detector.getAmplifiers()),
580 amplifiersToCheck.extend(detector.getAmplifiers())
583 amplifiersToCheck.extend(amplifier)
585 for amp
in amplifiersToCheck:
586 ampName = amp.getName()
588 raise RuntimeError(
"Amplifier %s is not in linearity data" %
592 self.
log.warn(
"Overriding amplifier defined linearityType (%s) for %s",
595 raise RuntimeError(
"Amplifier %s type %s does not match saved value %s" %
596 (ampName, amp.getLinearityType(), self.
linearityType[ampName]))
597 if (amp.getLinearityCoeffs().shape != self.
linearityCoeffs[ampName].shape
or not
598 np.allclose(amp.getLinearityCoeffs(), self.
linearityCoeffs[ampName], equal_nan=
True)):
600 self.
log.warn(
"Overriding amplifier defined linearityCoeffs (%s) for %s",
603 raise RuntimeError(
"Amplifier %s coeffs %s does not match saved value %s" %
607 """Apply the linearity to an image.
609 If the linearity parameters are populated, use those,
610 otherwise use the values from the detector.
614 image : `~lsst.afw.image.image`
616 detector : `~lsst.afw.cameraGeom.detector`
617 Detector to use for linearity parameters if not already
619 log : `~lsst.log.Log`, optional
620 Log object to use for logging.
635 if linearizer
is not None:
637 success, outOfRange = linearizer()(ampView, **{
'coeffs': self.
linearityCoeffs[ampName],
640 numOutOfRange += outOfRange
643 elif log
is not None:
644 log.warn(
"Amplifier %s did not linearize.",
648 numLinearized=numLinearized,
649 numOutOfRange=numOutOfRange
654 """Abstract base class functor for correcting non-linearity.
656 Subclasses must define __call__ and set class variable
657 LinearityType to a string that will be used for linearity type in
658 the cameraGeom.Amplifier.linearityType field.
660 All linearity corrections should be defined in terms of an
661 additive correction, such that:
663 corrected_value = uncorrected_value + f(uncorrected_value)
669 """Correct non-linearity.
673 image : `lsst.afw.image.Image`
674 Image to be corrected
676 Dictionary of parameter keywords:
678 Coefficient vector (`list` or `numpy.array`).
680 Lookup table data (`numpy.array`).
682 Logger to handle messages (`lsst.log.Log`).
687 If true, a correction was applied successfully.
692 Raised if the linearity type listed in the
693 detector does not match the class type.
698 class LinearizeLookupTable(LinearizeBase):
699 """Correct non-linearity with a persisted lookup table.
701 The lookup table consists of entries such that given
702 "coefficients" c0, c1:
704 for each i,j of image:
706 colInd = int(c1 + uncorrImage[i,j])
707 corrImage[i,j] = uncorrImage[i,j] + table[rowInd, colInd]
709 - c0: row index; used to identify which row of the table to use
710 (typically one per amplifier, though one can have multiple
711 amplifiers use the same table)
712 - c1: column index offset; added to the uncorrected image value
713 before truncation; this supports tables that can handle
714 negative image values; also, if the c1 ends with .5 then
715 the nearest index is used instead of truncating to the
718 LinearityType =
"LookupTable"
721 """Correct for non-linearity.
725 image : `lsst.afw.image.Image`
726 Image to be corrected
728 Dictionary of parameter keywords:
730 Columnation vector (`list` or `numpy.array`).
732 Lookup table data (`numpy.array`).
734 Logger to handle messages (`lsst.log.Log`).
739 If true, a correction was applied successfully.
744 Raised if the requested row index is out of the table
749 rowInd, colIndOffset = kwargs[
'coeffs'][0:2]
750 table = kwargs[
'table']
753 numTableRows = table.shape[0]
755 if rowInd < 0
or rowInd > numTableRows:
756 raise RuntimeError(
"LinearizeLookupTable rowInd=%s not in range[0, %s)" %
757 (rowInd, numTableRows))
758 tableRow = table[rowInd, :]
761 if numOutOfRange > 0
and log
is not None:
762 log.warn(
"%s pixels were out of range of the linearization table",
764 if numOutOfRange < image.getArray().size:
765 return True, numOutOfRange
767 return False, numOutOfRange
771 """Correct non-linearity with a polynomial mode.
773 corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
775 where c_i are the linearity coefficients for each amplifier.
776 Lower order coefficients are not included as they duplicate other
777 calibration parameters:
779 A coefficient multiplied by uncorrImage**0 is equivalent to
780 bias level. Irrelevant for correcting non-linearity.
782 A coefficient multiplied by uncorrImage**1 is proportional
783 to the gain. Not necessary for correcting non-linearity.
785 LinearityType =
"Polynomial"
788 """Correct non-linearity.
792 image : `lsst.afw.image.Image`
793 Image to be corrected
795 Dictionary of parameter keywords:
797 Coefficient vector (`list` or `numpy.array`).
798 If the order of the polynomial is n, this list
799 should have a length of n-1 ("k0" and "k1" are
800 not needed for the correction).
802 Logger to handle messages (`lsst.log.Log`).
807 If true, a correction was applied successfully.
809 if not np.any(np.isfinite(kwargs[
'coeffs'])):
811 if not np.any(kwargs[
'coeffs']):
814 ampArray = image.getArray()
815 correction = np.zeros_like(ampArray)
816 for order, coeff
in enumerate(kwargs[
'coeffs'], start=2):
817 correction += coeff * np.power(ampArray, order)
818 ampArray += correction
824 """Correct non-linearity with a squared model.
826 corrImage = uncorrImage + c0*uncorrImage^2
828 where c0 is linearity coefficient 0 for each amplifier.
830 LinearityType =
"Squared"
833 """Correct for non-linearity.
837 image : `lsst.afw.image.Image`
838 Image to be corrected
840 Dictionary of parameter keywords:
842 Coefficient vector (`list` or `numpy.array`).
844 Logger to handle messages (`lsst.log.Log`).
849 If true, a correction was applied successfully.
852 sqCoeff = kwargs[
'coeffs'][0]
854 ampArr = image.getArray()
855 ampArr *= (1 + sqCoeff*ampArr)
862 """Do not correct non-linearity.
864 LinearityType =
"Proportional"
867 """Do not correct for non-linearity.
871 image : `lsst.afw.image.Image`
872 Image to be corrected
874 Dictionary of parameter keywords:
876 Coefficient vector (`list` or `numpy.array`).
878 Logger to handle messages (`lsst.log.Log`).
883 If true, a correction was applied successfully.
889 """Do not correct non-linearity.
891 LinearityType =
"None"
894 """Do not correct for non-linearity.
898 image : `lsst.afw.image.Image`
899 Image to be corrected
901 Dictionary of parameter keywords:
903 Coefficient vector (`list` or `numpy.array`).
905 Logger to handle messages (`lsst.log.Log`).
910 If true, a correction was applied successfully.