Coverage for tests/test_isrTask.py: 14%
344 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 03:05 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 03:05 -0700
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.qa = IsrQaConfig()
81 self.task = IsrTask(config=self.config)
82 self.camera = isrMock.IsrMock().getCamera()
84 self.inputExp = isrMock.TrimmedRawMock().run()
85 self.amp = self.inputExp.getDetector()[0]
86 self.mi = self.inputExp.getMaskedImage()
88 def validateIsrData(self, results):
89 """results should be a struct with components that are
90 not None if included in the configuration file.
91 """
92 self.assertIsInstance(results, Struct)
93 if self.config.doBias is True:
94 self.assertIsNotNone(results.bias)
95 if self.config.doDark is True:
96 self.assertIsNotNone(results.dark)
97 if self.config.doFlat is True:
98 self.assertIsNotNone(results.flat)
99 if self.config.doFringe is True:
100 self.assertIsNotNone(results.fringes)
101 if self.config.doDefect is True:
102 self.assertIsNotNone(results.defects)
103 if self.config.doBrighterFatter is True:
104 self.assertIsNotNone(results.bfKernel)
105 if self.config.doAttachTransmissionCurve is True:
106 self.assertIsNotNone(results.opticsTransmission)
107 self.assertIsNotNone(results.filterTransmission)
108 self.assertIsNotNone(results.sensorTransmission)
109 self.assertIsNotNone(results.atmosphereTransmission)
111 def test_ensureExposure(self):
112 """Test that an exposure has a usable instance class.
113 """
114 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
115 afwImage.Exposure)
117 def test_convertItoF(self):
118 """Test conversion from integer to floating point pixels.
119 """
120 result = self.task.convertIntToFloat(self.inputExp)
121 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
122 self.assertEqual(result, self.inputExp)
124 def test_updateVariance(self):
125 """Expect The variance image should have a larger median value after
126 this operation.
127 """
128 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
129 self.task.updateVariance(self.inputExp, self.amp)
130 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
131 self.assertGreater(statAfter[0], statBefore[0])
132 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
133 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
135 def test_darkCorrection(self):
136 """Expect the median image value should decrease after this operation.
137 """
138 darkIm = isrMock.DarkMock().run()
140 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
141 self.task.darkCorrection(self.inputExp, darkIm)
142 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
143 self.assertLess(statAfter[0], statBefore[0])
144 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
145 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
147 def test_darkCorrection_noVisitInfo(self):
148 """Expect the median image value should decrease after this operation.
149 """
150 darkIm = isrMock.DarkMock().run()
151 darkIm.getInfo().setVisitInfo(None)
153 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
154 self.task.darkCorrection(self.inputExp, darkIm)
155 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
156 self.assertLess(statAfter[0], statBefore[0])
157 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
158 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
160 def test_flatCorrection(self):
161 """Expect the image median should increase (divide by < 1).
162 """
163 flatIm = isrMock.FlatMock().run()
165 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
166 self.task.flatCorrection(self.inputExp, flatIm)
167 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
168 self.assertGreater(statAfter[1], statBefore[1])
169 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
170 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
172 def test_saturationDetection(self):
173 """Expect the saturation level detection/masking to scale with
174 threshold.
175 """
176 ampB = self.amp.rebuild()
177 ampB.setSaturation(9000.0)
178 self.task.saturationDetection(self.inputExp, ampB.finish())
179 countBefore = countMaskedPixels(self.mi, "SAT")
181 ampB.setSaturation(8250.0)
182 self.task.saturationDetection(self.inputExp, ampB.finish())
183 countAfter = countMaskedPixels(self.mi, "SAT")
185 self.assertLessEqual(countBefore, countAfter)
186 self.assertEqual(countBefore, 43)
187 self.assertEqual(countAfter, 136)
189 def test_measureBackground(self):
190 """Expect the background measurement runs successfully and to save
191 metadata values.
192 """
193 self.config.qa.flatness.meshX = 20
194 self.config.qa.flatness.meshY = 20
195 self.task.measureBackground(self.inputExp, self.config.qa)
196 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
198 def test_flatContext(self):
199 """Expect the flat context manager runs successfully (applying both
200 flat and dark within the context), and results in the same
201 image data after completion.
202 """
203 darkExp = isrMock.DarkMock().run()
204 flatExp = isrMock.FlatMock().run()
206 mi = self.inputExp.getMaskedImage().clone()
207 with self.task.flatContext(self.inputExp, flatExp, darkExp):
208 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
209 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
211 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
214class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
215 """Test IsrTask methods using untrimmed raw data.
216 """
217 def setUp(self):
218 self.config = IsrTaskConfig()
219 self.config.qa = IsrQaConfig()
220 self.task = IsrTask(config=self.config)
222 self.mockConfig = isrMock.IsrMockConfig()
223 self.mockConfig.isTrimmed = False
224 self.doGenerateImage = True
225 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig)
226 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
228 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
229 self.amp = self.inputExp.getDetector()[0]
230 self.mi = self.inputExp.getMaskedImage()
232 def batchSetConfiguration(self, value):
233 """Set the configuration state to a consistent value.
235 Disable options we do not need as well.
237 Parameters
238 ----------
239 value : `bool`
240 Value to switch common ISR configuration options to.
241 """
242 self.config.qa.flatness.meshX = 20
243 self.config.qa.flatness.meshY = 20
244 self.config.doWrite = False
245 self.config.doLinearize = False
246 self.config.doCrosstalk = False
248 self.config.doConvertIntToFloat = value
249 self.config.doSaturation = value
250 self.config.doSuspect = value
251 self.config.doSetBadRegions = value
252 self.config.doOverscan = value
253 self.config.doBias = value
254 self.config.doVariance = value
255 self.config.doWidenSaturationTrails = value
256 self.config.doBrighterFatter = value
257 self.config.doDefect = value
258 self.config.doSaturationInterpolation = value
259 self.config.doDark = value
260 self.config.doStrayLight = value
261 self.config.doFlat = value
262 self.config.doFringe = value
263 self.config.doMeasureBackground = value
264 self.config.doVignette = value
265 self.config.doAttachTransmissionCurve = value
266 self.config.doUseOpticsTransmission = value
267 self.config.doUseFilterTransmission = value
268 self.config.doUseSensorTransmission = value
269 self.config.doUseAtmosphereTransmission = value
270 self.config.qa.saveStats = value
271 self.config.qa.doThumbnailOss = value
272 self.config.qa.doThumbnailFlattened = value
274 self.config.doApplyGains = not value
275 self.config.doCameraSpecificMasking = value
277 def validateIsrResults(self):
278 """results should be a struct with components that are
279 not None if included in the configuration file.
281 Returns
282 -------
283 results : `pipeBase.Struct`
284 Results struct generated from the current ISR configuration.
285 """
286 self.task = IsrTask(config=self.config)
287 results = self.task.run(self.inputExp,
288 camera=self.camera,
289 bias=self.dataContainer.get("bias"),
290 dark=self.dataContainer.get("dark"),
291 flat=self.dataContainer.get("flat"),
292 bfKernel=self.dataContainer.get("bfKernel"),
293 defects=self.dataContainer.get("defects"),
294 fringes=Struct(fringes=self.dataContainer.get("fringe"), seed=1234),
295 opticsTransmission=self.dataContainer.get("transmission_"),
296 filterTransmission=self.dataContainer.get("transmission_"),
297 sensorTransmission=self.dataContainer.get("transmission_"),
298 atmosphereTransmission=self.dataContainer.get("transmission_")
299 )
301 self.assertIsInstance(results, Struct)
302 self.assertIsInstance(results.exposure, afwImage.Exposure)
303 return results
305 def test_overscanCorrection(self):
306 """Expect that this should reduce the image variance with a full fit.
307 The default fitType of MEDIAN will reduce the median value.
309 This needs to operate on a RawMock() to have overscan data to use.
311 The output types may be different when fitType != MEDIAN.
312 """
313 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
314 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
315 self.assertIsInstance(oscanResults, Struct)
316 self.assertIsInstance(oscanResults.imageFit, float)
317 self.assertIsInstance(oscanResults.overscanFit, float)
318 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
320 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
321 self.assertLess(statAfter[0], statBefore[0])
323 def test_overscanCorrectionMedianPerRow(self):
324 """Expect that this should reduce the image variance with a full fit.
325 fitType of MEDIAN_PER_ROW will reduce the median value.
327 This needs to operate on a RawMock() to have overscan data to use.
329 The output types may be different when fitType != MEDIAN_PER_ROW.
330 """
331 self.config.overscan.fitType = 'MEDIAN_PER_ROW'
332 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
333 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
334 self.assertIsInstance(oscanResults, Struct)
335 self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF)
336 self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF)
337 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
339 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
340 self.assertLess(statAfter[0], statBefore[0])
342 def test_run_allTrue(self):
343 """Expect successful run with expected outputs when all non-exclusive
344 configuration options are on.
346 Output results should be tested more precisely by the
347 individual function tests.
349 """
350 self.batchSetConfiguration(True)
351 self.validateIsrResults()
353 def test_run_allFalse(self):
354 """Expect successful run with expected outputs when all non-exclusive
355 configuration options are off.
357 Output results should be tested more precisely by the
358 individual function tests.
360 """
361 self.batchSetConfiguration(False)
362 self.validateIsrResults()
364 def test_failCases(self):
365 """Expect failure with crosstalk enabled.
367 Output results should be tested more precisely by the
368 individual function tests.
369 """
370 self.batchSetConfiguration(True)
372 # This breaks it
373 self.config.doCrosstalk = True
375 with self.assertRaises(RuntimeError):
376 self.validateIsrResults()
378 def test_maskingCase_negativeVariance(self):
379 """Test masking cases of configuration parameters.
380 """
381 self.batchSetConfiguration(True)
382 self.config.overscanFitType = "POLY"
383 self.config.overscanOrder = 1
385 self.config.doSaturation = False
386 self.config.doWidenSaturationTrails = False
387 self.config.doSaturationInterpolation = False
388 self.config.doSuspect = False
389 self.config.doSetBadRegions = False
390 self.config.doDefect = False
391 self.config.doBrighterFatter = False
393 self.config.maskNegativeVariance = True
394 self.config.doInterpolate = False
396 results = self.validateIsrResults()
398 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
399 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
400 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
401 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
403 def test_maskingCase_noMasking(self):
404 """Test masking cases of configuration parameters.
405 """
406 self.batchSetConfiguration(True)
407 self.config.overscanFitType = "POLY"
408 self.config.overscanOrder = 1
410 self.config.doSaturation = False
411 self.config.doWidenSaturationTrails = False
412 self.config.doSaturationInterpolation = False
413 self.config.doSuspect = False
414 self.config.doSetBadRegions = False
415 self.config.doDefect = False
416 self.config.doBrighterFatter = False
418 self.config.maskNegativeVariance = False
419 self.config.doInterpolate = False
421 results = self.validateIsrResults()
423 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
424 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
425 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
426 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
428 def test_maskingCase_satMasking(self):
429 """Test masking cases of configuration parameters.
430 """
431 self.batchSetConfiguration(True)
432 self.config.overscanFitType = "POLY"
433 self.config.overscanOrder = 1
435 self.config.saturation = 20000.0
436 self.config.doSaturation = True
437 self.config.doWidenSaturationTrails = True
439 self.config.doSaturationInterpolation = False
440 self.config.doSuspect = False
441 self.config.doSetBadRegions = False
442 self.config.doDefect = False
443 self.config.doBrighterFatter = False
445 self.config.maskNegativeVariance = False # These are mock images.
447 results = self.validateIsrResults()
449 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
450 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
451 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
452 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
454 def test_maskingCase_satMaskingAndInterp(self):
455 """Test masking cases of configuration parameters.
456 """
457 self.batchSetConfiguration(True)
458 self.config.overscanFitType = "POLY"
459 self.config.overscanOrder = 1
461 self.config.saturation = 20000.0
462 self.config.doSaturation = True
463 self.config.doWidenSaturationTrails = True
464 self.config.doSaturationInterpolation = True
466 self.config.doSuspect = False
467 self.config.doSetBadRegions = False
468 self.config.doDefect = False
469 self.config.doBrighterFatter = False
471 self.config.maskNegativeVariance = False # These are mock images.
473 results = self.validateIsrResults()
475 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
476 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
477 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
478 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
480 def test_maskingCase_throughEdge(self):
481 """Test masking cases of configuration parameters.
482 """
483 self.batchSetConfiguration(True)
484 self.config.overscanFitType = "POLY"
485 self.config.overscanOrder = 1
487 self.config.saturation = 20000.0
488 self.config.doSaturation = True
489 self.config.doWidenSaturationTrails = True
490 self.config.doSaturationInterpolation = True
491 self.config.numEdgeSuspect = 5
492 self.config.doSuspect = True
494 self.config.doSetBadRegions = False
495 self.config.doDefect = False
496 self.config.doBrighterFatter = False
498 self.config.maskNegativeVariance = False # These are mock images.
500 results = self.validateIsrResults()
502 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
503 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
504 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
505 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
507 def test_maskingCase_throughDefects(self):
508 """Test masking cases of configuration parameters.
509 """
510 self.batchSetConfiguration(True)
511 self.config.overscanFitType = "POLY"
512 self.config.overscanOrder = 1
514 self.config.saturation = 20000.0
515 self.config.doSaturation = True
516 self.config.doWidenSaturationTrails = True
517 self.config.doSaturationInterpolation = True
518 self.config.numEdgeSuspect = 5
519 self.config.doSuspect = True
520 self.config.doDefect = True
522 self.config.doSetBadRegions = False
523 self.config.doBrighterFatter = False
525 self.config.maskNegativeVariance = False # These are mock images.
527 results = self.validateIsrResults()
529 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
530 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
531 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
532 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
534 def test_maskingCase_throughDefectsAmpEdges(self):
535 """Test masking cases of configuration parameters.
536 """
537 self.batchSetConfiguration(True)
538 self.config.overscanFitType = "POLY"
539 self.config.overscanOrder = 1
541 self.config.saturation = 20000.0
542 self.config.doSaturation = True
543 self.config.doWidenSaturationTrails = True
544 self.config.doSaturationInterpolation = True
545 self.config.numEdgeSuspect = 5
546 self.config.doSuspect = True
547 self.config.doDefect = True
548 self.config.edgeMaskLevel = 'AMP'
550 self.config.doSetBadRegions = False
551 self.config.doBrighterFatter = False
553 self.config.maskNegativeVariance = False # These are mock images.
555 results = self.validateIsrResults()
557 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
558 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
559 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
560 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
562 def test_maskingCase_throughBad(self):
563 """Test masking cases of configuration parameters.
564 """
565 self.batchSetConfiguration(True)
566 self.config.overscanFitType = "POLY"
567 self.config.overscanOrder = 1
569 self.config.saturation = 20000.0
570 self.config.doSaturation = True
571 self.config.doWidenSaturationTrails = True
572 self.config.doSaturationInterpolation = True
574 self.config.doSuspect = True
575 self.config.doDefect = True
576 self.config.doSetBadRegions = True
577 self.config.doBrighterFatter = False
579 self.config.maskNegativeVariance = False # These are mock images.
581 results = self.validateIsrResults()
583 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
584 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
585 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
586 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
589class MemoryTester(lsst.utils.tests.MemoryTestCase):
590 pass
593def setup_module(module):
594 lsst.utils.tests.init()
597if __name__ == "__main__": 597 ↛ 598line 597 didn't jump to line 598, because the condition on line 597 was never true
598 lsst.utils.tests.init()
599 unittest.main()