Coverage for tests/test_isrTask.py: 12%
362 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-25 00:31 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-25 00:31 -0700
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.qa = IsrQaConfig()
81 self.task = IsrTask(config=self.config)
82 self.dataRef = isrMock.DataRefMock()
83 self.camera = isrMock.IsrMock().getCamera()
85 self.inputExp = isrMock.TrimmedRawMock().run()
86 self.amp = self.inputExp.getDetector()[0]
87 self.mi = self.inputExp.getMaskedImage()
89 def validateIsrData(self, results):
90 """results should be a struct with components that are
91 not None if included in the configuration file.
92 """
93 self.assertIsInstance(results, Struct)
94 if self.config.doBias is True:
95 self.assertIsNotNone(results.bias)
96 if self.config.doDark is True:
97 self.assertIsNotNone(results.dark)
98 if self.config.doFlat is True:
99 self.assertIsNotNone(results.flat)
100 if self.config.doFringe is True:
101 self.assertIsNotNone(results.fringes)
102 if self.config.doDefect is True:
103 self.assertIsNotNone(results.defects)
104 if self.config.doBrighterFatter is True:
105 self.assertIsNotNone(results.bfKernel)
106 if self.config.doAttachTransmissionCurve is True:
107 self.assertIsNotNone(results.opticsTransmission)
108 self.assertIsNotNone(results.filterTransmission)
109 self.assertIsNotNone(results.sensorTransmission)
110 self.assertIsNotNone(results.atmosphereTransmission)
112 def test_readIsrData_noTrans(self):
113 """Test that all necessary calibration frames are retrieved.
114 """
115 self.config.doAttachTransmissionCurve = False
116 self.task = IsrTask(config=self.config)
117 results = self.task.readIsrData(self.dataRef, self.inputExp)
118 self.validateIsrData(results)
120 def test_readIsrData_withTrans(self):
121 """Test that all necessary calibration frames are retrieved.
122 """
123 self.config.doAttachTransmissionCurve = True
124 self.task = IsrTask(config=self.config)
125 results = self.task.readIsrData(self.dataRef, self.inputExp)
126 self.validateIsrData(results)
128 def test_ensureExposure(self):
129 """Test that an exposure has a usable instance class.
130 """
131 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
132 afwImage.Exposure)
134 def test_convertItoF(self):
135 """Test conversion from integer to floating point pixels.
136 """
137 result = self.task.convertIntToFloat(self.inputExp)
138 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
139 self.assertEqual(result, self.inputExp)
141 def test_updateVariance(self):
142 """Expect The variance image should have a larger median value after
143 this operation.
144 """
145 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
146 self.task.updateVariance(self.inputExp, self.amp)
147 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
148 self.assertGreater(statAfter[0], statBefore[0])
149 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
150 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
152 def test_darkCorrection(self):
153 """Expect the median image value should decrease after this operation.
154 """
155 darkIm = isrMock.DarkMock().run()
157 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
158 self.task.darkCorrection(self.inputExp, darkIm)
159 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
160 self.assertLess(statAfter[0], statBefore[0])
161 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
162 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
164 def test_darkCorrection_noVisitInfo(self):
165 """Expect the median image value should decrease after this operation.
166 """
167 darkIm = isrMock.DarkMock().run()
168 darkIm.getInfo().setVisitInfo(None)
170 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
171 self.task.darkCorrection(self.inputExp, darkIm)
172 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
173 self.assertLess(statAfter[0], statBefore[0])
174 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
175 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
177 def test_flatCorrection(self):
178 """Expect the image median should increase (divide by < 1).
179 """
180 flatIm = isrMock.FlatMock().run()
182 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
183 self.task.flatCorrection(self.inputExp, flatIm)
184 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
185 self.assertGreater(statAfter[1], statBefore[1])
186 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
187 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
189 def test_saturationDetection(self):
190 """Expect the saturation level detection/masking to scale with
191 threshold.
192 """
193 ampB = self.amp.rebuild()
194 ampB.setSaturation(9000.0)
195 self.task.saturationDetection(self.inputExp, ampB.finish())
196 countBefore = countMaskedPixels(self.mi, "SAT")
198 ampB.setSaturation(8250.0)
199 self.task.saturationDetection(self.inputExp, ampB.finish())
200 countAfter = countMaskedPixels(self.mi, "SAT")
202 self.assertLessEqual(countBefore, countAfter)
203 self.assertEqual(countBefore, 43)
204 self.assertEqual(countAfter, 136)
206 def test_measureBackground(self):
207 """Expect the background measurement runs successfully and to save
208 metadata values.
209 """
210 self.config.qa.flatness.meshX = 20
211 self.config.qa.flatness.meshY = 20
212 self.task.measureBackground(self.inputExp, self.config.qa)
213 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
215 def test_flatContext(self):
216 """Expect the flat context manager runs successfully (applying both
217 flat and dark within the context), and results in the same
218 image data after completion.
219 """
220 darkExp = isrMock.DarkMock().run()
221 flatExp = isrMock.FlatMock().run()
223 mi = self.inputExp.getMaskedImage().clone()
224 with self.task.flatContext(self.inputExp, flatExp, darkExp):
225 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
226 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
228 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
231class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
232 """Test IsrTask methods using untrimmed raw data.
233 """
234 def setUp(self):
235 self.config = IsrTaskConfig()
236 self.config.qa = IsrQaConfig()
237 self.task = IsrTask(config=self.config)
239 self.mockConfig = isrMock.IsrMockConfig()
240 self.mockConfig.isTrimmed = False
241 self.doGenerateImage = True
242 self.dataRef = isrMock.DataRefMock(config=self.mockConfig)
243 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
245 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
246 self.amp = self.inputExp.getDetector()[0]
247 self.mi = self.inputExp.getMaskedImage()
249 def batchSetConfiguration(self, value):
250 """Set the configuration state to a consistent value.
252 Disable options we do not need as well.
254 Parameters
255 ----------
256 value : `bool`
257 Value to switch common ISR configuration options to.
258 """
259 self.config.qa.flatness.meshX = 20
260 self.config.qa.flatness.meshY = 20
261 self.config.doWrite = False
262 self.config.doLinearize = False
263 self.config.doCrosstalk = False
265 self.config.doConvertIntToFloat = value
266 self.config.doSaturation = value
267 self.config.doSuspect = value
268 self.config.doSetBadRegions = value
269 self.config.doOverscan = value
270 self.config.doBias = value
271 self.config.doVariance = value
272 self.config.doWidenSaturationTrails = value
273 self.config.doBrighterFatter = value
274 self.config.doDefect = value
275 self.config.doSaturationInterpolation = value
276 self.config.doDark = value
277 self.config.doStrayLight = value
278 self.config.doFlat = value
279 self.config.doFringe = value
280 self.config.doMeasureBackground = value
281 self.config.doVignette = value
282 self.config.doAttachTransmissionCurve = value
283 self.config.doUseOpticsTransmission = value
284 self.config.doUseFilterTransmission = value
285 self.config.doUseSensorTransmission = value
286 self.config.doUseAtmosphereTransmission = value
287 self.config.qa.saveStats = value
288 self.config.qa.doThumbnailOss = value
289 self.config.qa.doThumbnailFlattened = value
291 self.config.doApplyGains = not value
292 self.config.doCameraSpecificMasking = value
294 def validateIsrResults(self):
295 """results should be a struct with components that are
296 not None if included in the configuration file.
298 Returns
299 -------
300 results : `pipeBase.Struct`
301 Results struct generated from the current ISR configuration.
302 """
303 self.task = IsrTask(config=self.config)
304 results = self.task.run(self.inputExp,
305 camera=self.camera,
306 bias=self.dataRef.get("bias"),
307 dark=self.dataRef.get("dark"),
308 flat=self.dataRef.get("flat"),
309 bfKernel=self.dataRef.get("bfKernel"),
310 defects=self.dataRef.get("defects"),
311 fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
312 opticsTransmission=self.dataRef.get("transmission_"),
313 filterTransmission=self.dataRef.get("transmission_"),
314 sensorTransmission=self.dataRef.get("transmission_"),
315 atmosphereTransmission=self.dataRef.get("transmission_")
316 )
318 self.assertIsInstance(results, Struct)
319 self.assertIsInstance(results.exposure, afwImage.Exposure)
320 return results
322 def test_overscanCorrection(self):
323 """Expect that this should reduce the image variance with a full fit.
324 The default fitType of MEDIAN will reduce the median value.
326 This needs to operate on a RawMock() to have overscan data to use.
328 The output types may be different when fitType != MEDIAN.
329 """
330 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
331 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
332 self.assertIsInstance(oscanResults, Struct)
333 self.assertIsInstance(oscanResults.imageFit, float)
334 self.assertIsInstance(oscanResults.overscanFit, float)
335 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
337 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
338 self.assertLess(statAfter[0], statBefore[0])
340 def test_overscanCorrectionMedianPerRow(self):
341 """Expect that this should reduce the image variance with a full fit.
342 fitType of MEDIAN_PER_ROW will reduce the median value.
344 This needs to operate on a RawMock() to have overscan data to use.
346 The output types may be different when fitType != MEDIAN_PER_ROW.
347 """
348 self.config.overscan.fitType = 'MEDIAN_PER_ROW'
349 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
350 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
351 self.assertIsInstance(oscanResults, Struct)
352 self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF)
353 self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF)
354 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
356 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
357 self.assertLess(statAfter[0], statBefore[0])
359 def test_runDataRef(self):
360 """Expect a dataRef to be handled correctly.
361 """
362 self.config.doLinearize = False
363 self.config.doWrite = False
364 self.task = IsrTask(config=self.config)
365 results = self.task.runDataRef(self.dataRef)
367 self.assertIsInstance(results, Struct)
368 self.assertIsInstance(results.exposure, afwImage.Exposure)
370 def test_run_allTrue(self):
371 """Expect successful run with expected outputs when all non-exclusive
372 configuration options are on.
374 Output results should be tested more precisely by the
375 individual function tests.
377 """
378 self.batchSetConfiguration(True)
379 self.validateIsrResults()
381 def test_run_allFalse(self):
382 """Expect successful run with expected outputs when all non-exclusive
383 configuration options are off.
385 Output results should be tested more precisely by the
386 individual function tests.
388 """
389 self.batchSetConfiguration(False)
390 self.validateIsrResults()
392 def test_failCases(self):
393 """Expect failure with crosstalk enabled.
395 Output results should be tested more precisely by the
396 individual function tests.
397 """
398 self.batchSetConfiguration(True)
400 # This breaks it
401 self.config.doCrosstalk = True
403 with self.assertRaises(RuntimeError):
404 self.validateIsrResults()
406 def test_maskingCase_negativeVariance(self):
407 """Test masking cases of configuration parameters.
408 """
409 self.batchSetConfiguration(True)
410 self.config.overscanFitType = "POLY"
411 self.config.overscanOrder = 1
413 self.config.doSaturation = False
414 self.config.doWidenSaturationTrails = False
415 self.config.doSaturationInterpolation = False
416 self.config.doSuspect = False
417 self.config.doSetBadRegions = False
418 self.config.doDefect = False
419 self.config.doBrighterFatter = False
421 self.config.maskNegativeVariance = True
422 self.config.doInterpolate = False
424 results = self.validateIsrResults()
426 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
427 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
428 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
429 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
431 def test_maskingCase_noMasking(self):
432 """Test masking cases of configuration parameters.
433 """
434 self.batchSetConfiguration(True)
435 self.config.overscanFitType = "POLY"
436 self.config.overscanOrder = 1
438 self.config.doSaturation = False
439 self.config.doWidenSaturationTrails = False
440 self.config.doSaturationInterpolation = False
441 self.config.doSuspect = False
442 self.config.doSetBadRegions = False
443 self.config.doDefect = False
444 self.config.doBrighterFatter = False
446 self.config.maskNegativeVariance = False
447 self.config.doInterpolate = False
449 results = self.validateIsrResults()
451 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
452 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
453 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
454 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
456 def test_maskingCase_satMasking(self):
457 """Test masking cases of configuration parameters.
458 """
459 self.batchSetConfiguration(True)
460 self.config.overscanFitType = "POLY"
461 self.config.overscanOrder = 1
463 self.config.saturation = 20000.0
464 self.config.doSaturation = True
465 self.config.doWidenSaturationTrails = True
467 self.config.doSaturationInterpolation = False
468 self.config.doSuspect = False
469 self.config.doSetBadRegions = False
470 self.config.doDefect = False
471 self.config.doBrighterFatter = False
473 self.config.maskNegativeVariance = False # These are mock images.
475 results = self.validateIsrResults()
477 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
478 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
479 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
480 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
482 def test_maskingCase_satMaskingAndInterp(self):
483 """Test masking cases of configuration parameters.
484 """
485 self.batchSetConfiguration(True)
486 self.config.overscanFitType = "POLY"
487 self.config.overscanOrder = 1
489 self.config.saturation = 20000.0
490 self.config.doSaturation = True
491 self.config.doWidenSaturationTrails = True
492 self.config.doSaturationInterpolation = True
494 self.config.doSuspect = False
495 self.config.doSetBadRegions = False
496 self.config.doDefect = False
497 self.config.doBrighterFatter = False
499 self.config.maskNegativeVariance = False # These are mock images.
501 results = self.validateIsrResults()
503 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
504 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
505 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
506 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
508 def test_maskingCase_throughEdge(self):
509 """Test masking cases of configuration parameters.
510 """
511 self.batchSetConfiguration(True)
512 self.config.overscanFitType = "POLY"
513 self.config.overscanOrder = 1
515 self.config.saturation = 20000.0
516 self.config.doSaturation = True
517 self.config.doWidenSaturationTrails = True
518 self.config.doSaturationInterpolation = True
519 self.config.numEdgeSuspect = 5
520 self.config.doSuspect = True
522 self.config.doSetBadRegions = False
523 self.config.doDefect = False
524 self.config.doBrighterFatter = False
526 self.config.maskNegativeVariance = False # These are mock images.
528 results = self.validateIsrResults()
530 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
531 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
532 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
533 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
535 def test_maskingCase_throughDefects(self):
536 """Test masking cases of configuration parameters.
537 """
538 self.batchSetConfiguration(True)
539 self.config.overscanFitType = "POLY"
540 self.config.overscanOrder = 1
542 self.config.saturation = 20000.0
543 self.config.doSaturation = True
544 self.config.doWidenSaturationTrails = True
545 self.config.doSaturationInterpolation = True
546 self.config.numEdgeSuspect = 5
547 self.config.doSuspect = True
548 self.config.doDefect = True
550 self.config.doSetBadRegions = False
551 self.config.doBrighterFatter = False
553 self.config.maskNegativeVariance = False # These are mock images.
555 results = self.validateIsrResults()
557 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
558 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
559 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
560 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
562 def test_maskingCase_throughDefectsAmpEdges(self):
563 """Test masking cases of configuration parameters.
564 """
565 self.batchSetConfiguration(True)
566 self.config.overscanFitType = "POLY"
567 self.config.overscanOrder = 1
569 self.config.saturation = 20000.0
570 self.config.doSaturation = True
571 self.config.doWidenSaturationTrails = True
572 self.config.doSaturationInterpolation = True
573 self.config.numEdgeSuspect = 5
574 self.config.doSuspect = True
575 self.config.doDefect = True
576 self.config.edgeMaskLevel = 'AMP'
578 self.config.doSetBadRegions = False
579 self.config.doBrighterFatter = False
581 self.config.maskNegativeVariance = False # These are mock images.
583 results = self.validateIsrResults()
585 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
586 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
587 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
588 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
590 def test_maskingCase_throughBad(self):
591 """Test masking cases of configuration parameters.
592 """
593 self.batchSetConfiguration(True)
594 self.config.overscanFitType = "POLY"
595 self.config.overscanOrder = 1
597 self.config.saturation = 20000.0
598 self.config.doSaturation = True
599 self.config.doWidenSaturationTrails = True
600 self.config.doSaturationInterpolation = True
602 self.config.doSuspect = True
603 self.config.doDefect = True
604 self.config.doSetBadRegions = True
605 self.config.doBrighterFatter = False
607 self.config.maskNegativeVariance = False # These are mock images.
609 results = self.validateIsrResults()
611 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
612 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
613 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
614 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
617class MemoryTester(lsst.utils.tests.MemoryTestCase):
618 pass
621def setup_module(module):
622 lsst.utils.tests.init()
625if __name__ == "__main__": 625 ↛ 626line 625 didn't jump to line 626, because the condition on line 625 was never true
626 lsst.utils.tests.init()
627 unittest.main()