Coverage for tests/test_isrTask.py: 14%
325 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-16 01:34 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-16 01:34 -0700
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.qa = IsrQaConfig()
81 self.task = IsrTask(config=self.config)
82 self.camera = isrMock.IsrMock().getCamera()
84 self.inputExp = isrMock.TrimmedRawMock().run()
85 self.amp = self.inputExp.getDetector()[0]
86 self.mi = self.inputExp.getMaskedImage()
88 def validateIsrData(self, results):
89 """results should be a struct with components that are
90 not None if included in the configuration file.
91 """
92 self.assertIsInstance(results, Struct)
93 if self.config.doBias is True:
94 self.assertIsNotNone(results.bias)
95 if self.config.doDark is True:
96 self.assertIsNotNone(results.dark)
97 if self.config.doFlat is True:
98 self.assertIsNotNone(results.flat)
99 if self.config.doFringe is True:
100 self.assertIsNotNone(results.fringes)
101 if self.config.doDefect is True:
102 self.assertIsNotNone(results.defects)
103 if self.config.doBrighterFatter is True:
104 self.assertIsNotNone(results.bfKernel)
105 if self.config.doAttachTransmissionCurve is True:
106 self.assertIsNotNone(results.opticsTransmission)
107 self.assertIsNotNone(results.filterTransmission)
108 self.assertIsNotNone(results.sensorTransmission)
109 self.assertIsNotNone(results.atmosphereTransmission)
111 def test_ensureExposure(self):
112 """Test that an exposure has a usable instance class.
113 """
114 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
115 afwImage.Exposure)
117 def test_convertItoF(self):
118 """Test conversion from integer to floating point pixels.
119 """
120 result = self.task.convertIntToFloat(self.inputExp)
121 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
122 self.assertEqual(result, self.inputExp)
124 def test_updateVariance(self):
125 """Expect The variance image should have a larger median value after
126 this operation.
127 """
128 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
129 self.task.updateVariance(self.inputExp, self.amp)
130 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
131 self.assertGreater(statAfter[0], statBefore[0])
132 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
133 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
135 def test_darkCorrection(self):
136 """Expect the median image value should decrease after this operation.
137 """
138 darkIm = isrMock.DarkMock().run()
140 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
141 self.task.darkCorrection(self.inputExp, darkIm)
142 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
143 self.assertLess(statAfter[0], statBefore[0])
144 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
145 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
147 def test_darkCorrection_noVisitInfo(self):
148 """Expect the median image value should decrease after this operation.
149 """
150 darkIm = isrMock.DarkMock().run()
151 darkIm.getInfo().setVisitInfo(None)
153 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
154 self.task.darkCorrection(self.inputExp, darkIm)
155 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
156 self.assertLess(statAfter[0], statBefore[0])
157 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
158 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
160 def test_flatCorrection(self):
161 """Expect the image median should increase (divide by < 1).
162 """
163 flatIm = isrMock.FlatMock().run()
165 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
166 self.task.flatCorrection(self.inputExp, flatIm)
167 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
168 self.assertGreater(statAfter[1], statBefore[1])
169 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
170 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
172 def test_saturationDetection(self):
173 """Expect the saturation level detection/masking to scale with
174 threshold.
175 """
176 ampB = self.amp.rebuild()
177 ampB.setSaturation(9000.0)
178 self.task.saturationDetection(self.inputExp, ampB.finish())
179 countBefore = countMaskedPixels(self.mi, "SAT")
181 ampB.setSaturation(8250.0)
182 self.task.saturationDetection(self.inputExp, ampB.finish())
183 countAfter = countMaskedPixels(self.mi, "SAT")
185 self.assertLessEqual(countBefore, countAfter)
186 self.assertEqual(countBefore, 43)
187 self.assertEqual(countAfter, 136)
189 def test_measureBackground(self):
190 """Expect the background measurement runs successfully and to save
191 metadata values.
192 """
193 self.config.qa.flatness.meshX = 20
194 self.config.qa.flatness.meshY = 20
195 self.task.measureBackground(self.inputExp, self.config.qa)
196 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
198 def test_flatContext(self):
199 """Expect the flat context manager runs successfully (applying both
200 flat and dark within the context), and results in the same
201 image data after completion.
202 """
203 darkExp = isrMock.DarkMock().run()
204 flatExp = isrMock.FlatMock().run()
206 mi = self.inputExp.getMaskedImage().clone()
207 with self.task.flatContext(self.inputExp, flatExp, darkExp):
208 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
209 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
211 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
214class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
215 """Test IsrTask methods using untrimmed raw data.
216 """
217 def setUp(self):
218 self.config = IsrTaskConfig()
219 self.config.qa = IsrQaConfig()
220 self.task = IsrTask(config=self.config)
222 self.mockConfig = isrMock.IsrMockConfig()
223 self.mockConfig.isTrimmed = False
224 self.doGenerateImage = True
225 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig)
226 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
228 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
229 self.amp = self.inputExp.getDetector()[0]
230 self.mi = self.inputExp.getMaskedImage()
232 def batchSetConfiguration(self, value):
233 """Set the configuration state to a consistent value.
235 Disable options we do not need as well.
237 Parameters
238 ----------
239 value : `bool`
240 Value to switch common ISR configuration options to.
241 """
242 self.config.qa.flatness.meshX = 20
243 self.config.qa.flatness.meshY = 20
244 self.config.doWrite = False
245 self.config.doLinearize = False
246 self.config.doCrosstalk = False
248 self.config.doConvertIntToFloat = value
249 self.config.doSaturation = value
250 self.config.doSuspect = value
251 self.config.doSetBadRegions = value
252 self.config.doOverscan = value
253 self.config.doBias = value
254 self.config.doVariance = value
255 self.config.doWidenSaturationTrails = value
256 self.config.doBrighterFatter = value
257 self.config.doDefect = value
258 self.config.doSaturationInterpolation = value
259 self.config.doDark = value
260 self.config.doStrayLight = value
261 self.config.doFlat = value
262 self.config.doFringe = value
263 self.config.doMeasureBackground = value
264 self.config.doVignette = value
265 self.config.doAttachTransmissionCurve = value
266 self.config.doUseOpticsTransmission = value
267 self.config.doUseFilterTransmission = value
268 self.config.doUseSensorTransmission = value
269 self.config.doUseAtmosphereTransmission = value
270 self.config.qa.saveStats = value
271 self.config.qa.doThumbnailOss = value
272 self.config.qa.doThumbnailFlattened = value
274 self.config.doApplyGains = not value
275 self.config.doCameraSpecificMasking = value
277 def validateIsrResults(self):
278 """results should be a struct with components that are
279 not None if included in the configuration file.
281 Returns
282 -------
283 results : `pipeBase.Struct`
284 Results struct generated from the current ISR configuration.
285 """
286 self.task = IsrTask(config=self.config)
287 results = self.task.run(self.inputExp,
288 camera=self.camera,
289 bias=self.dataContainer.get("bias"),
290 dark=self.dataContainer.get("dark"),
291 flat=self.dataContainer.get("flat"),
292 bfKernel=self.dataContainer.get("bfKernel"),
293 defects=self.dataContainer.get("defects"),
294 fringes=Struct(fringes=self.dataContainer.get("fringe"), seed=1234),
295 opticsTransmission=self.dataContainer.get("transmission_"),
296 filterTransmission=self.dataContainer.get("transmission_"),
297 sensorTransmission=self.dataContainer.get("transmission_"),
298 atmosphereTransmission=self.dataContainer.get("transmission_")
299 )
301 self.assertIsInstance(results, Struct)
302 self.assertIsInstance(results.exposure, afwImage.Exposure)
303 return results
305 def test_run_allTrue(self):
306 """Expect successful run with expected outputs when all non-exclusive
307 configuration options are on.
309 Output results should be tested more precisely by the
310 individual function tests.
312 """
313 self.batchSetConfiguration(True)
314 self.validateIsrResults()
316 def test_run_allFalse(self):
317 """Expect successful run with expected outputs when all non-exclusive
318 configuration options are off.
320 Output results should be tested more precisely by the
321 individual function tests.
323 """
324 self.batchSetConfiguration(False)
325 self.validateIsrResults()
327 def test_failCases(self):
328 """Expect failure with crosstalk enabled.
330 Output results should be tested more precisely by the
331 individual function tests.
332 """
333 self.batchSetConfiguration(True)
335 # This breaks it
336 self.config.doCrosstalk = True
338 with self.assertRaises(RuntimeError):
339 self.validateIsrResults()
341 def test_maskingCase_negativeVariance(self):
342 """Test masking cases of configuration parameters.
343 """
344 self.batchSetConfiguration(True)
345 self.config.overscan.fitType = "POLY"
346 self.config.overscan.order = 1
348 self.config.doSaturation = False
349 self.config.doWidenSaturationTrails = False
350 self.config.doSaturationInterpolation = False
351 self.config.doSuspect = False
352 self.config.doSetBadRegions = False
353 self.config.doDefect = False
354 self.config.doBrighterFatter = False
356 self.config.maskNegativeVariance = True
357 self.config.doInterpolate = False
359 results = self.validateIsrResults()
361 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
362 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
363 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
364 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
366 def test_maskingCase_noMasking(self):
367 """Test masking cases of configuration parameters.
368 """
369 self.batchSetConfiguration(True)
370 self.config.overscan.fitType = "POLY"
371 self.config.overscan.order = 1
373 self.config.doSaturation = False
374 self.config.doWidenSaturationTrails = False
375 self.config.doSaturationInterpolation = False
376 self.config.doSuspect = False
377 self.config.doSetBadRegions = False
378 self.config.doDefect = False
379 self.config.doBrighterFatter = False
381 self.config.maskNegativeVariance = False
382 self.config.doInterpolate = False
384 results = self.validateIsrResults()
386 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
387 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
388 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
389 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
391 def test_maskingCase_satMasking(self):
392 """Test masking cases of configuration parameters.
393 """
394 self.batchSetConfiguration(True)
395 self.config.overscan.fitType = "POLY"
396 self.config.overscan.order = 1
398 self.config.saturation = 20000.0
399 self.config.doSaturation = True
400 self.config.doWidenSaturationTrails = True
402 self.config.doSaturationInterpolation = False
403 self.config.doSuspect = False
404 self.config.doSetBadRegions = False
405 self.config.doDefect = False
406 self.config.doBrighterFatter = False
408 self.config.maskNegativeVariance = False # These are mock images.
410 results = self.validateIsrResults()
412 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
413 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
414 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
415 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
417 def test_maskingCase_satMaskingAndInterp(self):
418 """Test masking cases of configuration parameters.
419 """
420 self.batchSetConfiguration(True)
421 self.config.overscan.fitType = "POLY"
422 self.config.overscan.order = 1
424 self.config.saturation = 20000.0
425 self.config.doSaturation = True
426 self.config.doWidenSaturationTrails = True
427 self.config.doSaturationInterpolation = True
429 self.config.doSuspect = False
430 self.config.doSetBadRegions = False
431 self.config.doDefect = False
432 self.config.doBrighterFatter = False
434 self.config.maskNegativeVariance = False # These are mock images.
436 results = self.validateIsrResults()
438 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
439 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
440 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
441 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
443 def test_maskingCase_throughEdge(self):
444 """Test masking cases of configuration parameters.
445 """
446 self.batchSetConfiguration(True)
447 self.config.overscan.fitType = "POLY"
448 self.config.overscan.order = 1
450 self.config.saturation = 20000.0
451 self.config.doSaturation = True
452 self.config.doWidenSaturationTrails = True
453 self.config.doSaturationInterpolation = True
454 self.config.numEdgeSuspect = 5
455 self.config.doSuspect = True
457 self.config.doSetBadRegions = False
458 self.config.doDefect = False
459 self.config.doBrighterFatter = False
461 self.config.maskNegativeVariance = False # These are mock images.
463 results = self.validateIsrResults()
465 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
466 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
467 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
468 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
470 def test_maskingCase_throughDefects(self):
471 """Test masking cases of configuration parameters.
472 """
473 self.batchSetConfiguration(True)
474 self.config.overscan.fitType = "POLY"
475 self.config.overscan.order = 1
477 self.config.saturation = 20000.0
478 self.config.doSaturation = True
479 self.config.doWidenSaturationTrails = True
480 self.config.doSaturationInterpolation = True
481 self.config.numEdgeSuspect = 5
482 self.config.doSuspect = True
483 self.config.doDefect = True
485 self.config.doSetBadRegions = False
486 self.config.doBrighterFatter = False
488 self.config.maskNegativeVariance = False # These are mock images.
490 results = self.validateIsrResults()
492 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
493 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
494 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
495 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
497 def test_maskingCase_throughDefectsAmpEdges(self):
498 """Test masking cases of configuration parameters.
499 """
500 self.batchSetConfiguration(True)
501 self.config.overscan.fitType = "POLY"
502 self.config.overscan.order = 1
504 self.config.saturation = 20000.0
505 self.config.doSaturation = True
506 self.config.doWidenSaturationTrails = True
507 self.config.doSaturationInterpolation = True
508 self.config.numEdgeSuspect = 5
509 self.config.doSuspect = True
510 self.config.doDefect = True
511 self.config.edgeMaskLevel = 'AMP'
513 self.config.doSetBadRegions = False
514 self.config.doBrighterFatter = False
516 self.config.maskNegativeVariance = False # These are mock images.
518 results = self.validateIsrResults()
520 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
521 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
522 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
523 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
525 def test_maskingCase_throughBad(self):
526 """Test masking cases of configuration parameters.
527 """
528 self.batchSetConfiguration(True)
529 self.config.overscan.fitType = "POLY"
530 self.config.overscan.order = 1
532 self.config.saturation = 20000.0
533 self.config.doSaturation = True
534 self.config.doWidenSaturationTrails = True
535 self.config.doSaturationInterpolation = True
537 self.config.doSuspect = True
538 self.config.doDefect = True
539 self.config.doSetBadRegions = True
540 self.config.doBrighterFatter = False
542 self.config.maskNegativeVariance = False # These are mock images.
544 results = self.validateIsrResults()
546 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
547 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
548 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
549 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
552class MemoryTester(lsst.utils.tests.MemoryTestCase):
553 pass
556def setup_module(module):
557 lsst.utils.tests.init()
560if __name__ == "__main__": 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true
561 lsst.utils.tests.init()
562 unittest.main(failfast=True)