Coverage for tests/test_isrTask.py: 12%
328 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-22 12:23 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-22 12:23 +0000
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.overscan.doParallelOverscan = True
81 self.config.qa = IsrQaConfig()
82 self.task = IsrTask(config=self.config)
83 self.camera = isrMock.IsrMock().getCamera()
85 self.inputExp = isrMock.TrimmedRawMock().run()
86 self.amp = self.inputExp.getDetector()[0]
87 self.mi = self.inputExp.getMaskedImage()
89 def validateIsrData(self, results):
90 """results should be a struct with components that are
91 not None if included in the configuration file.
92 """
93 self.assertIsInstance(results, Struct)
94 if self.config.doBias is True:
95 self.assertIsNotNone(results.bias)
96 if self.config.doDark is True:
97 self.assertIsNotNone(results.dark)
98 if self.config.doFlat is True:
99 self.assertIsNotNone(results.flat)
100 if self.config.doFringe is True:
101 self.assertIsNotNone(results.fringes)
102 if self.config.doDefect is True:
103 self.assertIsNotNone(results.defects)
104 if self.config.doBrighterFatter is True:
105 self.assertIsNotNone(results.bfKernel)
106 if self.config.doAttachTransmissionCurve is True:
107 self.assertIsNotNone(results.opticsTransmission)
108 self.assertIsNotNone(results.filterTransmission)
109 self.assertIsNotNone(results.sensorTransmission)
110 self.assertIsNotNone(results.atmosphereTransmission)
112 def test_ensureExposure(self):
113 """Test that an exposure has a usable instance class.
114 """
115 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
116 afwImage.Exposure)
118 def test_convertItoF(self):
119 """Test conversion from integer to floating point pixels.
120 """
121 result = self.task.convertIntToFloat(self.inputExp)
122 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
123 self.assertEqual(result, self.inputExp)
125 def test_updateVariance(self):
126 """Expect The variance image should have a larger median value after
127 this operation.
128 """
129 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
130 self.task.updateVariance(self.inputExp, self.amp)
131 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
132 self.assertGreater(statAfter[0], statBefore[0])
133 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
134 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
136 def test_darkCorrection(self):
137 """Expect the median image value should decrease after this operation.
138 """
139 darkIm = isrMock.DarkMock().run()
141 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
142 self.task.darkCorrection(self.inputExp, darkIm)
143 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
144 self.assertLess(statAfter[0], statBefore[0])
145 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
146 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
148 def test_darkCorrection_noVisitInfo(self):
149 """Expect the median image value should decrease after this operation.
150 """
151 darkIm = isrMock.DarkMock().run()
152 darkIm.getInfo().setVisitInfo(None)
154 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
155 self.task.darkCorrection(self.inputExp, darkIm)
156 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
157 self.assertLess(statAfter[0], statBefore[0])
158 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
159 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
161 def test_flatCorrection(self):
162 """Expect the image median should increase (divide by < 1).
163 """
164 flatIm = isrMock.FlatMock().run()
166 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
167 self.task.flatCorrection(self.inputExp, flatIm)
168 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
169 self.assertGreater(statAfter[1], statBefore[1])
170 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
171 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
173 def test_saturationDetection(self):
174 """Expect the saturation level detection/masking to scale with
175 threshold.
176 """
177 ampB = self.amp.rebuild()
178 ampB.setSaturation(9000.0)
179 self.task.saturationDetection(self.inputExp, ampB.finish())
180 countBefore = countMaskedPixels(self.mi, "SAT")
182 ampB.setSaturation(8250.0)
183 self.task.saturationDetection(self.inputExp, ampB.finish())
184 countAfter = countMaskedPixels(self.mi, "SAT")
186 self.assertLessEqual(countBefore, countAfter)
187 self.assertEqual(countBefore, 43)
188 self.assertEqual(countAfter, 136)
190 def test_measureBackground(self):
191 """Expect the background measurement runs successfully and to save
192 metadata values.
193 """
194 self.config.qa.flatness.meshX = 20
195 self.config.qa.flatness.meshY = 20
196 self.task.measureBackground(self.inputExp, self.config.qa)
197 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
199 def test_flatContext(self):
200 """Expect the flat context manager runs successfully (applying both
201 flat and dark within the context), and results in the same
202 image data after completion.
203 """
204 darkExp = isrMock.DarkMock().run()
205 flatExp = isrMock.FlatMock().run()
207 mi = self.inputExp.getMaskedImage().clone()
208 with self.task.flatContext(self.inputExp, flatExp, darkExp):
209 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
210 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
212 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
215class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
216 """Test IsrTask methods using untrimmed raw data.
217 """
218 def setUp(self):
219 self.config = IsrTaskConfig()
220 self.config.overscan.doParallelOverscan = True
221 self.config.qa = IsrQaConfig()
222 self.task = IsrTask(config=self.config)
224 self.mockConfig = isrMock.IsrMockConfig()
225 self.mockConfig.isTrimmed = False
226 self.doGenerateImage = True
227 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig)
228 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
230 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
231 self.amp = self.inputExp.getDetector()[0]
232 self.mi = self.inputExp.getMaskedImage()
234 def batchSetConfiguration(self, value):
235 """Set the configuration state to a consistent value.
237 Disable options we do not need as well.
239 Parameters
240 ----------
241 value : `bool`
242 Value to switch common ISR configuration options to.
243 """
244 self.config.qa.flatness.meshX = 20
245 self.config.qa.flatness.meshY = 20
246 self.config.doWrite = False
247 self.config.doLinearize = False
248 self.config.doCrosstalk = False
250 self.config.doConvertIntToFloat = value
251 self.config.doSaturation = value
252 self.config.doSuspect = value
253 self.config.doSetBadRegions = value
254 self.config.doOverscan = value
255 self.config.doBias = value
256 self.config.doVariance = value
257 self.config.doWidenSaturationTrails = value
258 self.config.doBrighterFatter = value
259 self.config.doDefect = value
260 self.config.doSaturationInterpolation = value
261 self.config.doDark = value
262 self.config.doStrayLight = value
263 self.config.doFlat = value
264 self.config.doFringe = value
265 self.config.doMeasureBackground = value
266 self.config.doVignette = value
267 self.config.doAttachTransmissionCurve = value
268 self.config.doUseOpticsTransmission = value
269 self.config.doUseFilterTransmission = value
270 self.config.doUseSensorTransmission = value
271 self.config.doUseAtmosphereTransmission = value
272 self.config.qa.saveStats = value
273 self.config.qa.doThumbnailOss = value
274 self.config.qa.doThumbnailFlattened = value
276 self.config.doApplyGains = not value
277 self.config.doCameraSpecificMasking = value
279 def validateIsrResults(self):
280 """results should be a struct with components that are
281 not None if included in the configuration file.
283 Returns
284 -------
285 results : `pipeBase.Struct`
286 Results struct generated from the current ISR configuration.
287 """
288 self.task = IsrTask(config=self.config)
289 results = self.task.run(self.inputExp,
290 camera=self.camera,
291 bias=self.dataContainer.get("bias"),
292 dark=self.dataContainer.get("dark"),
293 flat=self.dataContainer.get("flat"),
294 bfKernel=self.dataContainer.get("bfKernel"),
295 defects=self.dataContainer.get("defects"),
296 fringes=Struct(fringes=self.dataContainer.get("fringe"), seed=1234),
297 opticsTransmission=self.dataContainer.get("transmission_"),
298 filterTransmission=self.dataContainer.get("transmission_"),
299 sensorTransmission=self.dataContainer.get("transmission_"),
300 atmosphereTransmission=self.dataContainer.get("transmission_")
301 )
303 self.assertIsInstance(results, Struct)
304 self.assertIsInstance(results.exposure, afwImage.Exposure)
305 return results
307 def test_run_allTrue(self):
308 """Expect successful run with expected outputs when all non-exclusive
309 configuration options are on.
311 Output results should be tested more precisely by the
312 individual function tests.
314 """
315 self.batchSetConfiguration(True)
316 self.validateIsrResults()
318 def test_run_allFalse(self):
319 """Expect successful run with expected outputs when all non-exclusive
320 configuration options are off.
322 Output results should be tested more precisely by the
323 individual function tests.
325 """
326 self.batchSetConfiguration(False)
327 self.validateIsrResults()
329 def test_failCases(self):
330 """Expect failure with crosstalk enabled.
332 Output results should be tested more precisely by the
333 individual function tests.
334 """
335 self.batchSetConfiguration(True)
337 # This breaks it
338 self.config.doCrosstalk = True
340 with self.assertRaises(RuntimeError):
341 self.validateIsrResults()
343 def test_maskingCase_negativeVariance(self):
344 """Test masking cases of configuration parameters.
345 """
346 self.batchSetConfiguration(True)
347 self.config.overscan.doParallelOverscan = False
348 self.config.overscan.fitType = "POLY"
349 self.config.overscan.order = 1
351 self.config.doSaturation = False
352 self.config.doWidenSaturationTrails = False
353 self.config.doSaturationInterpolation = False
354 self.config.doSuspect = False
355 self.config.doSetBadRegions = False
356 self.config.doDefect = False
357 self.config.doBrighterFatter = False
359 self.config.maskNegativeVariance = True
360 self.config.doInterpolate = False
362 results = self.validateIsrResults()
364 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
365 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
366 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
367 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
369 def test_maskingCase_noMasking(self):
370 """Test masking cases of configuration parameters.
371 """
372 self.batchSetConfiguration(True)
373 self.config.overscan.fitType = "POLY"
374 self.config.overscan.order = 1
376 self.config.doSaturation = False
377 self.config.doWidenSaturationTrails = False
378 self.config.doSaturationInterpolation = False
379 self.config.doSuspect = False
380 self.config.doSetBadRegions = False
381 self.config.doDefect = False
382 self.config.doBrighterFatter = False
384 self.config.maskNegativeVariance = False
385 self.config.doInterpolate = False
387 results = self.validateIsrResults()
389 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
390 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
391 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
392 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
394 def test_maskingCase_satMasking(self):
395 """Test masking cases of configuration parameters.
396 """
397 self.batchSetConfiguration(True)
398 self.config.overscan.fitType = "POLY"
399 self.config.overscan.order = 1
401 self.config.saturation = 20000.0
402 self.config.doSaturation = True
403 self.config.doWidenSaturationTrails = True
405 self.config.doSaturationInterpolation = False
406 self.config.doSuspect = False
407 self.config.doSetBadRegions = False
408 self.config.doDefect = False
409 self.config.doBrighterFatter = False
411 self.config.maskNegativeVariance = False # These are mock images.
413 results = self.validateIsrResults()
415 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
416 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
417 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
418 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
420 def test_maskingCase_satMaskingAndInterp(self):
421 """Test masking cases of configuration parameters.
422 """
423 self.batchSetConfiguration(True)
424 self.config.overscan.fitType = "POLY"
425 self.config.overscan.order = 1
427 self.config.saturation = 20000.0
428 self.config.doSaturation = True
429 self.config.doWidenSaturationTrails = True
430 self.config.doSaturationInterpolation = True
432 self.config.doSuspect = False
433 self.config.doSetBadRegions = False
434 self.config.doDefect = False
435 self.config.doBrighterFatter = False
437 self.config.maskNegativeVariance = False # These are mock images.
439 results = self.validateIsrResults()
441 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
442 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
443 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
444 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
446 def test_maskingCase_throughEdge(self):
447 """Test masking cases of configuration parameters.
448 """
449 self.batchSetConfiguration(True)
450 self.config.overscan.fitType = "POLY"
451 self.config.overscan.order = 1
453 self.config.saturation = 20000.0
454 self.config.doSaturation = True
455 self.config.doWidenSaturationTrails = True
456 self.config.doSaturationInterpolation = True
457 self.config.numEdgeSuspect = 5
458 self.config.doSuspect = True
460 self.config.doSetBadRegions = False
461 self.config.doDefect = False
462 self.config.doBrighterFatter = False
464 self.config.maskNegativeVariance = False # These are mock images.
466 results = self.validateIsrResults()
468 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
469 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
470 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
471 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
473 def test_maskingCase_throughDefects(self):
474 """Test masking cases of configuration parameters.
475 """
476 self.batchSetConfiguration(True)
477 self.config.overscan.fitType = "POLY"
478 self.config.overscan.order = 1
480 self.config.saturation = 20000.0
481 self.config.doSaturation = True
482 self.config.doWidenSaturationTrails = True
483 self.config.doSaturationInterpolation = True
484 self.config.numEdgeSuspect = 5
485 self.config.doSuspect = True
486 self.config.doDefect = True
488 self.config.doSetBadRegions = False
489 self.config.doBrighterFatter = False
491 self.config.maskNegativeVariance = False # These are mock images.
493 results = self.validateIsrResults()
495 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
496 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
497 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
498 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
500 def test_maskingCase_throughDefectsAmpEdges(self):
501 """Test masking cases of configuration parameters.
502 """
503 self.batchSetConfiguration(True)
504 self.config.overscan.fitType = "POLY"
505 self.config.overscan.order = 1
507 self.config.saturation = 20000.0
508 self.config.doSaturation = True
509 self.config.doWidenSaturationTrails = True
510 self.config.doSaturationInterpolation = True
511 self.config.numEdgeSuspect = 5
512 self.config.doSuspect = True
513 self.config.doDefect = True
514 self.config.edgeMaskLevel = 'AMP'
516 self.config.doSetBadRegions = False
517 self.config.doBrighterFatter = False
519 self.config.maskNegativeVariance = False # These are mock images.
521 results = self.validateIsrResults()
523 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
524 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
525 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
526 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
528 def test_maskingCase_throughBad(self):
529 """Test masking cases of configuration parameters.
530 """
531 self.batchSetConfiguration(True)
532 self.config.overscan.fitType = "POLY"
533 self.config.overscan.order = 1
535 self.config.saturation = 20000.0
536 self.config.doSaturation = True
537 self.config.doWidenSaturationTrails = True
538 self.config.doSaturationInterpolation = True
540 self.config.doSuspect = True
541 self.config.doDefect = True
542 self.config.doSetBadRegions = True
543 self.config.doBrighterFatter = False
545 self.config.maskNegativeVariance = False # These are mock images.
547 results = self.validateIsrResults()
549 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
550 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
551 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
552 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
555class MemoryTester(lsst.utils.tests.MemoryTestCase):
556 pass
559def setup_module(module):
560 lsst.utils.tests.init()
563if __name__ == "__main__": 563 ↛ 564line 563 didn't jump to line 564, because the condition on line 563 was never true
564 lsst.utils.tests.init()
565 unittest.main(failfast=True)