Coverage for tests/test_isrTask.py: 14%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.qa = IsrQaConfig()
81 self.task = IsrTask(config=self.config)
82 self.dataRef = isrMock.DataRefMock()
83 self.camera = isrMock.IsrMock().getCamera()
85 self.inputExp = isrMock.TrimmedRawMock().run()
86 self.amp = self.inputExp.getDetector()[0]
87 self.mi = self.inputExp.getMaskedImage()
89 def validateIsrData(self, results):
90 """results should be a struct with components that are
91 not None if included in the configuration file.
92 """
93 self.assertIsInstance(results, Struct)
94 if self.config.doBias is True:
95 self.assertIsNotNone(results.bias)
96 if self.config.doDark is True:
97 self.assertIsNotNone(results.dark)
98 if self.config.doFlat is True:
99 self.assertIsNotNone(results.flat)
100 if self.config.doFringe is True:
101 self.assertIsNotNone(results.fringes)
102 if self.config.doDefect is True:
103 self.assertIsNotNone(results.defects)
104 if self.config.doBrighterFatter is True:
105 self.assertIsNotNone(results.bfKernel)
106 if self.config.doAttachTransmissionCurve is True:
107 self.assertIsNotNone(results.opticsTransmission)
108 self.assertIsNotNone(results.filterTransmission)
109 self.assertIsNotNone(results.sensorTransmission)
110 self.assertIsNotNone(results.atmosphereTransmission)
112 def test_readIsrData_noTrans(self):
113 """Test that all necessary calibration frames are retrieved.
114 """
115 self.config.doAttachTransmissionCurve = False
116 self.task = IsrTask(config=self.config)
117 results = self.task.readIsrData(self.dataRef, self.inputExp)
118 self.validateIsrData(results)
120 def test_readIsrData_withTrans(self):
121 """Test that all necessary calibration frames are retrieved.
122 """
123 self.config.doAttachTransmissionCurve = True
124 self.task = IsrTask(config=self.config)
125 results = self.task.readIsrData(self.dataRef, self.inputExp)
126 self.validateIsrData(results)
128 def test_ensureExposure(self):
129 """Test that an exposure has a usable instance class.
130 """
131 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
132 afwImage.Exposure)
134 def test_convertItoF(self):
135 """Test conversion from integer to floating point pixels.
136 """
137 result = self.task.convertIntToFloat(self.inputExp)
138 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
139 self.assertEqual(result, self.inputExp)
141 def test_updateVariance(self):
142 """Expect The variance image should have a larger median value after
143 this operation.
144 """
145 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
146 self.task.updateVariance(self.inputExp, self.amp)
147 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
148 self.assertGreater(statAfter[0], statBefore[0])
149 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
150 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
152 def test_darkCorrection(self):
153 """Expect the median image value should decrease after this operation.
154 """
155 darkIm = isrMock.DarkMock().run()
157 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
158 self.task.darkCorrection(self.inputExp, darkIm)
159 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
160 self.assertLess(statAfter[0], statBefore[0])
161 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
162 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
164 def test_darkCorrection_noVisitInfo(self):
165 """Expect the median image value should decrease after this operation.
166 """
167 darkIm = isrMock.DarkMock().run()
168 darkIm.getInfo().setVisitInfo(None)
170 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
171 self.task.darkCorrection(self.inputExp, darkIm)
172 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
173 self.assertLess(statAfter[0], statBefore[0])
174 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
175 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
177 def test_flatCorrection(self):
178 """Expect the image median should increase (divide by < 1).
179 """
180 flatIm = isrMock.FlatMock().run()
182 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
183 self.task.flatCorrection(self.inputExp, flatIm)
184 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
185 self.assertGreater(statAfter[1], statBefore[1])
186 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
187 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
189 def test_saturationDetection(self):
190 """Expect the saturation level detection/masking to scale with
191 threshold.
192 """
193 ampB = self.amp.rebuild()
194 ampB.setSaturation(9000.0)
195 self.task.saturationDetection(self.inputExp, ampB.finish())
196 countBefore = countMaskedPixels(self.mi, "SAT")
198 ampB.setSaturation(8250.0)
199 self.task.saturationDetection(self.inputExp, ampB.finish())
200 countAfter = countMaskedPixels(self.mi, "SAT")
202 self.assertLessEqual(countBefore, countAfter)
203 self.assertEqual(countBefore, 43)
204 self.assertEqual(countAfter, 136)
206 def test_measureBackground(self):
207 """Expect the background measurement runs successfully and to save
208 metadata values.
209 """
210 self.config.qa.flatness.meshX = 20
211 self.config.qa.flatness.meshY = 20
212 self.task.measureBackground(self.inputExp, self.config.qa)
213 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
215 def test_flatContext(self):
216 """Expect the flat context manager runs successfully (applying both
217 flat and dark within the context), and results in the same
218 image data after completion.
219 """
220 darkExp = isrMock.DarkMock().run()
221 flatExp = isrMock.FlatMock().run()
223 mi = self.inputExp.getMaskedImage().clone()
224 with self.task.flatContext(self.inputExp, flatExp, darkExp):
225 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
226 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
228 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
231class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
232 """Test IsrTask methods using untrimmed raw data.
233 """
234 def setUp(self):
235 self.config = IsrTaskConfig()
236 self.config.qa = IsrQaConfig()
237 self.task = IsrTask(config=self.config)
239 self.mockConfig = isrMock.IsrMockConfig()
240 self.mockConfig.isTrimmed = False
241 self.doGenerateImage = True
242 self.dataRef = isrMock.DataRefMock(config=self.mockConfig)
243 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
245 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
246 self.amp = self.inputExp.getDetector()[0]
247 self.mi = self.inputExp.getMaskedImage()
249 def batchSetConfiguration(self, value):
250 """Set the configuration state to a consistent value.
252 Disable options we do not need as well.
254 Parameters
255 ----------
256 value : `bool`
257 Value to switch common ISR configuration options to.
258 """
259 self.config.qa.flatness.meshX = 20
260 self.config.qa.flatness.meshY = 20
261 self.config.doWrite = False
262 self.config.doLinearize = False
263 self.config.doCrosstalk = False
265 self.config.doConvertIntToFloat = value
266 self.config.doSaturation = value
267 self.config.doSuspect = value
268 self.config.doSetBadRegions = value
269 self.config.doOverscan = value
270 self.config.doBias = value
271 self.config.doVariance = value
272 self.config.doWidenSaturationTrails = value
273 self.config.doBrighterFatter = value
274 self.config.doDefect = value
275 self.config.doSaturationInterpolation = value
276 self.config.doDark = value
277 self.config.doStrayLight = value
278 self.config.doFlat = value
279 self.config.doFringe = value
280 self.config.doMeasureBackground = value
281 self.config.doVignette = value
282 self.config.doAttachTransmissionCurve = value
283 self.config.doUseOpticsTransmission = value
284 self.config.doUseFilterTransmission = value
285 self.config.doUseSensorTransmission = value
286 self.config.doUseAtmosphereTransmission = value
287 self.config.qa.saveStats = value
288 self.config.qa.doThumbnailOss = value
289 self.config.qa.doThumbnailFlattened = value
291 self.config.doApplyGains = not value
292 self.config.doCameraSpecificMasking = value
293 self.config.vignette.doWriteVignettePolygon = value
295 def validateIsrResults(self):
296 """results should be a struct with components that are
297 not None if included in the configuration file.
299 Returns
300 -------
301 results : `pipeBase.Struct`
302 Results struct generated from the current ISR configuration.
303 """
304 self.task = IsrTask(config=self.config)
305 results = self.task.run(self.inputExp,
306 camera=self.camera,
307 bias=self.dataRef.get("bias"),
308 dark=self.dataRef.get("dark"),
309 flat=self.dataRef.get("flat"),
310 bfKernel=self.dataRef.get("bfKernel"),
311 defects=self.dataRef.get("defects"),
312 fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
313 opticsTransmission=self.dataRef.get("transmission_"),
314 filterTransmission=self.dataRef.get("transmission_"),
315 sensorTransmission=self.dataRef.get("transmission_"),
316 atmosphereTransmission=self.dataRef.get("transmission_")
317 )
319 self.assertIsInstance(results, Struct)
320 self.assertIsInstance(results.exposure, afwImage.Exposure)
321 return results
323 def test_overscanCorrection(self):
324 """Expect that this should reduce the image variance with a full fit.
325 The default fitType of MEDIAN will reduce the median value.
327 This needs to operate on a RawMock() to have overscan data to use.
329 The output types may be different when fitType != MEDIAN.
330 """
331 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
332 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
333 self.assertIsInstance(oscanResults, Struct)
334 self.assertIsInstance(oscanResults.imageFit, float)
335 self.assertIsInstance(oscanResults.overscanFit, float)
336 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
338 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
339 self.assertLess(statAfter[0], statBefore[0])
341 def test_overscanCorrectionMedianPerRow(self):
342 """Expect that this should reduce the image variance with a full fit.
343 fitType of MEDIAN_PER_ROW will reduce the median value.
345 This needs to operate on a RawMock() to have overscan data to use.
347 The output types may be different when fitType != MEDIAN_PER_ROW.
348 """
349 self.config.overscan.fitType = 'MEDIAN_PER_ROW'
350 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
351 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
352 self.assertIsInstance(oscanResults, Struct)
353 self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF)
354 self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF)
355 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
357 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
358 self.assertLess(statAfter[0], statBefore[0])
360 def test_runDataRef(self):
361 """Expect a dataRef to be handled correctly.
362 """
363 self.config.doLinearize = False
364 self.config.doWrite = False
365 self.task = IsrTask(config=self.config)
366 results = self.task.runDataRef(self.dataRef)
368 self.assertIsInstance(results, Struct)
369 self.assertIsInstance(results.exposure, afwImage.Exposure)
371 def test_run_allTrue(self):
372 """Expect successful run with expected outputs when all non-exclusive
373 configuration options are on.
375 Output results should be tested more precisely by the
376 individual function tests.
378 """
379 self.batchSetConfiguration(True)
380 self.validateIsrResults()
382 def test_run_allFalse(self):
383 """Expect successful run with expected outputs when all non-exclusive
384 configuration options are off.
386 Output results should be tested more precisely by the
387 individual function tests.
389 """
390 self.batchSetConfiguration(False)
391 self.validateIsrResults()
393 def test_failCases(self):
394 """Expect failure with crosstalk enabled.
396 Output results should be tested more precisely by the
397 individual function tests.
398 """
399 self.batchSetConfiguration(True)
401 # This breaks it
402 self.config.doCrosstalk = True
404 with self.assertRaises(RuntimeError):
405 self.validateIsrResults()
407 def test_maskingCase_negativeVariance(self):
408 """Test masking cases of configuration parameters.
409 """
410 self.batchSetConfiguration(True)
411 self.config.overscanFitType = "POLY"
412 self.config.overscanOrder = 1
414 self.config.doSaturation = False
415 self.config.doWidenSaturationTrails = False
416 self.config.doSaturationInterpolation = False
417 self.config.doSuspect = False
418 self.config.doSetBadRegions = False
419 self.config.doDefect = False
420 self.config.doBrighterFatter = False
422 self.config.maskNegativeVariance = True
423 self.config.doInterpolate = False
425 results = self.validateIsrResults()
427 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
428 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
429 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
430 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
432 def test_maskingCase_noMasking(self):
433 """Test masking cases of configuration parameters.
434 """
435 self.batchSetConfiguration(True)
436 self.config.overscanFitType = "POLY"
437 self.config.overscanOrder = 1
439 self.config.doSaturation = False
440 self.config.doWidenSaturationTrails = False
441 self.config.doSaturationInterpolation = False
442 self.config.doSuspect = False
443 self.config.doSetBadRegions = False
444 self.config.doDefect = False
445 self.config.doBrighterFatter = False
447 self.config.maskNegativeVariance = False
448 self.config.doInterpolate = False
450 results = self.validateIsrResults()
452 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
453 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
454 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
455 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
457 def test_maskingCase_satMasking(self):
458 """Test masking cases of configuration parameters.
459 """
460 self.batchSetConfiguration(True)
461 self.config.overscanFitType = "POLY"
462 self.config.overscanOrder = 1
464 self.config.saturation = 20000.0
465 self.config.doSaturation = True
466 self.config.doWidenSaturationTrails = True
468 self.config.doSaturationInterpolation = False
469 self.config.doSuspect = False
470 self.config.doSetBadRegions = False
471 self.config.doDefect = False
472 self.config.doBrighterFatter = False
474 self.config.maskNegativeVariance = False # These are mock images.
476 results = self.validateIsrResults()
478 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
479 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
480 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
481 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
483 def test_maskingCase_satMaskingAndInterp(self):
484 """Test masking cases of configuration parameters.
485 """
486 self.batchSetConfiguration(True)
487 self.config.overscanFitType = "POLY"
488 self.config.overscanOrder = 1
490 self.config.saturation = 20000.0
491 self.config.doSaturation = True
492 self.config.doWidenSaturationTrails = True
493 self.config.doSaturationInterpolation = True
495 self.config.doSuspect = False
496 self.config.doSetBadRegions = False
497 self.config.doDefect = False
498 self.config.doBrighterFatter = False
500 self.config.maskNegativeVariance = False # These are mock images.
502 results = self.validateIsrResults()
504 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
505 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
506 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
507 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
509 def test_maskingCase_throughEdge(self):
510 """Test masking cases of configuration parameters.
511 """
512 self.batchSetConfiguration(True)
513 self.config.overscanFitType = "POLY"
514 self.config.overscanOrder = 1
516 self.config.saturation = 20000.0
517 self.config.doSaturation = True
518 self.config.doWidenSaturationTrails = True
519 self.config.doSaturationInterpolation = True
520 self.config.numEdgeSuspect = 5
521 self.config.doSuspect = True
523 self.config.doSetBadRegions = False
524 self.config.doDefect = False
525 self.config.doBrighterFatter = False
527 self.config.maskNegativeVariance = False # These are mock images.
529 results = self.validateIsrResults()
531 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
532 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
533 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
534 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
536 def test_maskingCase_throughDefects(self):
537 """Test masking cases of configuration parameters.
538 """
539 self.batchSetConfiguration(True)
540 self.config.overscanFitType = "POLY"
541 self.config.overscanOrder = 1
543 self.config.saturation = 20000.0
544 self.config.doSaturation = True
545 self.config.doWidenSaturationTrails = True
546 self.config.doSaturationInterpolation = True
547 self.config.numEdgeSuspect = 5
548 self.config.doSuspect = True
549 self.config.doDefect = True
551 self.config.doSetBadRegions = False
552 self.config.doBrighterFatter = False
554 self.config.maskNegativeVariance = False # These are mock images.
556 results = self.validateIsrResults()
558 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
559 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
560 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
561 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
563 def test_maskingCase_throughDefectsAmpEdges(self):
564 """Test masking cases of configuration parameters.
565 """
566 self.batchSetConfiguration(True)
567 self.config.overscanFitType = "POLY"
568 self.config.overscanOrder = 1
570 self.config.saturation = 20000.0
571 self.config.doSaturation = True
572 self.config.doWidenSaturationTrails = True
573 self.config.doSaturationInterpolation = True
574 self.config.numEdgeSuspect = 5
575 self.config.doSuspect = True
576 self.config.doDefect = True
577 self.config.edgeMaskLevel = 'AMP'
579 self.config.doSetBadRegions = False
580 self.config.doBrighterFatter = False
582 self.config.maskNegativeVariance = False # These are mock images.
584 results = self.validateIsrResults()
586 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
587 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
588 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
589 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
591 def test_maskingCase_throughBad(self):
592 """Test masking cases of configuration parameters.
593 """
594 self.batchSetConfiguration(True)
595 self.config.overscanFitType = "POLY"
596 self.config.overscanOrder = 1
598 self.config.saturation = 20000.0
599 self.config.doSaturation = True
600 self.config.doWidenSaturationTrails = True
601 self.config.doSaturationInterpolation = True
603 self.config.doSuspect = True
604 self.config.doDefect = True
605 self.config.doSetBadRegions = True
606 self.config.doBrighterFatter = False
608 self.config.maskNegativeVariance = False # These are mock images.
610 results = self.validateIsrResults()
612 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
613 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
614 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
615 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
618class MemoryTester(lsst.utils.tests.MemoryTestCase):
619 pass
622def setup_module(module):
623 lsst.utils.tests.init()
626if __name__ == "__main__": 626 ↛ 627line 626 didn't jump to line 627, because the condition on line 626 was never true
627 lsst.utils.tests.init()
628 unittest.main()