Coverage for tests/test_isrTask.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23import unittest
24import numpy as np
26import lsst.afw.image as afwImage
27import lsst.ip.isr.isrMock as isrMock
28import lsst.utils.tests
29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig)
30from lsst.ip.isr.isrQa import IsrQaConfig
31from lsst.pipe.base import Struct
34def countMaskedPixels(maskedImage, maskPlane):
35 """Function to count the number of masked pixels of a given type.
37 Parameters
38 ----------
39 maskedImage : `lsst.afw.image.MaskedImage`
40 Image to measure the mask on.
41 maskPlane : `str`
42 Name of the mask plane to count
44 Returns
45 -------
46 nMask : `int`
47 Number of masked pixels.
48 """
49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
50 isBit = maskedImage.getMask().getArray() & bitMask > 0
51 numBit = np.sum(isBit)
52 return numBit
55def computeImageMedianAndStd(image):
56 """Function to calculate median and std of image data.
58 Parameters
59 ----------
60 image : `lsst.afw.image.Image`
61 Image to measure statistics on.
63 Returns
64 -------
65 median : `float`
66 Image median.
67 std : `float`
68 Image stddev.
69 """
70 median = np.nanmedian(image.getArray())
71 std = np.nanstd(image.getArray())
72 return (median, std)
75class IsrTaskTestCases(lsst.utils.tests.TestCase):
76 """Test IsrTask methods with trimmed raw data.
77 """
78 def setUp(self):
79 self.config = IsrTaskConfig()
80 self.config.qa = IsrQaConfig()
81 self.task = IsrTask(config=self.config)
82 self.dataRef = isrMock.DataRefMock()
83 self.camera = isrMock.IsrMock().getCamera()
85 self.inputExp = isrMock.TrimmedRawMock().run()
86 self.amp = self.inputExp.getDetector()[0]
87 self.mi = self.inputExp.getMaskedImage()
89 def validateIsrData(self, results):
90 """results should be a struct with components that are
91 not None if included in the configuration file.
92 """
93 self.assertIsInstance(results, Struct)
94 if self.config.doBias is True:
95 self.assertIsNotNone(results.bias)
96 if self.config.doDark is True:
97 self.assertIsNotNone(results.dark)
98 if self.config.doFlat is True:
99 self.assertIsNotNone(results.flat)
100 if self.config.doFringe is True:
101 self.assertIsNotNone(results.fringes)
102 if self.config.doDefect is True:
103 self.assertIsNotNone(results.defects)
104 if self.config.doBrighterFatter is True:
105 self.assertIsNotNone(results.bfKernel)
106 if self.config.doAttachTransmissionCurve is True:
107 self.assertIsNotNone(results.opticsTransmission)
108 self.assertIsNotNone(results.filterTransmission)
109 self.assertIsNotNone(results.sensorTransmission)
110 self.assertIsNotNone(results.atmosphereTransmission)
112 def test_readIsrData_noTrans(self):
113 """Test that all necessary calibration frames are retrieved.
114 """
115 self.config.doAttachTransmissionCurve = False
116 self.task = IsrTask(config=self.config)
117 results = self.task.readIsrData(self.dataRef, self.inputExp)
118 self.validateIsrData(results)
120 def test_readIsrData_withTrans(self):
121 """Test that all necessary calibration frames are retrieved.
122 """
123 self.config.doAttachTransmissionCurve = True
124 self.task = IsrTask(config=self.config)
125 results = self.task.readIsrData(self.dataRef, self.inputExp)
126 self.validateIsrData(results)
128 def test_ensureExposure(self):
129 """Test that an exposure has a usable instance class.
130 """
131 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0),
132 afwImage.Exposure)
134 def test_convertItoF(self):
135 """Test conversion from integer to floating point pixels.
136 """
137 result = self.task.convertIntToFloat(self.inputExp)
138 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32"))
139 self.assertEqual(result, self.inputExp)
141 def test_updateVariance(self):
142 """Expect The variance image should have a larger median value after
143 this operation.
144 """
145 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
146 self.task.updateVariance(self.inputExp, self.amp)
147 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()])
148 self.assertGreater(statAfter[0], statBefore[0])
149 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
150 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
152 def test_darkCorrection(self):
153 """Expect the median image value should decrease after this operation.
154 """
155 darkIm = isrMock.DarkMock().run()
157 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
158 self.task.darkCorrection(self.inputExp, darkIm)
159 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
160 self.assertLess(statAfter[0], statBefore[0])
161 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
162 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
164 def test_darkCorrection_noVisitInfo(self):
165 """Expect the median image value should decrease after this operation.
166 """
167 darkIm = isrMock.DarkMock().run()
168 darkIm.getInfo().setVisitInfo(None)
170 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
171 self.task.darkCorrection(self.inputExp, darkIm)
172 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
173 self.assertLess(statAfter[0], statBefore[0])
174 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
175 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
177 def test_flatCorrection(self):
178 """Expect the image median should increase (divide by < 1).
179 """
180 flatIm = isrMock.FlatMock().run()
182 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
183 self.task.flatCorrection(self.inputExp, flatIm)
184 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()])
185 self.assertGreater(statAfter[1], statBefore[1])
186 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
187 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
189 def test_saturationDetection(self):
190 """Expect the saturation level detection/masking to scale with
191 threshold.
192 """
193 ampB = self.amp.rebuild()
194 ampB.setSaturation(9000.0)
195 self.task.saturationDetection(self.inputExp, ampB.finish())
196 countBefore = countMaskedPixels(self.mi, "SAT")
198 ampB.setSaturation(8250.0)
199 self.task.saturationDetection(self.inputExp, ampB.finish())
200 countAfter = countMaskedPixels(self.mi, "SAT")
202 self.assertLessEqual(countBefore, countAfter)
203 self.assertEqual(countBefore, 43)
204 self.assertEqual(countAfter, 136)
206 def test_measureBackground(self):
207 """Expect the background measurement runs successfully and to save
208 metadata values.
209 """
210 self.config.qa.flatness.meshX = 20
211 self.config.qa.flatness.meshY = 20
212 self.task.measureBackground(self.inputExp, self.config.qa)
213 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
215 def test_flatContext(self):
216 """Expect the flat context manager runs successfully (applying both
217 flat and dark within the context), and results in the same
218 image data after completion.
219 """
220 darkExp = isrMock.DarkMock().run()
221 flatExp = isrMock.FlatMock().run()
223 mi = self.inputExp.getMaskedImage().clone()
224 with self.task.flatContext(self.inputExp, flatExp, darkExp):
225 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage())
226 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
228 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
231class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
232 """Test IsrTask methods using untrimmed raw data.
233 """
234 def setUp(self):
235 self.config = IsrTaskConfig()
236 self.config.qa = IsrQaConfig()
237 self.task = IsrTask(config=self.config)
239 self.mockConfig = isrMock.IsrMockConfig()
240 self.mockConfig.isTrimmed = False
241 self.doGenerateImage = True
242 self.dataRef = isrMock.DataRefMock(config=self.mockConfig)
243 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
245 self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
246 self.amp = self.inputExp.getDetector()[0]
247 self.mi = self.inputExp.getMaskedImage()
249 def batchSetConfiguration(self, value):
250 """Set the configuration state to a consistent value.
252 Disable options we do not need as well.
254 Parameters
255 ----------
256 value : `bool`
257 Value to switch common ISR configuration options to.
258 """
259 self.config.qa.flatness.meshX = 20
260 self.config.qa.flatness.meshY = 20
261 self.config.doWrite = False
262 self.config.doLinearize = False
263 self.config.doCrosstalk = False
265 self.config.doConvertIntToFloat = value
266 self.config.doSaturation = value
267 self.config.doSuspect = value
268 self.config.doSetBadRegions = value
269 self.config.doOverscan = value
270 self.config.doBias = value
271 self.config.doVariance = value
272 self.config.doWidenSaturationTrails = value
273 self.config.doBrighterFatter = value
274 self.config.doDefect = value
275 self.config.doSaturationInterpolation = value
276 self.config.doDark = value
277 self.config.doStrayLight = value
278 self.config.doFlat = value
279 self.config.doFringe = value
280 self.config.doMeasureBackground = value
281 self.config.doVignette = value
282 self.config.doAttachTransmissionCurve = value
283 self.config.doUseOpticsTransmission = value
284 self.config.doUseFilterTransmission = value
285 self.config.doUseSensorTransmission = value
286 self.config.doUseAtmosphereTransmission = value
287 self.config.qa.saveStats = value
288 self.config.qa.doThumbnailOss = value
289 self.config.qa.doThumbnailFlattened = value
291 self.config.doApplyGains = not value
292 self.config.doCameraSpecificMasking = value
293 self.config.vignette.doWriteVignettePolygon = value
295 def validateIsrResults(self):
296 """results should be a struct with components that are
297 not None if included in the configuration file.
299 Returns
300 -------
301 results : `pipeBase.Struct`
302 Results struct generated from the current ISR configuration.
303 """
304 self.task = IsrTask(config=self.config)
305 results = self.task.run(self.inputExp,
306 camera=self.camera,
307 bias=self.dataRef.get("bias"),
308 dark=self.dataRef.get("dark"),
309 flat=self.dataRef.get("flat"),
310 bfKernel=self.dataRef.get("bfKernel"),
311 defects=self.dataRef.get("defects"),
312 fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
313 opticsTransmission=self.dataRef.get("transmission_"),
314 filterTransmission=self.dataRef.get("transmission_"),
315 sensorTransmission=self.dataRef.get("transmission_"),
316 atmosphereTransmission=self.dataRef.get("transmission_")
317 )
319 self.assertIsInstance(results, Struct)
320 self.assertIsInstance(results.exposure, afwImage.Exposure)
321 return results
323 def test_overscanCorrection(self):
324 """Expect that this should reduce the image variance with a full fit.
325 The default fitType of MEDIAN will reduce the median value.
327 This needs to operate on a RawMock() to have overscan data to use.
329 The output types may be different when fitType != MEDIAN.
330 """
331 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
332 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
333 self.assertIsInstance(oscanResults, Struct)
334 self.assertIsInstance(oscanResults.imageFit, float)
335 self.assertIsInstance(oscanResults.overscanFit, float)
336 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
338 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
339 self.assertLess(statAfter[0], statBefore[0])
341 def test_overscanCorrectionMedianPerRow(self):
342 """Expect that this should reduce the image variance with a full fit.
343 fitType of MEDIAN_PER_ROW will reduce the median value.
345 This needs to operate on a RawMock() to have overscan data to use.
347 The output types may be different when fitType != MEDIAN_PER_ROW.
348 """
349 self.config.overscan.fitType = 'MEDIAN_PER_ROW'
350 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
351 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
352 self.assertIsInstance(oscanResults, Struct)
353 self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF)
354 self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF)
355 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF)
357 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()])
358 self.assertLess(statAfter[0], statBefore[0])
360 def test_runDataRef(self):
361 """Expect a dataRef to be handled correctly.
362 """
363 self.config.doLinearize = False
364 self.config.doWrite = False
365 self.task = IsrTask(config=self.config)
366 results = self.task.runDataRef(self.dataRef)
368 self.assertIsInstance(results, Struct)
369 self.assertIsInstance(results.exposure, afwImage.Exposure)
371 def test_run_allTrue(self):
372 """Expect successful run with expected outputs when all non-exclusive
373 configuration options are on.
375 Output results should be tested more precisely by the
376 individual function tests.
378 """
379 self.batchSetConfiguration(True)
380 self.validateIsrResults()
382 def test_run_allFalse(self):
383 """Expect successful run with expected outputs when all non-exclusive
384 configuration options are off.
386 Output results should be tested more precisely by the
387 individual function tests.
389 """
390 self.batchSetConfiguration(False)
391 self.validateIsrResults()
393 def test_failCases(self):
394 """Expect failure with crosstalk enabled.
396 Output results should be tested more precisely by the
397 individual function tests.
398 """
399 self.batchSetConfiguration(True)
401 # This breaks it
402 self.config.doCrosstalk = True
404 with self.assertRaises(RuntimeError):
405 self.validateIsrResults()
407 def test_maskingCase_noMasking(self):
408 """Test masking cases of configuration parameters.
409 """
410 self.batchSetConfiguration(True)
411 self.config.overscanFitType = "POLY"
412 self.config.overscanOrder = 1
414 self.config.doSaturation = False
415 self.config.doWidenSaturationTrails = False
416 self.config.doSaturationInterpolation = False
417 self.config.doSuspect = False
418 self.config.doSetBadRegions = False
419 self.config.doDefect = False
420 self.config.doBrighterFatter = False
422 results = self.validateIsrResults()
424 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
425 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
426 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
427 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
429 def test_maskingCase_satMasking(self):
430 """Test masking cases of configuration parameters.
431 """
432 self.batchSetConfiguration(True)
433 self.config.overscanFitType = "POLY"
434 self.config.overscanOrder = 1
436 self.config.saturation = 20000.0
437 self.config.doSaturation = True
438 self.config.doWidenSaturationTrails = True
440 self.config.doSaturationInterpolation = False
441 self.config.doSuspect = False
442 self.config.doSetBadRegions = False
443 self.config.doDefect = False
444 self.config.doBrighterFatter = False
446 results = self.validateIsrResults()
448 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
449 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
450 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
451 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
453 def test_maskingCase_satMaskingAndInterp(self):
454 """Test masking cases of configuration parameters.
455 """
456 self.batchSetConfiguration(True)
457 self.config.overscanFitType = "POLY"
458 self.config.overscanOrder = 1
460 self.config.saturation = 20000.0
461 self.config.doSaturation = True
462 self.config.doWidenSaturationTrails = True
463 self.config.doSaturationInterpolation = True
465 self.config.doSuspect = False
466 self.config.doSetBadRegions = False
467 self.config.doDefect = False
468 self.config.doBrighterFatter = False
470 results = self.validateIsrResults()
472 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
473 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
474 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
475 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
477 def test_maskingCase_throughEdge(self):
478 """Test masking cases of configuration parameters.
479 """
480 self.batchSetConfiguration(True)
481 self.config.overscanFitType = "POLY"
482 self.config.overscanOrder = 1
484 self.config.saturation = 20000.0
485 self.config.doSaturation = True
486 self.config.doWidenSaturationTrails = True
487 self.config.doSaturationInterpolation = True
488 self.config.numEdgeSuspect = 5
489 self.config.doSuspect = True
491 self.config.doSetBadRegions = False
492 self.config.doDefect = False
493 self.config.doBrighterFatter = False
495 results = self.validateIsrResults()
497 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
498 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
499 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
500 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
502 def test_maskingCase_throughDefects(self):
503 """Test masking cases of configuration parameters.
504 """
505 self.batchSetConfiguration(True)
506 self.config.overscanFitType = "POLY"
507 self.config.overscanOrder = 1
509 self.config.saturation = 20000.0
510 self.config.doSaturation = True
511 self.config.doWidenSaturationTrails = True
512 self.config.doSaturationInterpolation = True
513 self.config.numEdgeSuspect = 5
514 self.config.doSuspect = True
515 self.config.doDefect = True
517 self.config.doSetBadRegions = False
518 self.config.doBrighterFatter = False
520 results = self.validateIsrResults()
522 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
523 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
524 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
525 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
527 def test_maskingCase_throughBad(self):
528 """Test masking cases of configuration parameters.
529 """
530 self.batchSetConfiguration(True)
531 self.config.overscanFitType = "POLY"
532 self.config.overscanOrder = 1
534 self.config.saturation = 20000.0
535 self.config.doSaturation = True
536 self.config.doWidenSaturationTrails = True
537 self.config.doSaturationInterpolation = True
539 self.config.doSuspect = True
540 self.config.doDefect = True
541 self.config.doSetBadRegions = True
542 self.config.doBrighterFatter = False
544 results = self.validateIsrResults()
546 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
547 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
548 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
549 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
552class MemoryTester(lsst.utils.tests.MemoryTestCase):
553 pass
556def setup_module(module):
557 lsst.utils.tests.init()
560if __name__ == "__main__": 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true
561 lsst.utils.tests.init()
562 unittest.main()