Coverage for tests/test_defects.py: 9%
414 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23"""Test cases for lsst.cp.pipe.defects.MeasureDefectsTask."""
25import unittest
26import numpy as np
27import copy
29import lsst.utils
30import lsst.utils.tests
32import lsst.afw.image as afwImage
33import lsst.ip.isr as ipIsr
34import lsst.cp.pipe as cpPipe
35from lsst.ip.isr import isrMock, countMaskedPixels
36from lsst.geom import Box2I, Point2I, Extent2I
37from lsst.daf.base import PropertyList
40class MeasureDefectsTaskTestCase(lsst.utils.tests.TestCase):
41 """A test case for the defect finding task."""
43 def setUp(self):
44 self.defaultConfig = cpPipe.defects.MeasureDefectsTask.ConfigClass()
46 self.flatMean = 2000
47 self.darkMean = 1
48 self.readNoiseAdu = 10
49 self.nSigmaBright = 8
50 self.nSigmaDark = 8
52 mockImageConfig = isrMock.IsrMock.ConfigClass()
54 # flatDrop is not really relevant as we replace the data
55 # but good to note it in case we change how this image is made
56 mockImageConfig.flatDrop = 0.99999
57 mockImageConfig.isTrimmed = True
59 self.flatExp = isrMock.FlatMock(config=mockImageConfig).run()
60 (shapeY, shapeX) = self.flatExp.getDimensions()
61 # x, y, size tuples
62 # always put edge defects at the start and change the value of nEdge
64 self.brightDefects = [(0, 15, 3, 3), (100, 123, 1, 1)]
66 self.darkDefects = [(5, 0, 1, 1), (7, 62, 2, 2)]
68 nEdge = 1 # NOTE: update if more edge defects are included
69 self.noEdges = slice(nEdge, None)
70 self.onlyEdges = slice(0, nEdge)
72 self.darkBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.darkDefects]
73 self.brightBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.brightDefects]
75 flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
76 darkWidth = self.readNoiseAdu
77 self.rng = np.random.RandomState(0)
78 flatData = self.rng.normal(self.flatMean, flatWidth, (shapeX, shapeY))
79 darkData = self.rng.normal(self.darkMean, darkWidth, (shapeX, shapeY))
81 # NOTE: darks and flats have same defects applied deliberately to both
82 for defect in self.brightDefects:
83 y, x, sy, sx = defect
84 # are these actually the numbers we want?
85 flatData[x:x+sx, y:y+sy] += self.nSigmaBright * flatWidth
86 darkData[x:x+sx, y:y+sy] += self.nSigmaBright * darkWidth
88 for defect in self.darkDefects:
89 y, x, sy, sx = defect
90 # are these actually the numbers we want?
91 flatData[x:x+sx, y:y+sy] -= self.nSigmaDark * flatWidth
92 darkData[x:x+sx, y:y+sy] -= self.nSigmaDark * darkWidth
94 self.darkExp = self.flatExp.clone()
95 self.spareImage = self.flatExp.clone() # for testing edge bits and misc
97 self.flatExp.image.array[:] = flatData
98 self.darkExp.image.array[:] = darkData
100 self.defaultTask = cpPipe.defects.MeasureDefectsTask()
102 self.allDefectsList = ipIsr.Defects()
103 self.brightDefectsList = ipIsr.Defects()
104 self.darkDefectsList = ipIsr.Defects()
106 # Set image types, the defects code will use them.
107 metaDataFlat = PropertyList()
108 metaDataFlat["IMGTYPE"] = "FLAT"
109 self.flatExp.setMetadata(metaDataFlat)
111 metaDataDark = PropertyList()
112 metaDataDark["IMGTYPE"] = "DARK"
113 self.darkExp.setMetadata(metaDataDark)
115 with self.allDefectsList.bulk_update():
116 with self.brightDefectsList.bulk_update():
117 for d in self.brightBBoxes:
118 self.brightDefectsList.append(d)
119 self.allDefectsList.append(d)
121 with self.darkDefectsList.bulk_update():
122 for d in self.darkBBoxes:
123 self.darkDefectsList.append(d)
124 self.allDefectsList.append(d)
126 def check_maskBlocks(self, inputDefects, expectedDefects):
127 """A helper function for the tests of
128 maskBlocksIfIntermitentBadPixelsInColumn.
130 """
131 config = copy.copy(self.defaultConfig)
132 config.badOnAndOffPixelColumnThreshold = 10
133 config.goodPixelColumnGapThreshold = 5
134 config.nPixBorderUpDown = 0
135 config.nPixBorderLeftRight = 0
137 task = self.defaultTask
138 task.config = config
140 defectsWithColumns, count = task.maskBlocksIfIntermitentBadPixelsInColumn(inputDefects)
141 boxesMeasured = []
142 for defect in defectsWithColumns:
143 boxesMeasured.append(defect.getBBox())
145 for boxInput in expectedDefects:
146 self.assertIn(boxInput, boxesMeasured)
148 # Check that the code did not mask anything extra by
149 # looking in both the input list and "expanded-column" list.
150 unionInputExpectedBoxes = []
151 for defect in inputDefects:
152 unionInputExpectedBoxes.append(defect.getBBox())
153 for defect in expectedDefects:
154 unionInputExpectedBoxes.append(defect)
156 # Check that code doesn't mask more than it is supposed to.
157 for boxMeas in boxesMeasured:
158 self.assertIn(boxMeas, unionInputExpectedBoxes)
160 def test_maskBlocks_full_column(self):
161 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
163 Tests that a contigous bad column does not get split by the
164 code.
166 The mock flat has a size of 200X204 pixels. This column has a
167 maximum length of 50 pixels, otherwise there would be a split
168 along the mock amp boundary.
170 Plots can be found in DM-19903 on Jira.
172 """
174 defects = self.allDefectsList
175 defects.append(Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50)))
176 expectedDefects = [Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))]
178 self.check_maskBlocks(defects, expectedDefects)
180 def test_maskBlocks_long_column(self):
181 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
183 Tests that a contigous bad column with Npix >=
184 badOnAndOffPixelColumnThreshold (10) does not get split by the
185 code.
187 Plots can be found in DM-19903 on Jira.
189 """
191 expectedDefects = [Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))]
192 defects = self.allDefectsList
193 defects.append(Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25)))
195 self.check_maskBlocks(defects, expectedDefects)
197 def test_maskBlocks_short_column(self):
198 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
200 Tests that a contigous bad column Npix <
201 badOnAndOffPixelColumnThreshold (10) does not get split by the
202 code.
204 Plots can be found in DM-19903 on Jira.
206 """
208 expectedDefects = [Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))]
209 defects = self.allDefectsList
210 defects.append(Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8)))
212 self.check_maskBlocks(defects, expectedDefects)
214 def test_maskBlocks_discontigous_to_single_block(self):
215 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
217 Npix discontiguous bad pixels in a column where Npix >=
218 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <
219 goodPixelColumnGapThreshold (5). Under these conditions, the
220 whole block of bad pixels (including good gaps) should be
221 masked.
223 Plots can be found in DM-19903 on Jira.
225 """
227 expectedDefects = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 48))]
228 defects = self.allDefectsList
229 badPixels = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 2)),
230 Box2I(corner=Point2I(30, 5), dimensions=Extent2I(1, 3)),
231 Box2I(corner=Point2I(30, 11), dimensions=Extent2I(1, 5)),
232 Box2I(corner=Point2I(30, 19), dimensions=Extent2I(1, 5)),
233 Box2I(corner=Point2I(30, 27), dimensions=Extent2I(1, 4)),
234 Box2I(corner=Point2I(30, 34), dimensions=Extent2I(1, 15))]
235 with defects.bulk_update():
236 for badBox in badPixels:
237 defects.append(badBox)
239 self.check_maskBlocks(defects, expectedDefects)
241 def test_maskBlocks_discontigous_less_than_thresholds(self):
242 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
244 Npix discontiguous bad pixels in a column where Npix <
245 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <
246 goodPixelColumnGapThreshold (5). Under these conditions, the
247 expected defect boxes should be the same as the input boxes.
249 Plots can be found in DM-19903 on Jira.
251 """
253 expectedDefects = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)),
254 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)),
255 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))]
256 defects = self.allDefectsList
257 badPixels = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)),
258 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)),
259 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))]
260 with defects.bulk_update():
261 for badBox in badPixels:
262 defects.append(badBox)
264 self.check_maskBlocks(defects, expectedDefects)
266 def test_maskBlocks_more_than_thresholds(self):
267 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
269 Npix discontiguous bad pixels in a column where Npix <
270 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <
271 goodPixelColumnGapThreshold (5). Npix=34 (> 10) bad pixels
272 total, 1 "good" gap with 13 pixels big enough (13 >= 5 good
273 pixels, from y=6 (1+5) to y=19).
275 Plots can be found in DM-19903 on Jira.
277 """
279 expectedDefects = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 7)),
280 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 30))]
281 defects = self.allDefectsList
282 badPixels = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 2)),
283 Box2I(corner=Point2I(40, 5), dimensions=Extent2I(1, 3)),
284 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 5)),
285 Box2I(corner=Point2I(40, 27), dimensions=Extent2I(1, 4)),
286 Box2I(corner=Point2I(40, 34), dimensions=Extent2I(1, 15))]
287 with defects.bulk_update():
288 for badBox in badPixels:
289 defects.append(badBox)
291 self.check_maskBlocks(defects, expectedDefects)
293 def test_maskBlocks_not_enough_bad_pixels_in_column(self):
294 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
296 Npix discontiguous bad pixels in a column where Npix <
297 badOnAndOffPixelColumnThreshold (10) and and gaps of good
298 pixels > goodPixelColumnGapThreshold (5). Since Npix <
299 badOnAndOffPixelColumnThreshold, then it doesn't matter that
300 the number of good pixels in gap >
301 goodPixelColumnGapThreshold. 5<10 bad pixels total, 1 "good"
302 gap big enough (29>=5 good pixels, from y =12 (10+2) to y=30)
304 Plots can be found in DM-19903 on Jira.
306 """
308 expectedDefects = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)),
309 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))]
310 defects = self.allDefectsList
311 badPixels = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)),
312 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))]
313 with defects.bulk_update():
314 for badBox in badPixels:
315 defects.append(badBox)
317 self.check_maskBlocks(defects, expectedDefects)
319 def test_maskBlocks_every_other_pixel_bad_greater_than_threshold(self):
320 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
322 Npix discontiguous bad pixels in a column where Npix >
323 badOnAndOffPixelColumnThreshold (10) and every other pixel is
324 bad.
326 Plots can be found in DM-19903 on Jira.
328 """
330 expectedDefects = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 31))]
331 defects = self.allDefectsList
332 badPixels = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 1)),
333 Box2I(corner=Point2I(50, 12), dimensions=Extent2I(1, 1)),
334 Box2I(corner=Point2I(50, 14), dimensions=Extent2I(1, 1)),
335 Box2I(corner=Point2I(50, 16), dimensions=Extent2I(1, 1)),
336 Box2I(corner=Point2I(50, 18), dimensions=Extent2I(1, 1)),
337 Box2I(corner=Point2I(50, 20), dimensions=Extent2I(1, 1)),
338 Box2I(corner=Point2I(50, 22), dimensions=Extent2I(1, 1)),
339 Box2I(corner=Point2I(50, 24), dimensions=Extent2I(1, 1)),
340 Box2I(corner=Point2I(50, 26), dimensions=Extent2I(1, 1)),
341 Box2I(corner=Point2I(50, 28), dimensions=Extent2I(1, 1)),
342 Box2I(corner=Point2I(50, 30), dimensions=Extent2I(1, 1)),
343 Box2I(corner=Point2I(50, 32), dimensions=Extent2I(1, 1)),
344 Box2I(corner=Point2I(50, 34), dimensions=Extent2I(1, 1)),
345 Box2I(corner=Point2I(50, 36), dimensions=Extent2I(1, 1)),
346 Box2I(corner=Point2I(50, 38), dimensions=Extent2I(1, 1)),
347 Box2I(corner=Point2I(50, 40), dimensions=Extent2I(1, 1))]
348 with defects.bulk_update():
349 for badBox in badPixels:
350 defects.append(badBox)
352 self.check_maskBlocks(defects, expectedDefects)
354 def test_maskBlocks_every_other_pixel_bad_less_than_threshold(self):
355 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
357 Npix discontiguous bad pixels in a column where Npix >
358 badOnAndOffPixelColumnThreshold (10) and every other pixel is
359 bad.
361 Plots can be found in DM-19903 on Jira.
363 """
365 expectedDefects = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)),
366 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)),
367 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)),
368 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)),
369 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)),
370 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))]
371 defects = self.allDefectsList
372 badPixels = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)),
373 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)),
374 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)),
375 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)),
376 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)),
377 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))]
378 with defects.bulk_update():
379 for badBox in badPixels:
380 defects.append(badBox)
382 self.check_maskBlocks(defects, expectedDefects)
384 def test_maskBlocks_blobs_one_side_good_less_than_threshold(self):
385 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
387 Npix discontiguous bad pixels in column with "blobs" of "m"
388 bad pixels to one side, m > badOnAndOffPixelColumnThreshold
389 (10), number of good pixel in gaps between blobs <
390 goodPixelColumnGapThreshold (5).
392 Plots can be found in DM-19903 on Jira.
394 """
396 expectedDefects = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 29)),
397 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 12))]
398 defects = self.allDefectsList
399 badPixels = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 18)),
400 Box2I(corner=Point2I(60, 20), dimensions=Extent2I(1, 10)),
401 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 2)),
402 Box2I(corner=Point2I(61, 6), dimensions=Extent2I(2, 8))]
403 with defects.bulk_update():
404 for badBox in badPixels:
405 defects.append(badBox)
407 self.check_maskBlocks(defects, expectedDefects)
409 def test_maskBlocks_blobs_other_side_good_less_than_threshold(self):
410 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
412 Npix discontiguous bad pixels in column with "blobs" of "m"
413 bad pixels to the other side, m >
414 badOnAndOffPixelColumnThreshold (10), number of good pixel in
415 gaps between blobs < goodPixelColumnGapThreshold (5).
417 Plots can be found in DM-19903 on Jira.
419 """
421 expectedDefects = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 29)),
422 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 12))]
423 defects = self.allDefectsList
424 badPixels = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 18)),
425 Box2I(corner=Point2I(70, 20), dimensions=Extent2I(1, 10)),
426 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 2)),
427 Box2I(corner=Point2I(68, 6), dimensions=Extent2I(2, 8))]
428 with defects.bulk_update():
429 for badBox in badPixels:
430 defects.append(badBox)
432 self.check_maskBlocks(defects, expectedDefects)
434 def test_maskBlocks_blob_both_sides_good_less_than_threshold(self):
435 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
437 Npix discontiguous bad pixels in column with "blobs" of "m"
438 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold
439 (10), number of good pixel in gaps between blobs <
440 goodPixelColumnGapThreshold (5).
442 Plots can be found in DM-19903 on Jira.
444 """
446 expectedDefects = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 29)),
447 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 12)),
448 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 12))]
449 defects = self.allDefectsList
450 badPixels = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 18)),
451 Box2I(corner=Point2I(75, 20), dimensions=Extent2I(1, 10)),
452 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 2)),
453 Box2I(corner=Point2I(73, 6), dimensions=Extent2I(2, 8)),
454 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 2)),
455 Box2I(corner=Point2I(76, 6), dimensions=Extent2I(2, 8))]
456 with defects.bulk_update():
457 for badBox in badPixels:
458 defects.append(badBox)
460 self.check_maskBlocks(defects, expectedDefects)
462 def test_maskBlocks_blob_one_side_good_greater_than_threshold(self):
463 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
465 Npix discontiguous bad pixels in column with "blobs" of "m"
466 bad pixels to one side, m > badOnAndOffPixelColumnThreshold
467 (10), number of good pixel in gaps between blobs >
468 goodPixelColumnGapThreshold (5).
470 Plots can be found in DM-19903 on Jira.
472 """
474 expectedDefects = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 29)),
475 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)),
476 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))]
477 defects = self.allDefectsList
478 badPixels = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 18)),
479 Box2I(corner=Point2I(80, 20), dimensions=Extent2I(1, 10)),
480 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)),
481 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))]
482 with defects.bulk_update():
483 for badBox in badPixels:
484 defects.append(badBox)
486 self.check_maskBlocks(defects, expectedDefects)
488 def test_maskBlocks_other_side_good_greater_than_threshold(self):
489 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
491 Npix discontiguous bad pixels in column with "blobs" of "m"
492 bad pixels to the other side, m >
493 badOnAndOffPixelColumnThreshold (10), number of good pixel in
494 gaps between blobs > goodPixelColumnGapThreshold (5).
496 Plots can be found in DM-19903 on Jira.
498 """
500 expectedDefects = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 29)),
501 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)),
502 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))]
503 defects = self.allDefectsList
504 badPixels = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 18)),
505 Box2I(corner=Point2I(87, 20), dimensions=Extent2I(1, 10)),
506 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)),
507 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))]
508 with defects.bulk_update():
509 for badBox in badPixels:
510 defects.append(badBox)
512 self.check_maskBlocks(defects, expectedDefects)
514 def test_maskBlocks_both_sides_good_greater_than_threshold(self):
515 """A test for maskBlocksIfIntermitentBadPixelsInColumn.
517 Npix discontiguous bad pixels in column with "blobs" of "m"
518 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold
519 (10), number of good pixel in gaps between blobs >
520 goodPixelColumnGapThreshold (5).
522 Plots can be found in DM-19903 on Jira.
524 """
526 expectedDefects = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 34)),
527 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 7)),
528 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 9)),
529 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 7)),
530 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 9))]
531 defects = self.allDefectsList
532 badPixels = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 12)),
533 Box2I(corner=Point2I(93, 15), dimensions=Extent2I(1, 20)),
534 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 2)),
535 Box2I(corner=Point2I(91, 7), dimensions=Extent2I(2, 2)),
536 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 2)),
537 Box2I(corner=Point2I(94, 7), dimensions=Extent2I(2, 2)),
538 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 3)),
539 Box2I(corner=Point2I(91, 24), dimensions=Extent2I(2, 3)),
540 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 3)),
541 Box2I(corner=Point2I(94, 24), dimensions=Extent2I(2, 3))]
542 with defects.bulk_update():
543 for badBox in badPixels:
544 defects.append(badBox)
546 self.check_maskBlocks(defects, expectedDefects)
548 def test_maskBlocks_y_out_of_order_dm38103(self):
549 """A test for maskBlocksIfIntermitentBadPixelsInColumn, y out of order.
551 This test is a variant of
552 notest_maskBlocks_every_other_pixel_bad_greater_than_threshold with
553 an extra out-of-y-order bad pixel to trigger DM-38103.
554 """
555 expectedDefects = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 31))]
556 defects = self.allDefectsList
557 badPixels = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 1)),
558 Box2I(corner=Point2I(50, 112), dimensions=Extent2I(1, 1)),
559 Box2I(corner=Point2I(50, 114), dimensions=Extent2I(1, 1)),
560 Box2I(corner=Point2I(50, 116), dimensions=Extent2I(1, 1)),
561 Box2I(corner=Point2I(50, 118), dimensions=Extent2I(1, 1)),
562 Box2I(corner=Point2I(50, 120), dimensions=Extent2I(1, 1)),
563 Box2I(corner=Point2I(50, 122), dimensions=Extent2I(1, 1)),
564 Box2I(corner=Point2I(50, 124), dimensions=Extent2I(1, 1)),
565 Box2I(corner=Point2I(50, 126), dimensions=Extent2I(1, 1)),
566 Box2I(corner=Point2I(50, 128), dimensions=Extent2I(1, 1)),
567 Box2I(corner=Point2I(50, 130), dimensions=Extent2I(1, 1)),
568 Box2I(corner=Point2I(50, 132), dimensions=Extent2I(1, 1)),
569 Box2I(corner=Point2I(50, 134), dimensions=Extent2I(1, 1)),
570 Box2I(corner=Point2I(50, 136), dimensions=Extent2I(1, 1)),
571 Box2I(corner=Point2I(50, 138), dimensions=Extent2I(1, 1)),
572 Box2I(corner=Point2I(50, 140), dimensions=Extent2I(1, 1)),
573 # This last point is out of order in y.
574 Box2I(corner=Point2I(50, 100), dimensions=Extent2I(1, 1))]
576 # Force defect normalization off in order to trigger DM-38301, because
577 # defects.fromFootprintList() which is called by _findHotAndColdPixels
578 # does not do normalization.
579 defects._bulk_update = True
580 for badBox in badPixels:
581 defects.append(badBox)
582 defects._bulk_update = False
584 self.check_maskBlocks(defects, expectedDefects)
586 def check_maskBadColumns(self, exp, inputDefects, expectedDefects):
587 """A helper function for the tests of
588 maskBadColumns.
590 """
591 config = copy.copy(self.defaultConfig)
592 config.badPixelsToFillColumnThreshold = 25
593 config.saturatedPixelsToFillColumnThreshold = 5
595 task = self.defaultTask
596 task.config = config
598 defectsWithColumns, count = task.maskBadColumns(exp, inputDefects)
600 self.assertEqual(count, len(expectedDefects))
602 boxesMeasured = []
603 for defect in defectsWithColumns:
604 boxesMeasured.append(defect.getBBox())
606 for boxInput in expectedDefects:
607 self.assertIn(boxInput, boxesMeasured)
609 # Check that the code did not mask anything extra by
610 # looking in both the input list and "expanded-column" list.
611 unionInputExpectedBoxes = []
612 for defect in inputDefects:
613 unionInputExpectedBoxes.append(defect.getBBox())
614 for defect in expectedDefects:
615 unionInputExpectedBoxes.append(defect)
617 # Check that code doesn't mask more than it is supposed to.
618 for boxMeas in boxesMeasured:
619 self.assertIn(boxMeas, unionInputExpectedBoxes)
621 def test_maskBadColumns_extend_full_columns(self):
622 """Test maskBadColumns, extend to full column.
623 """
624 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)),
625 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)),
626 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))]
627 defects = self.allDefectsList
628 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 30)))
629 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 25)))
630 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 30)))
632 self.check_maskBadColumns(self.flatExp, defects, expectedDefects)
634 def test_maskBadColumns_no_extend_partial_columns(self):
635 """Test maskBadColumns, do not extend to full column.
636 """
637 expectedDefects = []
638 defects = self.allDefectsList
639 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 20)))
640 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 22)))
641 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 24)))
643 self.check_maskBadColumns(self.flatExp, defects, expectedDefects)
645 def test_maskBadColumns_extend_saturated_columns(self):
646 """Test maskBadColumns, extend saturation to full column.
647 """
648 exp = self.flatExp.clone()
650 mask = afwImage.Mask.getPlaneBitMask("SAT")
652 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)),
653 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)),
654 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))]
655 defects = self.allDefectsList
657 # These defects are too small to trigger the former column extension
658 # (as tested in test_maskBadColumns_no_extend_partial_columns) but
659 # should still trigger the saturation extension code.
660 satColumns = [Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 5)),
661 Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 5)),
662 Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 5))]
663 for satColumn in satColumns:
664 exp.mask[satColumn] |= mask
665 defects.append(satColumn)
667 self.check_maskBadColumns(exp, defects, expectedDefects)
669 def check_dilateSaturatedColumns(self, exp, inputDefects, expectedDefects):
670 config = copy.copy(self.defaultConfig)
671 config.saturatedColumnDilationRadius = 2
673 task = self.defaultTask
674 task.config = config
676 defectsDilated = task.dilateSaturatedColumns(exp, inputDefects)
678 boxesMeasured = []
679 for defect in defectsDilated:
680 boxesMeasured.append(defect.getBBox())
682 for boxInput in expectedDefects:
683 self.assertIn(boxInput, boxesMeasured)
685 # Check that the code did not mask anything extra by
686 # looking in both the input list and "expanded-column" list.
687 unionInputExpectedBoxes = []
688 for defect in inputDefects:
689 unionInputExpectedBoxes.append(defect.getBBox())
690 for defect in expectedDefects:
691 unionInputExpectedBoxes.append(defect)
693 # Check that code doesn't mask more than it is supposed to.
694 for boxMeas in boxesMeasured:
695 self.assertIn(boxMeas, unionInputExpectedBoxes)
697 def test_dilateSaturatedColumns_saturated_column(self):
698 exp = self.flatExp.clone()
700 mask = afwImage.Mask.getPlaneBitMask("SAT")
702 # We include saturated defects hitting the side to ensure we do not
703 # have any overflow.
704 # The dilation radius is set to 2 pixels.
705 expectedDefects = [Box2I(corner=Point2I(20, 5), dimensions=Extent2I(5, 20)),
706 Box2I(corner=Point2I(197, 5), dimensions=Extent2I(3, 10)),
707 Box2I(corner=Point2I(0, 160), dimensions=Extent2I(4, 15))]
708 defects = self.allDefectsList
710 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)),
711 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)),
712 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))]
713 for satColumn in satColumns:
714 exp.mask[satColumn] |= mask
715 defects.append(satColumn)
717 self.check_dilateSaturatedColumns(exp, defects, expectedDefects)
719 def test_dilateSaturatedColumns_no_saturated_column(self):
720 exp = self.flatExp.clone()
722 # These are marked BAD but not saturated.
723 mask = afwImage.Mask.getPlaneBitMask("BAD")
725 expectedDefects = []
726 defects = self.allDefectsList
728 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)),
729 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)),
730 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))]
731 for satColumn in satColumns:
732 exp.mask[satColumn] |= mask
733 defects.append(satColumn)
735 self.check_dilateSaturatedColumns(exp, defects, expectedDefects)
737 def test_defectFindingAllSensor(self):
738 config = copy.copy(self.defaultConfig)
739 config.nPixBorderLeftRight = 0
740 config.nPixBorderUpDown = 0
742 task = self.defaultTask
743 task.config = config
745 defects = task._findHotAndColdPixels(self.flatExp)
747 allBBoxes = self.darkBBoxes + self.brightBBoxes
749 boxesMeasured = []
750 for defect in defects:
751 boxesMeasured.append(defect.getBBox())
753 for expectedBBox in allBBoxes:
754 self.assertIn(expectedBBox, boxesMeasured)
756 def test_defectFindingEdgeIgnore(self):
757 config = copy.copy(self.defaultConfig)
758 config.nPixBorderUpDown = 0
759 config.nPixBorderLeftRight = 7
760 task = self.defaultTask
761 task.config = config
762 defects = task._findHotAndColdPixels(self.flatExp)
764 shouldBeFound = self.darkBBoxes[self.noEdges] + self.brightBBoxes[self.noEdges]
766 boxesMeasured = []
767 for defect in defects:
768 boxesMeasured.append(defect.getBBox())
770 for expectedBBox in shouldBeFound:
771 self.assertIn(expectedBBox, boxesMeasured)
773 shouldBeMissed = self.darkBBoxes[self.onlyEdges] + self.brightBBoxes[self.onlyEdges]
774 for boxMissed in shouldBeMissed:
775 self.assertNotIn(boxMissed, boxesMeasured)
777 def valueThreshold(self, fileType, saturateAmpInFlat=False):
778 """Helper function to loop over flats and darks
779 to test thresholdType = 'VALUE'."""
780 config = copy.copy(self.defaultConfig)
781 config.thresholdType = 'VALUE'
782 task = self.defaultTask
783 task.config = config
785 for amp in self.flatExp.getDetector():
786 if amp.getName() == 'C:0,0':
787 regionC00 = amp.getBBox()
789 if fileType == 'dark':
790 exp = self.darkExp
791 shouldBeFound = self.brightBBoxes[self.noEdges]
792 else:
793 exp = self.flatExp
794 if saturateAmpInFlat:
795 exp.maskedImage[regionC00].image.array[:] = 0.0
796 # Amp C:0,0: minimum=(0, 0), maximum=(99, 50)
797 x = self.defaultConfig.nPixBorderUpDown
798 y = self.defaultConfig.nPixBorderLeftRight
799 width, height = regionC00.getEndX() - x, regionC00.getEndY() - y
800 # Defects code will mark whole saturated amp as defect box.
801 shouldBeFound = [Box2I(corner=Point2I(x, y), dimensions=Extent2I(width, height))]
802 else:
803 shouldBeFound = self.darkBBoxes[self.noEdges]
804 # Change the default a bit so it works for the
805 # existing simulated defects.
806 task.config.fracThresholdFlat = 0.9
808 defects = task._findHotAndColdPixels(exp)
810 boxesMeasured = []
811 for defect in defects:
812 boxesMeasured.append(defect.getBBox())
814 for expectedBBox in shouldBeFound:
815 self.assertIn(expectedBBox, boxesMeasured)
817 def test_valueThreshold(self):
818 for fileType in ['dark', 'flat']:
819 self.valueThreshold(fileType)
820 # stdDev = 0.0
821 self.valueThreshold('flat', saturateAmpInFlat=True)
823 def test_pixelCounting(self):
824 """Test that the number of defective pixels identified is as expected.
825 """
826 config = copy.copy(self.defaultConfig)
827 config.nPixBorderUpDown = 0
828 config.nPixBorderLeftRight = 0
829 task = self.defaultTask
830 task.config = config
831 defects = task._findHotAndColdPixels(self.flatExp)
833 defectArea = 0
834 for defect in defects:
835 defectArea += defect.getBBox().getArea()
837 # The columnar code will cover blocks of a column with
838 # on-and-off pixels, thus creating more bad pixels that what
839 # initially placed in self.brightDefects and self.darkDefects.
840 # Thus, defectArea should be >= crossCheck.
841 crossCheck = 0
842 for x, y, sx, sy in self.brightDefects:
843 crossCheck += sx*sy
844 for x, y, sx, sy in self.darkDefects:
845 crossCheck += sx*sy
847 # Test the result of _nPixFromDefects()
848 # via two different ways of calculating area.
849 self.assertEqual(defectArea, task._nPixFromDefects(defects))
850 # defectArea should be >= crossCheck
851 self.assertGreaterEqual(defectArea, crossCheck)
853 def test_getNumGoodPixels(self):
854 """Test the the number of pixels in the image not masked is as
855 expected.
856 """
857 testImage = self.flatExp.clone()
858 mi = testImage.maskedImage
860 imageSize = testImage.getBBox().getArea()
861 nGood = self.defaultTask._getNumGoodPixels(mi)
863 self.assertEqual(imageSize, nGood)
865 NODATABIT = mi.mask.getPlaneBitMask("NO_DATA")
867 noDataBox = Box2I(Point2I(31, 49), Extent2I(3, 6))
868 testImage.mask[noDataBox] |= NODATABIT
870 self.assertEqual(imageSize - noDataBox.getArea(), self.defaultTask._getNumGoodPixels(mi))
871 # check for misfire; we're setting NO_DATA here, not BAD
872 self.assertEqual(imageSize, self.defaultTask._getNumGoodPixels(mi, 'BAD'))
874 testImage.mask[noDataBox] ^= NODATABIT # XOR to reset what we did
875 self.assertEqual(imageSize, nGood)
877 BADBIT = mi.mask.getPlaneBitMask("BAD")
878 badBox = Box2I(Point2I(85, 98), Extent2I(4, 7))
879 testImage.mask[badBox] |= BADBIT
881 self.assertEqual(imageSize - badBox.getArea(), self.defaultTask._getNumGoodPixels(mi, 'BAD'))
883 def test_edgeMasking(self):
884 """Check that the right number of edge pixels are masked by
885 _setEdgeBits().
886 """
887 testImage = self.flatExp.clone()
888 mi = testImage.maskedImage
890 self.assertEqual(countMaskedPixels(mi, 'EDGE'), 0)
891 self.defaultTask._setEdgeBits(mi)
893 hEdge = self.defaultConfig.nPixBorderLeftRight
894 vEdge = self.defaultConfig.nPixBorderUpDown
895 xSize, ySize = mi.getDimensions()
897 nEdge = xSize*vEdge*2 + ySize*hEdge*2 - hEdge*vEdge*4
899 self.assertEqual(countMaskedPixels(mi, 'EDGE'), nEdge)
901 def test_badImage(self):
902 """Check that fully-bad images do not fail.
903 """
904 testImage = self.flatExp.clone()
905 testImage.image.array[:, :] = 125000
907 config = copy.copy(self.defaultConfig)
908 # Do not exclude any pixels, so the areas match.
909 config.nPixBorderUpDown = 0
910 config.nPixBorderLeftRight = 0
912 task = self.defaultTask
913 task.config = config
914 defects = task._findHotAndColdPixels(testImage)
916 defectArea = 0
917 for defect in defects:
918 defectArea += defect.getBBox().getArea()
919 self.assertEqual(defectArea, testImage.getBBox().getArea())
922class TestMemory(lsst.utils.tests.MemoryTestCase):
923 pass
926def setup_module(module):
927 lsst.utils.tests.init()
930if __name__ == "__main__": 930 ↛ 931line 930 didn't jump to line 931, because the condition on line 930 was never true
931 lsst.utils.tests.init()
932 unittest.main()