Coverage for tests/test_imageMapReduce.py: 15%
283 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-14 09:21 +0000
1#
2# LSST Data Management System
3# Copyright 2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
22import unittest
23import numpy as np
25import lsst.utils.tests
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.afw.geom as afwGeom
29import lsst.daf.base as dafBase
30import lsst.geom as geom
31import lsst.meas.algorithms as measAlg
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
35from lsst.ip.diffim.imageMapReduce import (ImageMapReduceTask, ImageMapReduceConfig,
36 ImageMapper, ImageMapperConfig)
39def setup_module(module):
40 lsst.utils.tests.init()
43def makeWcs(offset=0):
44 # taken from $AFW_DIR/tests/testMakeWcs.py
45 metadata = dafBase.PropertySet()
46 metadata.set("SIMPLE", "T")
47 metadata.set("BITPIX", -32)
48 metadata.set("NAXIS", 2)
49 metadata.set("NAXIS1", 1024)
50 metadata.set("NAXIS2", 1153)
51 metadata.set("RADESYS", 'FK5')
52 metadata.set("EQUINOX", 2000.)
53 metadata.setDouble("CRVAL1", 215.604025685476)
54 metadata.setDouble("CRVAL2", 53.1595451514076)
55 metadata.setDouble("CRPIX1", 1109.99981456774 + offset)
56 metadata.setDouble("CRPIX2", 560.018167811613 + offset)
57 metadata.set("CTYPE1", 'RA---SIN')
58 metadata.set("CTYPE2", 'DEC--SIN')
59 metadata.setDouble("CD1_1", 5.10808596133527E-05)
60 metadata.setDouble("CD1_2", 1.85579539217196E-07)
61 metadata.setDouble("CD2_2", -5.10281493481982E-05)
62 metadata.setDouble("CD2_1", -8.27440751733828E-07)
63 return afwGeom.makeSkyWcs(metadata)
66def getPsfMoments(psfArray):
67 # Borrowed and modified from meas_algorithms/testCoaddPsf
68 sumx2 = sumy2 = sumy = sumx = sumf = 0.0
69 for x in range(psfArray.shape[0]):
70 for y in range(psfArray.shape[1]):
71 f = psfArray[x, y]
72 sumx2 += x*x*f
73 sumy2 += y*y*f
74 sumx += x*f
75 sumy += y*f
76 sumf += f
77 xbar = sumx/sumf
78 ybar = sumy/sumf
79 mxx = sumx2 - 2*xbar*sumx + xbar*xbar*sumf
80 myy = sumy2 - 2*ybar*sumy + ybar*ybar*sumf
81 return sumf, xbar, ybar, mxx, myy
84def getPsfSecondMoments(psfArray):
85 sum, xbar, ybar, mxx, myy = getPsfMoments(psfArray)
86 return mxx, myy
89class AddAmountImageMapperConfig(ImageMapperConfig):
90 """Configuration parameters for the AddAmountImageMapper
91 """
92 addAmount = pexConfig.Field(
93 dtype=float,
94 doc="Amount to add to image",
95 default=10.
96 )
99class AddAmountImageMapper(ImageMapper):
100 """Image mapper subTask that adds a constant value to the input subexposure
101 """
102 ConfigClass = AddAmountImageMapperConfig
103 _DefaultName = "ip_diffim_AddAmountImageMapper"
105 def run(self, subExposure, expandedSubExp, fullBBox, addNans=False, **kwargs):
106 """Add `addAmount` to given `subExposure`.
108 Optionally add NaNs to check the NaN-safe 'copy' operation.
110 Parameters
111 ----------
112 subExposure : `afwImage.Exposure`
113 Input `subExposure` upon which to operate
114 expandedSubExp : `afwImage.Exposure`
115 Input expanded subExposure (not used here)
116 fullBBox : `lsst.geom.Box2I`
117 Bounding box of original exposure (not used here)
118 addNaNs : boolean
119 Set a single pixel of `subExposure` to `np.nan`
120 kwargs
121 Arbitrary keyword arguments (ignored)
123 Returns
124 -------
125 `pipeBase.Struct` containing (with name 'subExposure') the
126 copy of `subExposure` to which `addAmount` has been added
127 """
128 subExp = subExposure.clone()
129 img = subExp.maskedImage
130 img += self.config.addAmount
131 if addNans:
132 img.image.array[0, 0] = np.nan
133 return pipeBase.Struct(subExposure=subExp)
136class AddAmountImageMapReduceConfig(ImageMapReduceConfig):
137 """Configuration parameters for the AddAmountImageMapReduceTask
138 """
139 mapper = pexConfig.ConfigurableField(
140 doc="Mapper subtask to run on each subimage",
141 target=AddAmountImageMapper,
142 )
145class GetMeanImageMapper(ImageMapper):
146 """ImageMapper subtask that computes and returns the mean value of the
147 input sub-exposure
148 """
149 ConfigClass = AddAmountImageMapperConfig # Doesn't need its own config
150 _DefaultName = "ip_diffim_GetMeanImageMapper"
152 def run(self, subExposure, expandedSubExp, fullBBox, **kwargs):
153 """Compute the mean of the given `subExposure`
155 Parameters
156 ----------
157 subExposure : `afwImage.Exposure`
158 Input `subExposure` upon which to operate
159 expandedSubExp : `afwImage.Exposure`
160 Input expanded subExposure (not used here)
161 fullBBox : `lsst.geom.Box2I`
162 Bounding box of original exposure (not used here)
163 kwargs
164 Arbitrary keyword arguments (ignored)
166 Returns
167 -------
168 `pipeBase.Struct` containing the mean value of `subExposure`
169 image plane. We name it 'subExposure' to enable the correct
170 test in `testNotNoneReduceWithNonExposureMapper`. In real
171 operations, use something like 'mean' for the name.
172 """
173 subMI = subExposure.maskedImage
174 statObj = afwMath.makeStatistics(subMI, afwMath.MEAN)
175 return pipeBase.Struct(subExposure=statObj.getValue())
178class GetMeanImageMapReduceConfig(ImageMapReduceConfig):
179 """Configuration parameters for the GetMeanImageMapReduceTask
180 """
181 mapper = pexConfig.ConfigurableField(
182 doc="Mapper subtask to run on each subimage",
183 target=GetMeanImageMapper,
184 )
187class ImageMapReduceTest(lsst.utils.tests.TestCase):
188 """A test case for the image gridded processing task
189 """
190 def setUp(self):
191 self.longMessage = True
192 self._makeImage()
194 def tearDown(self):
195 del self.exposure
197 def _makeImage(self):
198 self.exposure = afwImage.ExposureF(128, 128)
199 self.exposure.setPsf(measAlg.DoubleGaussianPsf(11, 11, 2.0, 3.7))
200 mi = self.exposure.maskedImage
201 mi.set(0.)
202 self.exposure.setWcs(makeWcs()) # required for PSF construction via CoaddPsf
204 def testCopySumNoOverlaps(self):
205 self._testCopySumNoOverlaps(reduceOp='copy', withNaNs=False)
206 self._testCopySumNoOverlaps(reduceOp='copy', withNaNs=True)
207 self._testCopySumNoOverlaps(reduceOp='sum', withNaNs=False)
208 self._testCopySumNoOverlaps(reduceOp='sum', withNaNs=True)
210 def _testCopySumNoOverlaps(self, reduceOp='copy', withNaNs=False):
211 """Test sample grid task that adds 5.0 to input image and uses
212 `reduceOperation = 'copy'`. Optionally add NaNs to subimages.
213 """
214 config = AddAmountImageMapReduceConfig()
215 task = ImageMapReduceTask(config)
216 config.mapper.addAmount = 5.
217 config.reducer.reduceOperation = reduceOp
218 newExp = task.run(self.exposure, addNans=withNaNs).exposure
219 newMI = newExp.maskedImage
220 newArr = newMI.image.array
221 isnan = np.isnan(newArr)
222 if not withNaNs:
223 self.assertEqual(np.sum(isnan), 0,
224 msg='Failed on withNaNs: %s' % str(withNaNs))
226 mi = self.exposure.image.array
227 if reduceOp != 'sum':
228 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.,
229 msg='Failed on withNaNs: %s' % str(withNaNs))
230 else: # We don't construct a new PSF if reduceOperation == 'copy'.
231 self._testCoaddPsf(newExp)
233 def testAverageWithOverlaps(self):
234 self._testAverageWithOverlaps(withNaNs=False)
235 self._testAverageWithOverlaps(withNaNs=True)
237 def _testAverageWithOverlaps(self, withNaNs=False):
238 """Test sample grid task that adds 5.0 to input image and uses
239 'average' `reduceOperation`. Optionally add NaNs to subimages.
240 """
241 config = AddAmountImageMapReduceConfig()
242 config.gridStepX = config.gridStepY = 8.
243 config.reducer.reduceOperation = 'average'
244 task = ImageMapReduceTask(config)
245 config.mapper.addAmount = 5.
246 newExp = task.run(self.exposure, addNans=withNaNs).exposure
247 newMI = newExp.maskedImage
248 newArr = newMI.image.array
249 mi = self.exposure.maskedImage
250 isnan = np.isnan(newArr)
251 if not withNaNs:
252 self.assertEqual(np.sum(isnan), 0,
253 msg='Failed on withNaNs: %s' % str(withNaNs))
255 mi = self.exposure.image.array
256 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.,
257 msg='Failed on withNaNs: %s' % str(withNaNs))
258 self._testCoaddPsf(newExp)
260 def _testCoaddPsf(self, newExposure):
261 """Test that the new CoaddPsf of the `newExposure` returns PSF images
262 ~identical to the input PSF of `self.exposure` across a grid
263 covering the entire exposure bounding box.
264 """
265 origPsf = self.exposure.getPsf()
266 newPsf = newExposure.getPsf()
267 self.assertTrue(isinstance(newPsf, measAlg.CoaddPsf))
268 extentX = int(self.exposure.getWidth()*0.05)
269 extentY = int(self.exposure.getHeight()*0.05)
270 for x in np.linspace(extentX, self.exposure.getWidth()-extentX, 10):
271 for y in np.linspace(extentY, self.exposure.getHeight()-extentY, 10):
272 point = geom.Point2D(np.rint(x), np.rint(y))
273 oPsf = origPsf.computeImage(point).array
274 nPsf = newPsf.computeImage(point).array
275 if oPsf.shape[0] < nPsf.shape[0]: # sometimes CoaddPsf does this.
276 oPsf = np.pad(oPsf, ((1, 1), (0, 0)), mode='constant')
277 elif oPsf.shape[0] > nPsf.shape[0]:
278 nPsf = np.pad(nPsf, ((1, 1), (0, 0)), mode='constant')
279 if oPsf.shape[1] < nPsf.shape[1]: # sometimes CoaddPsf does this.
280 oPsf = np.pad(oPsf, ((0, 0), (1, 1)), mode='constant')
281 elif oPsf.shape[1] > nPsf.shape[1]:
282 nPsf = np.pad(nPsf, ((0, 0), (1, 1)), mode='constant')
283 # pixel-wise comparison -- pretty stringent
284 self.assertFloatsAlmostEqual(oPsf, nPsf, atol=1e-4, msg='Failed on Psf')
286 origMmts = np.array(getPsfSecondMoments(oPsf))
287 newMmts = np.array(getPsfSecondMoments(nPsf))
288 self.assertFloatsAlmostEqual(origMmts, newMmts, atol=1e-4, msg='Failed on Psf')
290 def testAverageVersusCopy(self):
291 self._testAverageVersusCopy(withNaNs=False)
292 self._testAverageVersusCopy(withNaNs=True)
294 def _testAverageVersusCopy(self, withNaNs=False):
295 """Re-run `testExampleTaskNoOverlaps` and `testExampleTaskWithOverlaps`
296 on a more complex image (with random noise). Ensure that the results are
297 identical (within between 'copy' and 'average' reduceOperation.
298 """
299 exposure1 = self.exposure.clone()
300 img = exposure1.image
301 afwMath.randomGaussianImage(img, afwMath.Random())
302 exposure2 = exposure1.clone()
304 config = AddAmountImageMapReduceConfig()
305 task = ImageMapReduceTask(config)
306 config.mapper.addAmount = 5.
307 newExp = task.run(exposure1, addNans=withNaNs).exposure
308 newMI1 = newExp.maskedImage
310 config.gridStepX = config.gridStepY = 8.
311 config.reducer.reduceOperation = 'average'
312 task = ImageMapReduceTask(config)
313 newExp = task.run(exposure2, addNans=withNaNs).exposure
314 newMI2 = newExp.maskedImage
316 newMA1 = newMI1.image.array
317 isnan = np.isnan(newMA1)
318 if not withNaNs:
319 self.assertEqual(np.sum(isnan), 0)
320 newMA2 = newMI2.image.array
322 # Because the average uses a float accumulator, we can have differences, set a tolerance.
323 # Turns out (in practice for this test), only 7 pixels seem to have a small difference.
324 self.assertFloatsAlmostEqual(newMA1[~isnan], newMA2[~isnan], rtol=1e-7)
326 def testMean(self):
327 """Test sample grid task that returns the mean of the subimages and uses
328 'none' `reduceOperation`.
329 """
330 config = GetMeanImageMapReduceConfig()
331 config.reducer.reduceOperation = 'none'
332 task = ImageMapReduceTask(config)
333 testExposure = self.exposure.clone()
334 testExposure.maskedImage.set(1.234)
335 subMeans = task.run(testExposure).result
336 subMeans = [x.subExposure for x in subMeans]
338 self.assertEqual(len(subMeans), len(task.boxes0))
339 firstPixel = testExposure.image.array[0, 0]
340 self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
342 def testCellCentroids(self):
343 """Test sample grid task which is provided a set of `cellCentroids` and
344 returns the mean of the subimages surrounding those centroids using 'none'
345 for `reduceOperation`.
346 """
347 config = GetMeanImageMapReduceConfig()
348 config.gridStepX = config.gridStepY = 8.
349 config.reducer.reduceOperation = 'none'
350 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
351 config.cellCentroidsY = config.cellCentroidsX
352 task = ImageMapReduceTask(config)
353 testExposure = self.exposure.clone()
354 testExposure.maskedImage.set(1.234)
355 subMeans = task.run(testExposure).result
356 subMeans = [x.subExposure for x in subMeans]
358 self.assertEqual(len(subMeans), len(config.cellCentroidsX))
359 firstPixel = testExposure.image.array[0, 0]
360 self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
362 def testCellCentroidsWrongLength(self):
363 """Test sample grid task which is provided a set of `cellCentroids` and
364 returns the mean of the subimages surrounding those centroids using 'none'
365 for `reduceOperation`. In this case, we ensure that len(task.boxes0) !=
366 len(task.boxes1) and check for ValueError.
367 """
368 config = GetMeanImageMapReduceConfig()
369 config.reducer.reduceOperation = 'none'
370 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
371 config.cellCentroidsY = [i for i in np.linspace(0, 128, 50)]
372 task = ImageMapReduceTask(config)
373 task._generateGrid(self.exposure)
374 del task.boxes0[-1] # remove the last box
375 with self.assertRaises(ValueError):
376 task.run(self.exposure)
378 def testMasks(self):
379 """Test the mask for an exposure produced by a sample grid task
380 where we provide a set of `cellCentroids` and thus should have
381 many invalid pixels.
382 """
383 config = AddAmountImageMapReduceConfig()
384 config.gridStepX = config.gridStepY = 8.
385 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
386 config.cellCentroidsY = config.cellCentroidsX
387 config.reducer.reduceOperation = 'average'
388 task = ImageMapReduceTask(config)
389 config.mapper.addAmount = 5.
390 newExp = task.run(self.exposure).exposure
391 newMI = newExp.maskedImage
392 newArr = newMI.image.array
393 mi = self.exposure.maskedImage
394 isnan = np.isnan(newArr)
395 self.assertGreater(np.sum(isnan), 1000)
397 mi = self.exposure.image.array
398 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.)
400 mask = newMI.mask # Now check the mask
401 self.assertGreater(mask.getMaskPlane('INVALID_MAPREDUCE'), 0)
402 maskBit = mask.getPlaneBitMask('INVALID_MAPREDUCE')
403 nMasked = np.sum(np.bitwise_and(mask.array, maskBit) != 0)
404 self.assertGreater(nMasked, 1000)
405 self.assertEqual(np.sum(np.isnan(newArr)), nMasked)
407 def testNotNoneReduceWithNonExposureMapper(self):
408 """Test that a combination of a mapper that returns a non-exposure
409 cannot work correctly with a reducer with reduceOperation='none'.
410 Should raise a TypeError.
411 """
412 config = GetMeanImageMapReduceConfig() # mapper returns a float (mean)
413 config.gridStepX = config.gridStepY = 8.
414 config.reducer.reduceOperation = 'average' # not 'none'!
415 task = ImageMapReduceTask(config)
416 with self.assertRaises(TypeError):
417 task.run(self.exposure)
419 def testGridValidity(self):
420 """Test sample grids with various spacings and sizes and other options.
421 """
422 expectedVal = 1.
423 n_tests = 0
425 for reduceOp in ('copy', 'average'):
426 for adjustGridOption in ('spacing', 'size', 'none'):
427 for gstepx in range(11, 3, -4):
428 for gsizex in gstepx + np.array([0, 1, 2]):
429 for gstepy in range(11, 3, -4):
430 for gsizey in gstepy + np.array([0, 1, 2]):
431 config = AddAmountImageMapReduceConfig()
432 config.reducer.reduceOperation = reduceOp
433 n_tests += 1
434 self._runGridValidity(config, gstepx, gsizex,
435 gstepy, gsizey, adjustGridOption,
436 expectedVal)
437 print("Ran a total of %d grid validity tests." % n_tests)
439 def _runGridValidity(self, config, gstepx, gsizex, gstepy, gsizey,
440 adjustGridOption, expectedVal=1.):
441 """Method to test the grid validity given an input config.
443 Here we also iterate over scaleByFwhm in (True, False) and
444 ensure that we get more `boxes` when `scaleByFwhm=False` than
445 vice versa.
447 Parameters
448 ----------
449 config : `ipDiffim.AddAmountImageMapReduceConfig`
450 input AddAmountImageMapReduceConfig
451 gstepx : `float`
452 grid x-direction step size
453 gsizex : `float`
454 grid x-direction box size
455 gstepy : `float`
456 grid y-direction step size
457 gsizey : `float`
458 grid y-direction box size
459 expectedVal : `float`
460 float to add to exposure (to compare for testing)
461 """
462 config.mapper.addAmount = expectedVal
463 lenBoxes = [0, 0]
464 for scaleByFwhm in (True, False):
465 config.scaleByFwhm = scaleByFwhm
466 if scaleByFwhm:
467 config.gridStepX = float(gstepx)
468 config.cellSizeX = float(gsizex)
469 config.gridStepY = float(gstepy)
470 config.cellSizeY = float(gsizey)
471 else: # otherwise the grid is too fine and elements too small.
472 config.gridStepX = gstepx * 3.
473 config.cellSizeX = gsizex * 3.
474 config.gridStepY = gstepy * 3.
475 config.cellSizeY = gsizey * 3.
476 config.adjustGridOption = adjustGridOption
477 task = ImageMapReduceTask(config)
478 task._generateGrid(self.exposure)
479 ind = 0 if scaleByFwhm else 1
480 lenBoxes[ind] = len(task.boxes0)
481 newExp = task.run(self.exposure).exposure
482 newMI = newExp.maskedImage
483 newArr = newMI.image.array
484 isnan = np.isnan(newArr)
485 self.assertEqual(np.sum(isnan), 0, msg='Failed NaN (%d), on config: %s' %
486 (np.sum(isnan), str(config)))
488 mi = self.exposure.image.array
489 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - expectedVal,
490 msg='Failed on config: %s' % str(config))
492 self.assertLess(lenBoxes[0], lenBoxes[1], msg='Failed lengths on config: %s' %
493 str(config))
496class MemoryTester(lsst.utils.tests.MemoryTestCase):
497 pass
500if __name__ == "__main__": 500 ↛ 501line 500 didn't jump to line 501, because the condition on line 500 was never true
501 lsst.utils.tests.init()
502 unittest.main()