Coverage for tests/test_actions.py: 14%
268 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-04 03:08 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-04 03:08 -0800
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import numpy as np
26import pandas as pd
27from lsst.analysis.tools.actions.scalar.scalarActions import (
28 ApproxFloor,
29 CountAction,
30 MeanAction,
31 MedianAction,
32 SigmaMadAction,
33 StdevAction,
34)
35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction
36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize
37from lsst.analysis.tools.actions.vector.selectors import (
38 CoaddPlotFlagSelector,
39 FlagSelector,
40 GalaxySelector,
41 RangeSelector,
42 SkyObjectSelector,
43 SnSelector,
44 StarSelector,
45 VectorSelector,
46)
47from lsst.analysis.tools.actions.vector.vectorActions import (
48 ConstantValue,
49 DivideVector,
50 DownselectVector,
51 ExtinctionCorrectedMagDiff,
52 FractionalDifference,
53 LoadVector,
54 MagColumnNanoJansky,
55 MagDiff,
56 SubtractVector,
57)
60class TestScalarActions(unittest.TestCase):
61 """ "Test ScalarActions"""
63 def setUp(self):
64 x = np.arange(100, dtype=float)
65 x[31] = np.nan
66 x[41] = np.nan
67 x[59] = np.nan
68 y = x**2
69 self.data = {"r_y": x, "i_y": y}
70 self.mask = np.zeros(100, dtype=bool)
71 self.mask[1:50] = 1
73 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
74 action = cls(vectorKey="{band}_y")
75 schema = [col for col, colType in action.getInputSchema()]
76 self.assertEqual(schema, ["{band}_y"])
77 result = action(self.data, band="i")
78 self.assertAlmostEqual(result, truth)
79 result = action(self.data, mask=self.mask, band="i")
80 self.assertAlmostEqual(result, maskedTruth)
82 def testMedianAction(self):
83 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
85 def testMeanAction(self):
86 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
88 def testStdevAction(self):
89 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
91 def testSigmaMadAction(self):
92 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
94 def testCountAction(self):
95 self._testScalarActionAlmostEqual(CountAction, 97, 47)
97 def testApproxFloorAction(self):
98 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
101class TestVectorActions(unittest.TestCase):
102 """Test VectorActions"""
104 def setUp(self):
105 self.size = 5
106 r = np.arange(self.size) + 1
107 i = r**2
108 rFlag = np.ones(self.size)
109 rFlag[1] = 0
110 iFlag = np.ones(self.size)
111 iFlag[3] = 0
112 data = {
113 "r_vector": r,
114 "i_vector": i,
115 "r_flag": rFlag,
116 "i_flag": iFlag,
117 "g_vector": 3 * r,
118 "E(B-V)": r[::-1],
119 "r_ixx": r**2,
120 "r_iyy": r[::-1] ** 2,
121 "r_ixy": r * r[::-1],
122 "g_iixx": r**2,
123 "g_iiyy": r[::-1] ** 2,
124 "g_iixy": r * r[::-1],
125 }
127 self.data = pd.DataFrame.from_dict(data)
129 def _checkSchema(self, action, truth):
130 schema = sorted([col for col, colType in action.getInputSchema()])
131 self.assertEqual(schema, truth)
133 def testDownselectVector(self):
134 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
135 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
136 result = action(self.data, band="r")
137 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
138 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
140 def testMagColumnNanoJansky(self):
141 truth = [
142 31.4,
143 29.89485002168,
144 29.0143937264,
145 28.38970004336,
146 27.90514997832,
147 ]
148 action = MagColumnNanoJansky(vectorKey="{band}_vector")
149 result = action(self.data, band="i")
150 self._checkSchema(action, ["{band}_vector"])
151 np.testing.assert_array_almost_equal(result, truth)
153 def testFractionalDifference(self):
154 actionA = LoadVector(vectorKey="{band1}_vector")
155 actionB = LoadVector(vectorKey="{band2}_vector")
156 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
157 diff = FractionalDifference(actionA=actionA, actionB=actionB)
158 result = diff(self.data, band1="r", band2="i")
159 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
160 np.testing.assert_array_almost_equal(result, truth)
162 def testConstant(self):
163 truth = [42.0]
164 action = ConstantValue(value=truth[0])
165 self._checkSchema(action, [])
166 result = action({})
167 np.testing.assert_array_equal(result, truth)
169 def testSubtract(self):
170 actionA = LoadVector(vectorKey="{band1}_vector")
171 actionB = LoadVector(vectorKey="{band2}_vector")
172 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
173 diff = SubtractVector(actionA=actionA, actionB=actionB)
174 result = diff(self.data, band1="r", band2="i")
175 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
176 np.testing.assert_array_almost_equal(result, truth)
178 def testDivide(self):
179 actionA = LoadVector(vectorKey="{band1}_vector")
180 actionB = LoadVector(vectorKey="{band2}_vector")
181 truth = 1 / np.arange(1, 6)
182 diff = DivideVector(actionA=actionA, actionB=actionB)
183 result = diff(self.data, band1="r", band2="i")
184 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
185 np.testing.assert_array_almost_equal(result, truth)
187 def testMagDiff(self):
188 # Use the same units as the data so that we know the difference exactly
189 # without conversions.
190 # testExtinctionCorrectedMagDiff will test the conversions
191 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
192 action = MagDiff(
193 col1="{band1}_vector",
194 col2="{band2}_vector",
195 fluxUnits1="mag(AB)",
196 fluxUnits2="mag(AB)",
197 returnMillimags=False,
198 )
199 result = action(self.data, band1="g", band2="r")
200 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
201 np.testing.assert_array_almost_equal(result, truth.value)
203 def testExtinctionCorrectedMagDiff(self):
204 for returnMillimags in (True, False):
205 # Check that conversions are working properly
206 magDiff = MagDiff(
207 col1="{band1}_vector",
208 col2="{band2}_vector",
209 fluxUnits1="jansky",
210 fluxUnits2="jansky",
211 returnMillimags=returnMillimags,
212 )
213 action = ExtinctionCorrectedMagDiff(
214 magDiff=magDiff,
215 band1="g",
216 band2="r",
217 ebvCol="E(B-V)",
218 extinctionCoeffs={"g": 0.2, "r": 1.5},
219 )
221 result = action(self.data, band1="g", band2="r")
222 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
223 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
224 diff = lhs - rhs
225 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
226 if returnMillimags:
227 diff = diff.to(u.mmag)
228 correction = correction.to(u.mmag)
229 truth = diff - correction
230 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
231 np.testing.assert_array_almost_equal(result, truth.value)
233 # Test with hard coded bands
234 magDiff = MagDiff(
235 col1="g_vector",
236 col2="r_vector",
237 fluxUnits1="jansky",
238 fluxUnits2="jansky",
239 returnMillimags=False,
240 )
241 action = ExtinctionCorrectedMagDiff(
242 magDiff=magDiff,
243 ebvCol="E(B-V)",
244 extinctionCoeffs={"g": 0.2, "r": 1.5},
245 )
246 result = action(self.data)
247 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
248 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
249 diff = lhs - rhs
250 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
251 truth = diff - correction
252 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
253 np.testing.assert_array_almost_equal(result, truth.value)
255 def testCalcBinnedStats(self):
256 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size + 1)
257 prefix = "a_"
258 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
259 result = stats(self.data)
260 median = (1 + self.size) / 2.0
261 truth = {
262 stats.name_mask: np.ones(self.size),
263 stats.name_median: median,
264 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)),
265 stats.name_count: self.size,
266 stats.name_select_maximum: self.size,
267 stats.name_select_median: median,
268 stats.name_select_minimum: 1,
269 "range_maximum": self.size + 1,
270 "range_minimum": 0,
271 }
272 self.assertEqual(list(result.keys()), list(truth.keys()))
274 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
275 del truth[stats.name_sigmaMad]
277 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
278 del truth[stats.name_mask]
280 for key, value in truth.items():
281 self.assertEqual(result[key], value, key)
283 def testCalcShapeSize(self):
284 xx = self.data["r_ixx"]
285 yy = self.data["r_iyy"]
286 xy = self.data["r_ixy"]
288 # Test determinant with defaults
289 action = CalcShapeSize()
290 result = action(self.data, band="r")
291 schema = [col for col, colType in action.getInputSchema()]
292 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
293 truth = 0.25 * (xx * yy - xy**2)
294 np.testing.assert_array_almost_equal(result, truth)
296 # Test trace with columns specified
297 action = CalcShapeSize(
298 colXx="{band}_iixx",
299 colYy="{band}_iiyy",
300 colXy="{band}_iixy",
301 sizeType="trace",
302 )
303 result = action(self.data, band="g")
304 schema = [col for col, colType in action.getInputSchema()]
305 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
306 truth = np.sqrt(0.5 * (xx + yy))
307 np.testing.assert_array_almost_equal(result, truth)
310class TestVectorSelectors(unittest.TestCase):
311 def setUp(self):
312 self.size = 20
313 falseFlags = {
314 "{band}_psfFlux_flag": [1],
315 "{band}_pixelFlags_saturatedCenter": [3],
316 "{band}_extendedness_flag": [5],
317 "xy_flag": [7],
318 "i_pixelFlags_edge": [13],
319 "r_pixelFlags_edge": [15],
320 "sky_object": [13, 15, 17],
321 }
323 trueFlags = {
324 "detect_isPatchInner": [9],
325 "detect_isDeblendedSource": [11],
326 }
328 flux = np.arange(self.size) * 10
329 fluxErr = np.ones(self.size) * 0.1
330 extendedness = np.arange(20) / 20 - 0.1
332 self.data = {
333 "r_psfFlux": flux,
334 "r_psfFluxErr": fluxErr,
335 "i_cmodelFlux": flux[::-1],
336 "i_cmodelFluxError": fluxErr[::-1],
337 "r_cmodelFlux": flux,
338 "r_cmodelFluxError": fluxErr,
339 "i_extendedness": extendedness,
340 "i_extended": extendedness,
341 }
342 bands = ("r", "i")
343 for band in bands:
344 for flag, bits in falseFlags.items():
345 vector = np.zeros(self.size, dtype=bool)
346 for bit in bits:
347 vector[bit] = 1
348 self.data[flag.format(band=band)] = vector
350 for flag, bits in trueFlags.items():
351 vector = np.ones(self.size, dtype=bool)
352 for bit in bits:
353 vector[bit] = 0
354 self.data[flag.format(band=band)] = vector
356 def _checkSchema(self, action, truth):
357 schema = [col for col, colType in action.getInputSchema()]
358 self.assertEqual(sorted(schema), sorted(truth))
360 def testFlagSelector(self):
361 selector = FlagSelector(
362 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
363 )
364 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
365 result = selector(self.data, band="r")
366 truth = np.ones(self.size, dtype=bool)
367 truth[1] = False
368 truth[9] = False
369 np.testing.assert_array_equal(result, truth)
371 def testCoaddPlotFlagSelector(self):
372 # Test defaults
373 # Bands needs to be set to something otherwise it
374 # will crash as the default value looks it up
375 # elsewhere.
376 selector = CoaddPlotFlagSelector(bands=["i"])
377 keys = [
378 "{band}_psfFlux_flag",
379 "{band}_pixelFlags_saturatedCenter",
380 "{band}_extendedness_flag",
381 "xy_flag",
382 "detect_isPatchInner",
383 "detect_isDeblendedSource",
384 ]
385 self._checkSchema(selector, keys)
387 result = selector(self.data)
388 truth = np.ones(self.size, dtype=bool)
389 for bit in (1, 3, 5, 7, 9, 11):
390 truth[bit] = 0
391 np.testing.assert_array_equal(result, truth)
393 # Test bands override
394 selector = CoaddPlotFlagSelector(
395 bands=["i", "r"],
396 selectWhenFalse=["{band}_psfFlux_flag"],
397 selectWhenTrue=["detect_isDeblendedSource"],
398 )
399 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
400 result = selector(self.data)
401 truth = np.ones(self.size, dtype=bool)
402 for bit in (1, 11):
403 truth[bit] = 0
404 np.testing.assert_array_equal(result, truth)
406 def testRangeSelector(self):
407 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
408 self._checkSchema(selector, ["r_psfFlux"])
409 result = self.data["r_psfFlux"][selector(self.data)]
410 truth = [30, 40]
411 np.testing.assert_array_equal(result, truth)
413 def testSnSelector(self):
414 # test defaults
415 selector = SnSelector()
416 keys = [
417 "{band}_psfFlux",
418 "{band}_psfFluxErr",
419 ]
420 self._checkSchema(selector, keys)
421 result = selector(self.data, bands=["r"])
422 truth = np.ones(self.size, dtype=bool)
423 truth[:6] = 0
424 np.testing.assert_array_equal(result, truth)
426 # test overrides
427 selector = SnSelector(
428 fluxType="{band}_cmodelFlux",
429 threshold=200.0,
430 uncertaintySuffix="Error",
431 bands=["r", "i"],
432 )
433 keys = [
434 "{band}_cmodelFlux",
435 "{band}_cmodelFluxError",
436 ]
437 self._checkSchema(selector, keys)
438 result = selector(self.data)
439 truth = np.ones(self.size, dtype=bool)
440 truth[:3] = 0
441 truth[-3:] = 0
442 np.testing.assert_array_equal(result, truth)
444 def testSkyObjectSelector(self):
445 # Test default
446 selector = SkyObjectSelector()
447 keys = ["{band}_pixelFlags_edge", "sky_object"]
448 self._checkSchema(selector, keys)
449 result = selector(self.data)
450 truth = np.zeros(self.size, dtype=bool)
451 truth[15] = 1
452 truth[17] = 1
453 np.testing.assert_array_equal(result, truth)
455 # Test overrides
456 selector = SkyObjectSelector(bands=["i", "r"])
457 self._checkSchema(selector, keys)
458 result = selector(self.data)
459 truth = np.zeros(self.size, dtype=bool)
460 truth[17] = 1
461 np.testing.assert_array_equal(result, truth)
463 def testStarSelector(self):
464 # test default
465 selector = StarSelector()
466 self._checkSchema(selector, ["{band}_extendedness"])
467 result = selector(self.data, band="i")
468 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
469 np.testing.assert_array_almost_equal(result, truth)
471 # Test overrides
472 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
473 result = selector(self.data, band="i")
474 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
475 np.testing.assert_array_almost_equal(result, truth)
477 def testGalaxySelector(self):
478 # test default
479 selector = GalaxySelector()
480 self._checkSchema(selector, ["{band}_extendedness"])
481 result = selector(self.data, band="i")
482 truth = self.data["i_extendedness"] > 0.5
483 np.testing.assert_array_almost_equal(result, truth)
485 # Test overrides
486 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
487 result = selector(self.data, band="i")
488 truth = self.data["i_extendedness"] > 0.3
489 np.testing.assert_array_almost_equal(result, truth)
491 def testVectorSelector(self):
492 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
493 self._checkSchema(selector, ["{band}_psfFlux_flag"])
494 result = selector(self.data, band="i")
495 truth = np.zeros(self.size, dtype=bool)
496 truth[1] = True
497 np.testing.assert_array_equal(result, truth)