Coverage for tests/test_actions.py: 14%
268 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-05 00:58 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-05 00:58 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import numpy as np
26import pandas as pd
27from lsst.analysis.tools.actions.scalar.scalarActions import (
28 ApproxFloor,
29 CountAction,
30 MeanAction,
31 MedianAction,
32 SigmaMadAction,
33 StdevAction,
34)
35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction
36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize
37from lsst.analysis.tools.actions.vector.selectors import (
38 CoaddPlotFlagSelector,
39 FlagSelector,
40 GalaxySelector,
41 RangeSelector,
42 SkyObjectSelector,
43 SnSelector,
44 StarSelector,
45 VectorSelector,
46)
47from lsst.analysis.tools.actions.vector.vectorActions import (
48 ConstantValue,
49 DivideVector,
50 DownselectVector,
51 ExtinctionCorrectedMagDiff,
52 FractionalDifference,
53 LoadVector,
54 MagColumnNanoJansky,
55 MagDiff,
56 SubtractVector,
57)
60class TestScalarActions(unittest.TestCase):
61 """ "Test ScalarActions"""
63 def setUp(self):
64 x = np.arange(100, dtype=float)
65 x[31] = np.nan
66 x[41] = np.nan
67 x[59] = np.nan
68 y = x**2
69 self.data = {"r_y": x, "i_y": y}
70 self.mask = np.zeros(100, dtype=bool)
71 self.mask[1:50] = 1
73 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
74 action = cls(vectorKey="{band}_y")
75 schema = [col for col, colType in action.getInputSchema()]
76 self.assertEqual(schema, ["{band}_y"])
77 result = action(self.data, band="i")
78 self.assertAlmostEqual(result, truth)
79 result = action(self.data, mask=self.mask, band="i")
80 self.assertAlmostEqual(result, maskedTruth)
82 def testMedianAction(self):
83 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
85 def testMeanAction(self):
86 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
88 def testStdevAction(self):
89 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
91 def testSigmaMadAction(self):
92 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
94 def testCountAction(self):
95 self._testScalarActionAlmostEqual(CountAction, 97, 47)
97 def testApproxFloorAction(self):
98 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
101class TestVectorActions(unittest.TestCase):
102 """Test VectorActions"""
104 def setUp(self):
105 self.size = 5
106 r = np.arange(self.size) + 1
107 i = r**2
108 rFlag = np.ones(self.size)
109 rFlag[1] = 0
110 iFlag = np.ones(self.size)
111 iFlag[3] = 0
112 data = {
113 "r_vector": r,
114 "i_vector": i,
115 "r_flag": rFlag,
116 "i_flag": iFlag,
117 "g_vector": 3 * r,
118 "E(B-V)": r[::-1],
119 "r_ixx": r**2,
120 "r_iyy": r[::-1] ** 2,
121 "r_ixy": r * r[::-1],
122 "g_iixx": r**2,
123 "g_iiyy": r[::-1] ** 2,
124 "g_iixy": r * r[::-1],
125 }
127 self.data = pd.DataFrame.from_dict(data)
129 def _checkSchema(self, action, truth):
130 schema = sorted([col for col, colType in action.getInputSchema()])
131 self.assertEqual(schema, truth)
133 def testDownselectVector(self):
134 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
135 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
136 result = action(self.data, band="r")
137 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
138 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
140 def testMagColumnNanoJansky(self):
141 truth = [
142 31.4,
143 29.89485002168,
144 29.0143937264,
145 28.38970004336,
146 27.90514997832,
147 ]
148 action = MagColumnNanoJansky(vectorKey="{band}_vector")
149 result = action(self.data, band="i")
150 self._checkSchema(action, ["{band}_vector"])
151 np.testing.assert_array_almost_equal(result, truth)
153 def testFractionalDifference(self):
154 actionA = LoadVector(vectorKey="{band1}_vector")
155 actionB = LoadVector(vectorKey="{band2}_vector")
156 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
157 diff = FractionalDifference(actionA=actionA, actionB=actionB)
158 result = diff(self.data, band1="r", band2="i")
159 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
160 np.testing.assert_array_almost_equal(result, truth)
162 def testConstant(self):
163 truth = [42.0]
164 action = ConstantValue(value=truth[0])
165 self._checkSchema(action, [])
166 result = action({})
167 np.testing.assert_array_equal(result, truth)
169 def testSubtract(self):
170 actionA = LoadVector(vectorKey="{band1}_vector")
171 actionB = LoadVector(vectorKey="{band2}_vector")
172 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
173 diff = SubtractVector(actionA=actionA, actionB=actionB)
174 result = diff(self.data, band1="r", band2="i")
175 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
176 np.testing.assert_array_almost_equal(result, truth)
178 def testDivide(self):
179 actionA = LoadVector(vectorKey="{band1}_vector")
180 actionB = LoadVector(vectorKey="{band2}_vector")
181 truth = 1 / np.arange(1, 6)
182 diff = DivideVector(actionA=actionA, actionB=actionB)
183 result = diff(self.data, band1="r", band2="i")
184 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
185 np.testing.assert_array_almost_equal(result, truth)
187 def testMagDiff(self):
188 # Use the same units as the data so that we know the difference exactly
189 # without conversions.
190 # testExtinctionCorrectedMagDiff will test the conversions
191 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
192 action = MagDiff(
193 col1="{band1}_vector",
194 col2="{band2}_vector",
195 fluxUnits1="mag(AB)",
196 fluxUnits2="mag(AB)",
197 returnMillimags=False,
198 )
199 result = action(self.data, band1="g", band2="r")
200 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
201 np.testing.assert_array_almost_equal(result, truth.value)
203 def testExtinctionCorrectedMagDiff(self):
204 for returnMillimags in (True, False):
205 # Check that conversions are working properly
206 magDiff = MagDiff(
207 col1="{band1}_vector",
208 col2="{band2}_vector",
209 fluxUnits1="jansky",
210 fluxUnits2="jansky",
211 returnMillimags=returnMillimags,
212 )
213 action = ExtinctionCorrectedMagDiff(
214 magDiff=magDiff,
215 band1="g",
216 band2="r",
217 ebvCol="E(B-V)",
218 extinctionCoeffs={"g": 0.2, "r": 1.5},
219 )
221 result = action(self.data, band1="g", band2="r")
222 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
223 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
224 diff = lhs - rhs
225 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
226 if returnMillimags:
227 diff = diff.to(u.mmag)
228 correction = correction.to(u.mmag)
229 truth = diff - correction
230 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
231 np.testing.assert_array_almost_equal(result, truth.value)
233 # Test with hard coded bands
234 magDiff = MagDiff(
235 col1="g_vector",
236 col2="r_vector",
237 fluxUnits1="jansky",
238 fluxUnits2="jansky",
239 returnMillimags=False,
240 )
241 action = ExtinctionCorrectedMagDiff(
242 magDiff=magDiff,
243 ebvCol="E(B-V)",
244 extinctionCoeffs={"g": 0.2, "r": 1.5},
245 )
246 result = action(self.data)
247 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
248 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
249 diff = lhs - rhs
250 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
251 truth = diff - correction
252 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
253 np.testing.assert_array_almost_equal(result, truth.value)
255 def testCalcBinnedStats(self):
256 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size + 1)
257 prefix = "a_"
258 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
259 result = stats(self.data)
260 median = (1 + self.size) / 2.0
261 truth = {
262 stats.name_mask: np.ones(self.size),
263 stats.name_median: median,
264 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)),
265 stats.name_count: self.size,
266 stats.name_select_maximum: self.size,
267 stats.name_select_median: median,
268 stats.name_select_minimum: 1,
269 "range_maximum": self.size + 1,
270 "range_minimum": 0,
271 }
272 self.assertEqual(list(result.keys()), list(truth.keys()))
274 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
275 del truth[stats.name_sigmaMad]
277 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
278 del truth[stats.name_mask]
280 for key, value in truth.items():
281 self.assertEqual(result[key], value, key)
283 def testCalcShapeSize(self):
284 xx = self.data["r_ixx"]
285 yy = self.data["r_iyy"]
286 xy = self.data["r_ixy"]
288 # Test determinant with defaults
289 action = CalcShapeSize()
290 result = action(self.data, band="r")
291 schema = [col for col, colType in action.getInputSchema()]
292 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
293 truth = 0.25 * (xx * yy - xy**2)
294 np.testing.assert_array_almost_equal(result, truth)
296 # Test trace with columns specified
297 action = CalcShapeSize(
298 colXx="{band}_iixx",
299 colYy="{band}_iiyy",
300 colXy="{band}_iixy",
301 sizeType="trace",
302 )
303 result = action(self.data, band="g")
304 schema = [col for col, colType in action.getInputSchema()]
305 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
306 truth = np.sqrt(0.5 * (xx + yy))
307 np.testing.assert_array_almost_equal(result, truth)
310class TestVectorSelectors(unittest.TestCase):
311 def setUp(self):
312 self.size = 20
313 falseFlags = {
314 "{band}_psfFlux_flag": [1],
315 "{band}_pixelFlags_saturatedCenter": [3],
316 "{band}_extendedness_flag": [5],
317 "xy_flag": [7],
318 "i_pixelFlags_edge": [13],
319 "r_pixelFlags_edge": [15],
320 "sky_object": [13, 15, 17],
321 }
323 trueFlags = {
324 "detect_isPatchInner": [9],
325 "detect_isDeblendedSource": [11],
326 }
328 flux = np.arange(self.size) * 10
329 fluxErr = np.ones(self.size) * 0.1
330 extendedness = np.arange(20) / 20 - 0.1
332 self.data = {
333 "r_psfFlux": flux,
334 "r_psfFluxErr": fluxErr,
335 "i_cmodelFlux": flux[::-1],
336 "i_cmodelFluxError": fluxErr[::-1],
337 "r_cmodelFlux": flux,
338 "r_cmodelFluxError": fluxErr,
339 "i_extendedness": extendedness,
340 "i_extended": extendedness,
341 }
342 bands = ("r", "i")
343 for band in bands:
344 for flag, bits in falseFlags.items():
345 vector = np.zeros(self.size, dtype=bool)
346 for bit in bits:
347 vector[bit] = 1
348 self.data[flag.format(band=band)] = vector
350 for flag, bits in trueFlags.items():
351 vector = np.ones(self.size, dtype=bool)
352 for bit in bits:
353 vector[bit] = 0
354 self.data[flag.format(band=band)] = vector
356 def _checkSchema(self, action, truth):
357 schema = [col for col, colType in action.getInputSchema()]
358 self.assertEqual(sorted(schema), sorted(truth))
360 def testFlagSelector(self):
361 selector = FlagSelector(
362 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
363 )
364 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
365 result = selector(self.data, band="r")
366 truth = np.ones(self.size, dtype=bool)
367 truth[1] = False
368 truth[9] = False
369 np.testing.assert_array_equal(result, truth)
371 def testCoaddPlotFlagSelector(self):
372 # Test defaults
373 selector = CoaddPlotFlagSelector()
374 keys = [
375 "{band}_psfFlux_flag",
376 "{band}_pixelFlags_saturatedCenter",
377 "{band}_extendedness_flag",
378 "xy_flag",
379 "detect_isPatchInner",
380 "detect_isDeblendedSource",
381 ]
382 self._checkSchema(selector, keys)
384 result = selector(self.data)
385 truth = np.ones(self.size, dtype=bool)
386 for bit in (1, 3, 5, 7, 9, 11):
387 truth[bit] = 0
388 np.testing.assert_array_equal(result, truth)
390 # Test bands override
391 selector = CoaddPlotFlagSelector(
392 bands=["i", "r"],
393 selectWhenFalse=["{band}_psfFlux_flag"],
394 selectWhenTrue=["detect_isDeblendedSource"],
395 )
396 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
397 result = selector(self.data)
398 truth = np.ones(self.size, dtype=bool)
399 for bit in (1, 11):
400 truth[bit] = 0
401 np.testing.assert_array_equal(result, truth)
403 def testRangeSelector(self):
404 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
405 self._checkSchema(selector, ["r_psfFlux"])
406 result = self.data["r_psfFlux"][selector(self.data)]
407 truth = [30, 40]
408 np.testing.assert_array_equal(result, truth)
410 def testSnSelector(self):
411 # test defaults
412 selector = SnSelector()
413 keys = [
414 "{band}_psfFlux",
415 "{band}_psfFluxErr",
416 ]
417 self._checkSchema(selector, keys)
418 result = selector(self.data, bands=["r"])
419 truth = np.ones(self.size, dtype=bool)
420 truth[:6] = 0
421 np.testing.assert_array_equal(result, truth)
423 # test overrides
424 selector = SnSelector(
425 fluxType="{band}_cmodelFlux",
426 threshold=200.0,
427 uncertaintySuffix="Error",
428 bands=["r", "i"],
429 )
430 keys = [
431 "{band}_cmodelFlux",
432 "{band}_cmodelFluxError",
433 ]
434 self._checkSchema(selector, keys)
435 result = selector(self.data)
436 truth = np.ones(self.size, dtype=bool)
437 truth[:3] = 0
438 truth[-3:] = 0
439 np.testing.assert_array_equal(result, truth)
441 def testSkyObjectSelector(self):
442 # Test default
443 selector = SkyObjectSelector()
444 keys = ["{band}_pixelFlags_edge", "sky_object"]
445 self._checkSchema(selector, keys)
446 result = selector(self.data)
447 truth = np.zeros(self.size, dtype=bool)
448 truth[15] = 1
449 truth[17] = 1
450 np.testing.assert_array_equal(result, truth)
452 # Test overrides
453 selector = SkyObjectSelector(bands=["i", "r"])
454 self._checkSchema(selector, keys)
455 result = selector(self.data)
456 truth = np.zeros(self.size, dtype=bool)
457 truth[17] = 1
458 np.testing.assert_array_equal(result, truth)
460 def testStarSelector(self):
461 # test default
462 selector = StarSelector()
463 self._checkSchema(selector, ["{band}_extendedness"])
464 result = selector(self.data, band="i")
465 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
466 np.testing.assert_array_almost_equal(result, truth)
468 # Test overrides
469 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
470 result = selector(self.data, band="i")
471 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
472 np.testing.assert_array_almost_equal(result, truth)
474 def testGalaxySelector(self):
475 # test default
476 selector = GalaxySelector()
477 self._checkSchema(selector, ["{band}_extendedness"])
478 result = selector(self.data, band="i")
479 truth = self.data["i_extendedness"] > 0.5
480 np.testing.assert_array_almost_equal(result, truth)
482 # Test overrides
483 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
484 result = selector(self.data, band="i")
485 truth = self.data["i_extendedness"] > 0.3
486 np.testing.assert_array_almost_equal(result, truth)
488 def testVectorSelector(self):
489 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
490 self._checkSchema(selector, ["{band}_psfFlux_flag"])
491 result = selector(self.data, band="i")
492 truth = np.zeros(self.size, dtype=bool)
493 truth[1] = True
494 np.testing.assert_array_equal(result, truth)