Coverage for tests/test_actions.py: 15%
276 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 04:05 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 04:05 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import numpy as np
26import pandas as pd
27from lsst.analysis.tools.actions.scalar.scalarActions import (
28 ApproxFloor,
29 CountAction,
30 MeanAction,
31 MedianAction,
32 SigmaMadAction,
33 StdevAction,
34)
35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction
36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize
37from lsst.analysis.tools.actions.vector.mathActions import (
38 AddVector,
39 ConstantValue,
40 DivideVector,
41 FractionalDifference,
42 MultiplyVector,
43 SubtractVector,
44)
45from lsst.analysis.tools.actions.vector.selectors import (
46 CoaddPlotFlagSelector,
47 FlagSelector,
48 GalaxySelector,
49 RangeSelector,
50 SkyObjectSelector,
51 SnSelector,
52 StarSelector,
53 VectorSelector,
54)
55from lsst.analysis.tools.actions.vector.vectorActions import (
56 ConvertFluxToMag,
57 DownselectVector,
58 ExtinctionCorrectedMagDiff,
59 LoadVector,
60 MagDiff,
61)
64class TestScalarActions(unittest.TestCase):
65 """ "Test ScalarActions"""
67 def setUp(self):
68 x = np.arange(100, dtype=float)
69 x[31] = np.nan
70 x[41] = np.nan
71 x[59] = np.nan
72 y = x**2
73 self.data = {"r_y": x, "i_y": y}
74 self.mask = np.zeros(100, dtype=bool)
75 self.mask[1:50] = 1
77 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
78 action = cls(vectorKey="{band}_y")
79 schema = [col for col, colType in action.getInputSchema()]
80 self.assertEqual(schema, ["{band}_y"])
81 result = action(self.data, band="i")
82 self.assertAlmostEqual(result, truth)
83 result = action(self.data, mask=self.mask, band="i")
84 self.assertAlmostEqual(result, maskedTruth)
86 def testMedianAction(self):
87 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
89 def testMeanAction(self):
90 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
92 def testStdevAction(self):
93 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
95 def testSigmaMadAction(self):
96 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
98 def testCountAction(self):
99 self._testScalarActionAlmostEqual(CountAction, 97, 47)
101 def testApproxFloorAction(self):
102 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
105class TestVectorActions(unittest.TestCase):
106 """Test VectorActions"""
108 def setUp(self):
109 self.size = 5
110 r = np.arange(self.size) + 1
111 i = r**2
112 rFlag = np.ones(self.size)
113 rFlag[1] = 0
114 iFlag = np.ones(self.size)
115 iFlag[3] = 0
116 data = {
117 "r_vector": r,
118 "i_vector": i,
119 "r_flag": rFlag,
120 "i_flag": iFlag,
121 "g_vector": 3 * r,
122 "E(B-V)": r[::-1],
123 "r_ixx": r**2,
124 "r_iyy": r[::-1] ** 2,
125 "r_ixy": r * r[::-1],
126 "g_iixx": r**2,
127 "g_iiyy": r[::-1] ** 2,
128 "g_iixy": r * r[::-1],
129 }
131 self.data = pd.DataFrame.from_dict(data)
133 def _checkSchema(self, action, truth):
134 schema = sorted([col for col, colType in action.getInputSchema()])
135 self.assertEqual(schema, truth)
137 # VectorActions with their own files
139 def testCalcBinnedStats(self):
140 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size)
141 prefix = "a_"
142 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
143 result = stats(self.data)
144 mask = selector(self.data)
145 values = self.data["r_vector"][mask]
146 median = np.median(values)
147 truth = {
148 stats.name_mask: mask,
149 stats.name_median: median,
150 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(values - median)),
151 stats.name_count: np.sum(mask),
152 stats.name_select_maximum: np.max(values),
153 stats.name_select_median: median,
154 stats.name_select_minimum: np.min(values),
155 "range_maximum": self.size,
156 "range_minimum": 0,
157 }
158 self.assertEqual(list(result.keys()), list(truth.keys()))
160 np.testing.assert_array_almost_equal(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
161 del truth[stats.name_sigmaMad]
163 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
164 del truth[stats.name_mask]
166 for key, value in truth.items():
167 self.assertEqual(result[key], value, key)
169 # def testCalcRhoStatistics(self): TODO: implement
171 def testCalcShapeSize(self):
172 xx = self.data["r_ixx"]
173 yy = self.data["r_iyy"]
174 xy = self.data["r_ixy"]
176 # Test determinant with defaults
177 action = CalcShapeSize()
178 result = action(self.data, band="r")
179 schema = [col for col, colType in action.getInputSchema()]
180 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
181 truth = 0.25 * (xx * yy - xy**2)
182 np.testing.assert_array_almost_equal(result, truth)
184 # Test trace with columns specified
185 action = CalcShapeSize(
186 colXx="{band}_iixx",
187 colYy="{band}_iiyy",
188 colXy="{band}_iixy",
189 sizeType="trace",
190 )
191 result = action(self.data, band="g")
192 schema = [col for col, colType in action.getInputSchema()]
193 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
194 truth = np.sqrt(0.5 * (xx + yy))
195 np.testing.assert_array_almost_equal(result, truth)
197 # def testCalcE(self): TODO: implement
199 # def testCalcEDiff(self): TODO: implement
201 # def testCalcE1(self): TODO: implement
203 # def testCalcE2(self): TODO: implement
205 # MathActions
207 def _testMath(self, ActionType, truth, compare_exact: bool = False):
208 actionA = LoadVector(vectorKey="{band1}_vector")
209 actionB = LoadVector(vectorKey="{band2}_vector")
210 action = ActionType(actionA=actionA, actionB=actionB)
211 result = action(self.data, band1="r", band2="i")
212 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey])
213 if compare_exact:
214 np.testing.assert_array_equal(result, truth)
215 else:
216 np.testing.assert_array_almost_equal(result, truth)
218 def testConstant(self):
219 truth = [42.0]
220 action = ConstantValue(value=truth[0])
221 self._checkSchema(action, [])
222 result = action({})
223 np.testing.assert_array_equal(result, truth)
225 def testAdd(self):
226 truth = [2.0, 6.0, 12.0, 20.0, 30.0]
227 self._testMath(AddVector, truth, True)
229 def testSubtract(self):
230 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
231 self._testMath(SubtractVector, truth, True)
233 def testMultiply(self):
234 truth = [1.0, 8.0, 27.0, 64.0, 125.0]
235 self._testMath(MultiplyVector, truth, False)
237 def testDivide(self):
238 truth = 1 / np.arange(1, 6)
239 self._testMath(DivideVector, truth, False)
241 def testFractionalDifference(self):
242 actionA = LoadVector(vectorKey="{band1}_vector")
243 actionB = LoadVector(vectorKey="{band2}_vector")
244 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
245 diff = FractionalDifference(actionA=actionA, actionB=actionB)
246 result = diff(self.data, band1="r", band2="i")
247 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
248 np.testing.assert_array_almost_equal(result, truth)
250 # Basic vectorActions
252 # def testLoadVector(self): TODO: implement
254 def testDownselectVector(self):
255 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
256 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
257 result = action(self.data, band="r")
258 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
259 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
261 # def testMultiCriteriaDownselectVector(self): TODO: implement
263 # Astronomical vectorActions
265 # def testCalcSn(self): TODO: implement
267 def testConvertFluxToMag(self):
268 truth = [
269 31.4,
270 29.89485002168,
271 29.0143937264,
272 28.38970004336,
273 27.90514997832,
274 ]
275 action = ConvertFluxToMag(vectorKey="{band}_vector")
276 result = action(self.data, band="i")
277 self._checkSchema(action, ["{band}_vector"])
278 np.testing.assert_array_almost_equal(result, truth)
280 # def testConvertUnits(self): TODO: implement
282 def testMagDiff(self):
283 # Use the same units as the data so that we know the difference exactly
284 # without conversions.
285 # testExtinctionCorrectedMagDiff will test the conversions
286 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
287 action = MagDiff(
288 col1="{band1}_vector",
289 col2="{band2}_vector",
290 fluxUnits1="mag(AB)",
291 fluxUnits2="mag(AB)",
292 returnMillimags=False,
293 )
294 result = action(self.data, band1="g", band2="r")
295 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
296 np.testing.assert_array_almost_equal(result, truth.value)
298 def testExtinctionCorrectedMagDiff(self):
299 for returnMillimags in (True, False):
300 # Check that conversions are working properly
301 magDiff = MagDiff(
302 col1="{band1}_vector",
303 col2="{band2}_vector",
304 fluxUnits1="jansky",
305 fluxUnits2="jansky",
306 returnMillimags=returnMillimags,
307 )
308 action = ExtinctionCorrectedMagDiff(
309 magDiff=magDiff,
310 band1="g",
311 band2="r",
312 ebvCol="E(B-V)",
313 extinctionCoeffs={"g": 0.2, "r": 1.5},
314 )
316 result = action(self.data, band1="g", band2="r")
317 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
318 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
319 diff = lhs - rhs
320 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
321 if returnMillimags:
322 diff = diff.to(u.mmag)
323 correction = correction.to(u.mmag)
324 truth = diff - correction
325 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
326 np.testing.assert_array_almost_equal(result, truth.value)
328 # Test with hard coded bands
329 magDiff = MagDiff(
330 col1="g_vector",
331 col2="r_vector",
332 fluxUnits1="jansky",
333 fluxUnits2="jansky",
334 returnMillimags=False,
335 )
336 action = ExtinctionCorrectedMagDiff(
337 magDiff=magDiff,
338 ebvCol="E(B-V)",
339 extinctionCoeffs={"g": 0.2, "r": 1.5},
340 )
341 result = action(self.data)
342 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
343 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
344 diff = lhs - rhs
345 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
346 truth = diff - correction
347 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
348 np.testing.assert_array_almost_equal(result, truth.value)
350 # def testRAcosDec(self): TODO: implement
352 # Statistical vectorActions
354 # def testPerGroupStatistic(self): TODO: implement
356 # def testResidualWithPerGroupStatistic(self): TODO: implement
359class TestVectorSelectors(unittest.TestCase):
360 def setUp(self):
361 self.size = 20
362 falseFlags = {
363 "{band}_psfFlux_flag": [1],
364 "{band}_pixelFlags_saturatedCenter": [3],
365 "{band}_extendedness_flag": [5],
366 "xy_flag": [7],
367 "i_pixelFlags_edge": [13],
368 "r_pixelFlags_edge": [15],
369 "sky_object": [13, 15, 17],
370 }
372 trueFlags = {
373 "detect_isPatchInner": [9],
374 "detect_isDeblendedSource": [11],
375 }
377 flux = np.arange(self.size) * 10
378 fluxErr = np.ones(self.size) * 0.1
379 extendedness = np.arange(20) / 20 - 0.1
381 self.data = {
382 "r_psfFlux": flux,
383 "r_psfFluxErr": fluxErr,
384 "i_cmodelFlux": flux[::-1],
385 "i_cmodelFluxError": fluxErr[::-1],
386 "r_cmodelFlux": flux,
387 "r_cmodelFluxError": fluxErr,
388 "i_extendedness": extendedness,
389 "i_extended": extendedness,
390 }
391 bands = ("r", "i")
392 for band in bands:
393 for flag, bits in falseFlags.items():
394 vector = np.zeros(self.size, dtype=bool)
395 for bit in bits:
396 vector[bit] = 1
397 self.data[flag.format(band=band)] = vector
399 for flag, bits in trueFlags.items():
400 vector = np.ones(self.size, dtype=bool)
401 for bit in bits:
402 vector[bit] = 0
403 self.data[flag.format(band=band)] = vector
405 def _checkSchema(self, action, truth):
406 schema = [col for col, colType in action.getInputSchema()]
407 self.assertEqual(sorted(schema), sorted(truth))
409 def testFlagSelector(self):
410 selector = FlagSelector(
411 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
412 )
413 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
414 result = selector(self.data, band="r")
415 truth = np.ones(self.size, dtype=bool)
416 truth[1] = False
417 truth[9] = False
418 np.testing.assert_array_equal(result, truth)
420 def testCoaddPlotFlagSelector(self):
421 # Test defaults
422 # Bands needs to be set to something otherwise it
423 # will crash as the default value looks it up
424 # elsewhere.
425 selector = CoaddPlotFlagSelector(bands=["i"])
426 keys = [
427 "{band}_psfFlux_flag",
428 "{band}_pixelFlags_saturatedCenter",
429 "{band}_extendedness_flag",
430 "xy_flag",
431 "detect_isPatchInner",
432 "detect_isDeblendedSource",
433 ]
434 self._checkSchema(selector, keys)
436 result = selector(self.data)
437 truth = np.ones(self.size, dtype=bool)
438 for bit in (1, 3, 5, 7, 9, 11):
439 truth[bit] = 0
440 np.testing.assert_array_equal(result, truth)
442 # Test bands override
443 selector = CoaddPlotFlagSelector(
444 bands=["i", "r"],
445 selectWhenFalse=["{band}_psfFlux_flag"],
446 selectWhenTrue=["detect_isDeblendedSource"],
447 )
448 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
449 result = selector(self.data)
450 truth = np.ones(self.size, dtype=bool)
451 for bit in (1, 11):
452 truth[bit] = 0
453 np.testing.assert_array_equal(result, truth)
455 def testRangeSelector(self):
456 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
457 self._checkSchema(selector, ["r_psfFlux"])
458 result = self.data["r_psfFlux"][selector(self.data)]
459 truth = [30, 40]
460 np.testing.assert_array_equal(result, truth)
462 def testSnSelector(self):
463 # test defaults
464 selector = SnSelector()
465 keys = [
466 "{band}_psfFlux",
467 "{band}_psfFluxErr",
468 ]
469 self._checkSchema(selector, keys)
470 result = selector(self.data, bands=["r"])
471 truth = np.ones(self.size, dtype=bool)
472 truth[:6] = 0
473 np.testing.assert_array_equal(result, truth)
475 # test overrides
476 selector = SnSelector(
477 fluxType="{band}_cmodelFlux",
478 threshold=200.0,
479 uncertaintySuffix="Error",
480 bands=["r", "i"],
481 )
482 keys = [
483 "{band}_cmodelFlux",
484 "{band}_cmodelFluxError",
485 ]
486 self._checkSchema(selector, keys)
487 result = selector(self.data)
488 truth = np.ones(self.size, dtype=bool)
489 truth[:3] = 0
490 truth[-3:] = 0
491 np.testing.assert_array_equal(result, truth)
493 def testSkyObjectSelector(self):
494 # Test default
495 selector = SkyObjectSelector()
496 keys = ["{band}_pixelFlags_edge", "sky_object"]
497 self._checkSchema(selector, keys)
498 result = selector(self.data)
499 truth = np.zeros(self.size, dtype=bool)
500 truth[15] = 1
501 truth[17] = 1
502 np.testing.assert_array_equal(result, truth)
504 # Test overrides
505 selector = SkyObjectSelector(bands=["i", "r"])
506 self._checkSchema(selector, keys)
507 result = selector(self.data)
508 truth = np.zeros(self.size, dtype=bool)
509 truth[17] = 1
510 np.testing.assert_array_equal(result, truth)
512 def testStarSelector(self):
513 # test default
514 selector = StarSelector()
515 self._checkSchema(selector, ["{band}_extendedness"])
516 result = selector(self.data, band="i")
517 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
518 np.testing.assert_array_almost_equal(result, truth)
520 # Test overrides
521 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
522 result = selector(self.data, band="i")
523 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
524 np.testing.assert_array_almost_equal(result, truth)
526 def testGalaxySelector(self):
527 # test default
528 selector = GalaxySelector()
529 self._checkSchema(selector, ["{band}_extendedness"])
530 result = selector(self.data, band="i")
531 truth = self.data["i_extendedness"] > 0.5
532 np.testing.assert_array_almost_equal(result, truth)
534 # Test overrides
535 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
536 result = selector(self.data, band="i")
537 truth = self.data["i_extendedness"] > 0.3
538 np.testing.assert_array_almost_equal(result, truth)
540 def testVectorSelector(self):
541 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
542 self._checkSchema(selector, ["{band}_psfFlux_flag"])
543 result = selector(self.data, band="i")
544 truth = np.zeros(self.size, dtype=bool)
545 truth[1] = True
546 np.testing.assert_array_equal(result, truth)