Coverage for tests/test_actions.py: 15%
274 statements
« prev ^ index » next coverage.py v7.2.6, created at 2023-05-26 03:18 -0700
« prev ^ index » next coverage.py v7.2.6, created at 2023-05-26 03:18 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import numpy as np
26import pandas as pd
27from lsst.analysis.tools.actions.scalar.scalarActions import (
28 ApproxFloor,
29 CountAction,
30 MeanAction,
31 MedianAction,
32 SigmaMadAction,
33 StdevAction,
34)
35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction
36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize
37from lsst.analysis.tools.actions.vector.mathActions import (
38 AddVector,
39 ConstantValue,
40 DivideVector,
41 FractionalDifference,
42 MultiplyVector,
43 SubtractVector,
44)
45from lsst.analysis.tools.actions.vector.selectors import (
46 CoaddPlotFlagSelector,
47 FlagSelector,
48 GalaxySelector,
49 RangeSelector,
50 SkyObjectSelector,
51 SnSelector,
52 StarSelector,
53 VectorSelector,
54)
55from lsst.analysis.tools.actions.vector.vectorActions import (
56 ConvertFluxToMag,
57 DownselectVector,
58 ExtinctionCorrectedMagDiff,
59 LoadVector,
60 MagDiff,
61)
64class TestScalarActions(unittest.TestCase):
65 """ "Test ScalarActions"""
67 def setUp(self):
68 x = np.arange(100, dtype=float)
69 x[31] = np.nan
70 x[41] = np.nan
71 x[59] = np.nan
72 y = x**2
73 self.data = {"r_y": x, "i_y": y}
74 self.mask = np.zeros(100, dtype=bool)
75 self.mask[1:50] = 1
77 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
78 action = cls(vectorKey="{band}_y")
79 schema = [col for col, colType in action.getInputSchema()]
80 self.assertEqual(schema, ["{band}_y"])
81 result = action(self.data, band="i")
82 self.assertAlmostEqual(result, truth)
83 result = action(self.data, mask=self.mask, band="i")
84 self.assertAlmostEqual(result, maskedTruth)
86 def testMedianAction(self):
87 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
89 def testMeanAction(self):
90 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
92 def testStdevAction(self):
93 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
95 def testSigmaMadAction(self):
96 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
98 def testCountAction(self):
99 self._testScalarActionAlmostEqual(CountAction, 97, 47)
101 def testApproxFloorAction(self):
102 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
105class TestVectorActions(unittest.TestCase):
106 """Test VectorActions"""
108 def setUp(self):
109 self.size = 5
110 r = np.arange(self.size) + 1
111 i = r**2
112 rFlag = np.ones(self.size)
113 rFlag[1] = 0
114 iFlag = np.ones(self.size)
115 iFlag[3] = 0
116 data = {
117 "r_vector": r,
118 "i_vector": i,
119 "r_flag": rFlag,
120 "i_flag": iFlag,
121 "g_vector": 3 * r,
122 "E(B-V)": r[::-1],
123 "r_ixx": r**2,
124 "r_iyy": r[::-1] ** 2,
125 "r_ixy": r * r[::-1],
126 "g_iixx": r**2,
127 "g_iiyy": r[::-1] ** 2,
128 "g_iixy": r * r[::-1],
129 }
131 self.data = pd.DataFrame.from_dict(data)
133 def _checkSchema(self, action, truth):
134 schema = sorted([col for col, colType in action.getInputSchema()])
135 self.assertEqual(schema, truth)
137 # VectorActions with their own files
139 def testCalcBinnedStats(self):
140 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size + 1)
141 prefix = "a_"
142 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
143 result = stats(self.data)
144 median = (1 + self.size) / 2.0
145 truth = {
146 stats.name_mask: np.ones(self.size),
147 stats.name_median: median,
148 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)),
149 stats.name_count: self.size,
150 stats.name_select_maximum: self.size,
151 stats.name_select_median: median,
152 stats.name_select_minimum: 1,
153 "range_maximum": self.size + 1,
154 "range_minimum": 0,
155 }
156 self.assertEqual(list(result.keys()), list(truth.keys()))
158 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
159 del truth[stats.name_sigmaMad]
161 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
162 del truth[stats.name_mask]
164 for key, value in truth.items():
165 self.assertEqual(result[key], value, key)
167 # def testCalcRhoStatistics(self): TODO: implement
169 def testCalcShapeSize(self):
170 xx = self.data["r_ixx"]
171 yy = self.data["r_iyy"]
172 xy = self.data["r_ixy"]
174 # Test determinant with defaults
175 action = CalcShapeSize()
176 result = action(self.data, band="r")
177 schema = [col for col, colType in action.getInputSchema()]
178 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
179 truth = 0.25 * (xx * yy - xy**2)
180 np.testing.assert_array_almost_equal(result, truth)
182 # Test trace with columns specified
183 action = CalcShapeSize(
184 colXx="{band}_iixx",
185 colYy="{band}_iiyy",
186 colXy="{band}_iixy",
187 sizeType="trace",
188 )
189 result = action(self.data, band="g")
190 schema = [col for col, colType in action.getInputSchema()]
191 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
192 truth = np.sqrt(0.5 * (xx + yy))
193 np.testing.assert_array_almost_equal(result, truth)
195 # def testCalcE(self): TODO: implement
197 # def testCalcEDiff(self): TODO: implement
199 # def testCalcE1(self): TODO: implement
201 # def testCalcE2(self): TODO: implement
203 # MathActions
205 def _testMath(self, ActionType, truth, compare_exact: bool = False):
206 actionA = LoadVector(vectorKey="{band1}_vector")
207 actionB = LoadVector(vectorKey="{band2}_vector")
208 action = ActionType(actionA=actionA, actionB=actionB)
209 result = action(self.data, band1="r", band2="i")
210 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey])
211 if compare_exact:
212 np.testing.assert_array_equal(result, truth)
213 else:
214 np.testing.assert_array_almost_equal(result, truth)
216 def testConstant(self):
217 truth = [42.0]
218 action = ConstantValue(value=truth[0])
219 self._checkSchema(action, [])
220 result = action({})
221 np.testing.assert_array_equal(result, truth)
223 def testAdd(self):
224 truth = [2.0, 6.0, 12.0, 20.0, 30.0]
225 self._testMath(AddVector, truth, True)
227 def testSubtract(self):
228 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
229 self._testMath(SubtractVector, truth, True)
231 def testMultiply(self):
232 truth = [1.0, 8.0, 27.0, 64.0, 125.0]
233 self._testMath(MultiplyVector, truth, False)
235 def testDivide(self):
236 truth = 1 / np.arange(1, 6)
237 self._testMath(DivideVector, truth, False)
239 def testFractionalDifference(self):
240 actionA = LoadVector(vectorKey="{band1}_vector")
241 actionB = LoadVector(vectorKey="{band2}_vector")
242 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
243 diff = FractionalDifference(actionA=actionA, actionB=actionB)
244 result = diff(self.data, band1="r", band2="i")
245 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
246 np.testing.assert_array_almost_equal(result, truth)
248 # Basic vectorActions
250 # def testLoadVector(self): TODO: implement
252 def testDownselectVector(self):
253 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
254 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
255 result = action(self.data, band="r")
256 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
257 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
259 # def testMultiCriteriaDownselectVector(self): TODO: implement
261 # Astronomical vectorActions
263 # def testCalcSn(self): TODO: implement
265 def testConvertFluxToMag(self):
266 truth = [
267 31.4,
268 29.89485002168,
269 29.0143937264,
270 28.38970004336,
271 27.90514997832,
272 ]
273 action = ConvertFluxToMag(vectorKey="{band}_vector")
274 result = action(self.data, band="i")
275 self._checkSchema(action, ["{band}_vector"])
276 np.testing.assert_array_almost_equal(result, truth)
278 # def testConvertUnits(self): TODO: implement
280 def testMagDiff(self):
281 # Use the same units as the data so that we know the difference exactly
282 # without conversions.
283 # testExtinctionCorrectedMagDiff will test the conversions
284 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
285 action = MagDiff(
286 col1="{band1}_vector",
287 col2="{band2}_vector",
288 fluxUnits1="mag(AB)",
289 fluxUnits2="mag(AB)",
290 returnMillimags=False,
291 )
292 result = action(self.data, band1="g", band2="r")
293 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
294 np.testing.assert_array_almost_equal(result, truth.value)
296 def testExtinctionCorrectedMagDiff(self):
297 for returnMillimags in (True, False):
298 # Check that conversions are working properly
299 magDiff = MagDiff(
300 col1="{band1}_vector",
301 col2="{band2}_vector",
302 fluxUnits1="jansky",
303 fluxUnits2="jansky",
304 returnMillimags=returnMillimags,
305 )
306 action = ExtinctionCorrectedMagDiff(
307 magDiff=magDiff,
308 band1="g",
309 band2="r",
310 ebvCol="E(B-V)",
311 extinctionCoeffs={"g": 0.2, "r": 1.5},
312 )
314 result = action(self.data, band1="g", band2="r")
315 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
316 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
317 diff = lhs - rhs
318 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
319 if returnMillimags:
320 diff = diff.to(u.mmag)
321 correction = correction.to(u.mmag)
322 truth = diff - correction
323 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
324 np.testing.assert_array_almost_equal(result, truth.value)
326 # Test with hard coded bands
327 magDiff = MagDiff(
328 col1="g_vector",
329 col2="r_vector",
330 fluxUnits1="jansky",
331 fluxUnits2="jansky",
332 returnMillimags=False,
333 )
334 action = ExtinctionCorrectedMagDiff(
335 magDiff=magDiff,
336 ebvCol="E(B-V)",
337 extinctionCoeffs={"g": 0.2, "r": 1.5},
338 )
339 result = action(self.data)
340 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
341 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
342 diff = lhs - rhs
343 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
344 truth = diff - correction
345 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
346 np.testing.assert_array_almost_equal(result, truth.value)
348 # def testRAcosDec(self): TODO: implement
350 # Statistical vectorActions
352 # def testPerGroupStatistic(self): TODO: implement
354 # def testResidualWithPerGroupStatistic(self): TODO: implement
357class TestVectorSelectors(unittest.TestCase):
358 def setUp(self):
359 self.size = 20
360 falseFlags = {
361 "{band}_psfFlux_flag": [1],
362 "{band}_pixelFlags_saturatedCenter": [3],
363 "{band}_extendedness_flag": [5],
364 "xy_flag": [7],
365 "i_pixelFlags_edge": [13],
366 "r_pixelFlags_edge": [15],
367 "sky_object": [13, 15, 17],
368 }
370 trueFlags = {
371 "detect_isPatchInner": [9],
372 "detect_isDeblendedSource": [11],
373 }
375 flux = np.arange(self.size) * 10
376 fluxErr = np.ones(self.size) * 0.1
377 extendedness = np.arange(20) / 20 - 0.1
379 self.data = {
380 "r_psfFlux": flux,
381 "r_psfFluxErr": fluxErr,
382 "i_cmodelFlux": flux[::-1],
383 "i_cmodelFluxError": fluxErr[::-1],
384 "r_cmodelFlux": flux,
385 "r_cmodelFluxError": fluxErr,
386 "i_extendedness": extendedness,
387 "i_extended": extendedness,
388 }
389 bands = ("r", "i")
390 for band in bands:
391 for flag, bits in falseFlags.items():
392 vector = np.zeros(self.size, dtype=bool)
393 for bit in bits:
394 vector[bit] = 1
395 self.data[flag.format(band=band)] = vector
397 for flag, bits in trueFlags.items():
398 vector = np.ones(self.size, dtype=bool)
399 for bit in bits:
400 vector[bit] = 0
401 self.data[flag.format(band=band)] = vector
403 def _checkSchema(self, action, truth):
404 schema = [col for col, colType in action.getInputSchema()]
405 self.assertEqual(sorted(schema), sorted(truth))
407 def testFlagSelector(self):
408 selector = FlagSelector(
409 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
410 )
411 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
412 result = selector(self.data, band="r")
413 truth = np.ones(self.size, dtype=bool)
414 truth[1] = False
415 truth[9] = False
416 np.testing.assert_array_equal(result, truth)
418 def testCoaddPlotFlagSelector(self):
419 # Test defaults
420 # Bands needs to be set to something otherwise it
421 # will crash as the default value looks it up
422 # elsewhere.
423 selector = CoaddPlotFlagSelector(bands=["i"])
424 keys = [
425 "{band}_psfFlux_flag",
426 "{band}_pixelFlags_saturatedCenter",
427 "{band}_extendedness_flag",
428 "xy_flag",
429 "detect_isPatchInner",
430 "detect_isDeblendedSource",
431 ]
432 self._checkSchema(selector, keys)
434 result = selector(self.data)
435 truth = np.ones(self.size, dtype=bool)
436 for bit in (1, 3, 5, 7, 9, 11):
437 truth[bit] = 0
438 np.testing.assert_array_equal(result, truth)
440 # Test bands override
441 selector = CoaddPlotFlagSelector(
442 bands=["i", "r"],
443 selectWhenFalse=["{band}_psfFlux_flag"],
444 selectWhenTrue=["detect_isDeblendedSource"],
445 )
446 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
447 result = selector(self.data)
448 truth = np.ones(self.size, dtype=bool)
449 for bit in (1, 11):
450 truth[bit] = 0
451 np.testing.assert_array_equal(result, truth)
453 def testRangeSelector(self):
454 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
455 self._checkSchema(selector, ["r_psfFlux"])
456 result = self.data["r_psfFlux"][selector(self.data)]
457 truth = [30, 40]
458 np.testing.assert_array_equal(result, truth)
460 def testSnSelector(self):
461 # test defaults
462 selector = SnSelector()
463 keys = [
464 "{band}_psfFlux",
465 "{band}_psfFluxErr",
466 ]
467 self._checkSchema(selector, keys)
468 result = selector(self.data, bands=["r"])
469 truth = np.ones(self.size, dtype=bool)
470 truth[:6] = 0
471 np.testing.assert_array_equal(result, truth)
473 # test overrides
474 selector = SnSelector(
475 fluxType="{band}_cmodelFlux",
476 threshold=200.0,
477 uncertaintySuffix="Error",
478 bands=["r", "i"],
479 )
480 keys = [
481 "{band}_cmodelFlux",
482 "{band}_cmodelFluxError",
483 ]
484 self._checkSchema(selector, keys)
485 result = selector(self.data)
486 truth = np.ones(self.size, dtype=bool)
487 truth[:3] = 0
488 truth[-3:] = 0
489 np.testing.assert_array_equal(result, truth)
491 def testSkyObjectSelector(self):
492 # Test default
493 selector = SkyObjectSelector()
494 keys = ["{band}_pixelFlags_edge", "sky_object"]
495 self._checkSchema(selector, keys)
496 result = selector(self.data)
497 truth = np.zeros(self.size, dtype=bool)
498 truth[15] = 1
499 truth[17] = 1
500 np.testing.assert_array_equal(result, truth)
502 # Test overrides
503 selector = SkyObjectSelector(bands=["i", "r"])
504 self._checkSchema(selector, keys)
505 result = selector(self.data)
506 truth = np.zeros(self.size, dtype=bool)
507 truth[17] = 1
508 np.testing.assert_array_equal(result, truth)
510 def testStarSelector(self):
511 # test default
512 selector = StarSelector()
513 self._checkSchema(selector, ["{band}_extendedness"])
514 result = selector(self.data, band="i")
515 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
516 np.testing.assert_array_almost_equal(result, truth)
518 # Test overrides
519 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
520 result = selector(self.data, band="i")
521 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
522 np.testing.assert_array_almost_equal(result, truth)
524 def testGalaxySelector(self):
525 # test default
526 selector = GalaxySelector()
527 self._checkSchema(selector, ["{band}_extendedness"])
528 result = selector(self.data, band="i")
529 truth = self.data["i_extendedness"] > 0.5
530 np.testing.assert_array_almost_equal(result, truth)
532 # Test overrides
533 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
534 result = selector(self.data, band="i")
535 truth = self.data["i_extendedness"] > 0.3
536 np.testing.assert_array_almost_equal(result, truth)
538 def testVectorSelector(self):
539 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
540 self._checkSchema(selector, ["{band}_psfFlux_flag"])
541 result = selector(self.data, band="i")
542 truth = np.zeros(self.size, dtype=bool)
543 truth[1] = True
544 np.testing.assert_array_equal(result, truth)