Coverage for tests/test_actions.py: 15%
323 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-03 10:03 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-03 10:03 +0000
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import lsst.utils.tests
26import numpy as np
27import pandas as pd
28from lsst.analysis.tools.actions.keyedData.calcDistances import CalcRelativeDistances
29from lsst.analysis.tools.actions.scalar import (
30 ApproxFloor,
31 CountAction,
32 MeanAction,
33 MedianAction,
34 SigmaMadAction,
35 StdevAction,
36)
37from lsst.analysis.tools.actions.vector import (
38 CalcBinnedStatsAction,
39 CalcMomentSize,
40 ConvertFluxToMag,
41 DownselectVector,
42 ExtinctionCorrectedMagDiff,
43 LoadVector,
44 MagDiff,
45)
46from lsst.analysis.tools.actions.vector.mathActions import (
47 AddVector,
48 ConstantValue,
49 DivideVector,
50 FractionalDifference,
51 Log10Vector,
52 MultiplyVector,
53 RaiseFromBaseVector,
54 RaiseToPowerVector,
55 SqrtVector,
56 SquareVector,
57 SubtractVector,
58)
59from lsst.analysis.tools.actions.vector.selectors import (
60 CoaddPlotFlagSelector,
61 FlagSelector,
62 GalaxySelector,
63 RangeSelector,
64 SkyObjectSelector,
65 SnSelector,
66 StarSelector,
67 VectorSelector,
68)
69from lsst.pex.config import FieldValidationError
72class TestScalarActions(unittest.TestCase):
73 """ "Test ScalarActions"""
75 def setUp(self):
76 x = np.arange(100, dtype=float)
77 x[31] = np.nan
78 x[41] = np.nan
79 x[59] = np.nan
80 y = x**2
81 self.data = {"r_y": x, "i_y": y}
82 self.mask = np.zeros(100, dtype=bool)
83 self.mask[1:50] = 1
85 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
86 action = cls(vectorKey="{band}_y")
87 schema = [col for col, colType in action.getInputSchema()]
88 self.assertEqual(schema, ["{band}_y"])
89 result = action(self.data, band="i")
90 self.assertAlmostEqual(result, truth)
91 result = action(self.data, mask=self.mask, band="i")
92 self.assertAlmostEqual(result, maskedTruth)
94 def testMedianAction(self):
95 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
97 def testMeanAction(self):
98 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
100 def testStdevAction(self):
101 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
103 def testSigmaMadAction(self):
104 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
106 def testCountAction(self):
107 self._testScalarActionAlmostEqual(CountAction, 97, 47)
109 def testApproxFloorAction(self):
110 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
113class TestVectorActions(unittest.TestCase):
114 """Test VectorActions"""
116 def setUp(self):
117 self.size = 5
118 r = np.arange(self.size) + 1
119 i = r**2
120 rFlag = np.ones(self.size)
121 rFlag[1] = 0
122 iFlag = np.ones(self.size)
123 iFlag[3] = 0
124 data = {
125 "r_vector": r,
126 "i_vector": i,
127 "r_flag": rFlag,
128 "i_flag": iFlag,
129 "g_vector": 3 * r,
130 "E(B-V)": r[::-1],
131 "r_ixx": r**2,
132 "r_iyy": r[::-1] ** 2,
133 "r_ixy": r * r[::-1],
134 "g_iixx": r**2,
135 "g_iiyy": r[::-1] ** 2,
136 "g_iixy": r * r[::-1],
137 }
139 self.data = pd.DataFrame.from_dict(data)
141 def _checkSchema(self, action, truth):
142 schema = sorted([col for col, colType in action.getInputSchema()])
143 self.assertEqual(schema, truth)
145 # VectorActions with their own files
147 def testCalcBinnedStats(self):
148 selector = RangeSelector(vectorKey="r_vector", minimum=0, maximum=self.size)
149 prefix = "a_"
150 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
151 result = stats(self.data)
152 mask = selector(self.data)
153 values = self.data["r_vector"][mask]
154 median = np.median(values)
155 truth = {
156 stats.name_mask: mask,
157 stats.name_median: median,
158 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(values - median)),
159 stats.name_count: np.sum(mask),
160 stats.name_select_maximum: np.max(values),
161 stats.name_select_median: median,
162 stats.name_select_minimum: np.min(values),
163 "range_maximum": self.size,
164 "range_minimum": 0,
165 }
166 self.assertEqual(list(result.keys()), list(truth.keys()))
168 np.testing.assert_array_almost_equal(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
169 del truth[stats.name_sigmaMad]
171 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
172 del truth[stats.name_mask]
174 for key, value in truth.items():
175 self.assertEqual(result[key], value, key)
177 # def testCalcRhoStatistics(self): TODO: implement
179 def testCalcMomentSize(self):
180 xx = self.data["r_ixx"]
181 yy = self.data["r_iyy"]
182 xy = self.data["r_ixy"]
184 # Test determinant with defaults
185 action = CalcMomentSize()
186 result = action(self.data, band="r")
187 schema = [col for col, colType in action.getInputSchema()]
188 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
189 truth = 0.25 * (xx * yy - xy**2)
190 np.testing.assert_array_almost_equal(result, truth)
192 # Test trace with columns specified
193 action = CalcMomentSize(
194 colXx="{band}_iixx",
195 colYy="{band}_iiyy",
196 colXy="{band}_iixy",
197 sizeType="trace",
198 )
199 result = action(self.data, band="g")
200 schema = [col for col, colType in action.getInputSchema()]
201 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
202 truth = np.sqrt(0.5 * (xx + yy))
203 np.testing.assert_array_almost_equal(result, truth)
205 CalcMomentSize(sizeType="trace", colXy=None).validate()
206 with self.assertRaises(FieldValidationError):
207 CalcMomentSize(sizeType="determinant", colXy=None).validate()
209 # def testCalcE(self): TODO: implement
211 # def testCalcEDiff(self): TODO: implement
213 # def testCalcE1(self): TODO: implement
215 # def testCalcE2(self): TODO: implement
217 # MathActions
219 def _testMath(self, ActionType, truth, num_vectors: int = 2, compare_exact: bool = False, **kwargs):
220 letters = ("A", "B")
221 bands = ("r", "i")
222 actions = {
223 f"action{letters[num]}": LoadVector(vectorKey=f"{{band{num+1}}}_vector")
224 for num in range(num_vectors)
225 }
226 action = ActionType(**actions, **kwargs)
227 kwargs_bands = {f"band{num+1}": bands[num] for num in range(num_vectors)}
228 result = action(self.data, **kwargs_bands)
229 self._checkSchema(action, [action.vectorKey for action in actions.values()])
230 if compare_exact:
231 np.testing.assert_array_equal(result, truth)
232 else:
233 np.testing.assert_array_almost_equal(result, truth)
235 def testConstant(self):
236 truth = [42.0]
237 action = ConstantValue(value=truth[0])
238 self._checkSchema(action, [])
239 result = action({})
240 np.testing.assert_array_equal(result, truth)
242 def testAdd(self):
243 truth = [2.0, 6.0, 12.0, 20.0, 30.0]
244 self._testMath(AddVector, truth, compare_exact=True)
246 def testSubtract(self):
247 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
248 self._testMath(SubtractVector, truth, compare_exact=True)
250 def testMultiply(self):
251 truth = [1.0, 8.0, 27.0, 64.0, 125.0]
252 self._testMath(MultiplyVector, truth, compare_exact=False)
254 def testDivide(self):
255 truth = 1 / np.arange(1, 6)
256 self._testMath(DivideVector, truth, compare_exact=False)
258 def testSqrt(self):
259 truth = np.sqrt(np.arange(1, 6))
260 self._testMath(SqrtVector, truth, compare_exact=False, num_vectors=1)
262 def testSquare(self):
263 truth = np.arange(1, 6) ** 2
264 self._testMath(SquareVector, truth, compare_exact=True, num_vectors=1)
266 def testRaiseFromBase(self):
267 power = np.arange(1, 6)
268 for base in (-2.3, 0.6):
269 truth = base**power
270 self._testMath(RaiseFromBaseVector, truth, compare_exact=False, base=base, num_vectors=1)
272 def testRaiseToPower(self):
273 base = np.arange(1, 6)
274 for power in (-2.3, 0.6):
275 truth = base**power
276 self._testMath(RaiseToPowerVector, truth, compare_exact=False, power=power, num_vectors=1)
278 def testLog10(self):
279 truth = np.log10(np.arange(1, 6))
280 self._testMath(Log10Vector, truth, compare_exact=False, num_vectors=1)
282 def testFractionalDifference(self):
283 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
284 self._testMath(FractionalDifference, truth, compare_exact=False)
286 # Basic vectorActions
288 # def testLoadVector(self): TODO: implement
290 def testDownselectVector(self):
291 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
292 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
293 result = action(self.data, band="r")
294 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
295 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
297 # def testMultiCriteriaDownselectVector(self): TODO: implement
299 # Astronomical vectorActions
301 # def testCalcSn(self): TODO: implement
303 def testConvertFluxToMag(self):
304 truth = [
305 31.4,
306 29.89485002168,
307 29.0143937264,
308 28.38970004336,
309 27.90514997832,
310 ]
311 action = ConvertFluxToMag(vectorKey="{band}_vector")
312 result = action(self.data, band="i")
313 self._checkSchema(action, ["{band}_vector"])
314 np.testing.assert_array_almost_equal(result, truth)
316 # def testConvertUnits(self): TODO: implement
318 def testMagDiff(self):
319 # Use the same units as the data so that we know the difference exactly
320 # without conversions.
321 # testExtinctionCorrectedMagDiff will test the conversions
322 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
323 action = MagDiff(
324 col1="{band1}_vector",
325 col2="{band2}_vector",
326 fluxUnits1="mag(AB)",
327 fluxUnits2="mag(AB)",
328 returnMillimags=False,
329 )
330 result = action(self.data, band1="g", band2="r")
331 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
332 np.testing.assert_array_almost_equal(result, truth.value)
334 def testExtinctionCorrectedMagDiff(self):
335 for returnMillimags in (True, False):
336 # Check that conversions are working properly
337 magDiff = MagDiff(
338 col1="{band1}_vector",
339 col2="{band2}_vector",
340 fluxUnits1="jansky",
341 fluxUnits2="jansky",
342 returnMillimags=returnMillimags,
343 )
344 action = ExtinctionCorrectedMagDiff(
345 magDiff=magDiff,
346 band1="g",
347 band2="r",
348 ebvCol="E(B-V)",
349 extinctionCoeffs={"g": 0.2, "r": 1.5},
350 )
352 result = action(self.data, band1="g", band2="r")
353 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
354 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
355 diff = lhs - rhs
356 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
357 if returnMillimags:
358 diff = diff.to(u.mmag)
359 correction = correction.to(u.mmag)
360 truth = diff - correction
361 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
362 np.testing.assert_array_almost_equal(result, truth.value)
364 # Test with hard coded bands
365 magDiff = MagDiff(
366 col1="g_vector",
367 col2="r_vector",
368 fluxUnits1="jansky",
369 fluxUnits2="jansky",
370 returnMillimags=False,
371 )
372 action = ExtinctionCorrectedMagDiff(
373 magDiff=magDiff,
374 ebvCol="E(B-V)",
375 extinctionCoeffs={"g": 0.2, "r": 1.5},
376 )
377 result = action(self.data)
378 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
379 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
380 diff = lhs - rhs
381 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
382 truth = diff - correction
383 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
384 np.testing.assert_array_almost_equal(result, truth.value)
386 # def testRAcosDec(self): TODO: implement
388 # Statistical vectorActions
390 # def testPerGroupStatistic(self): TODO: implement
392 # def testResidualWithPerGroupStatistic(self): TODO: implement
395class TestVectorSelectors(unittest.TestCase):
396 def setUp(self):
397 self.size = 20
398 falseFlags = {
399 "{band}_psfFlux_flag": [1],
400 "{band}_pixelFlags_saturatedCenter": [3],
401 "{band}_extendedness_flag": [5],
402 "xy_flag": [7],
403 "i_pixelFlags_edge": [13],
404 "r_pixelFlags_edge": [15],
405 "sky_object": [13, 15, 17],
406 }
408 trueFlags = {
409 "detect_isPatchInner": [9],
410 "detect_isDeblendedSource": [11],
411 }
413 flux = np.arange(self.size) * 10
414 fluxErr = np.ones(self.size) * 0.1
415 extendedness = np.arange(20) / 20 - 0.1
417 self.data = {
418 "r_psfFlux": flux,
419 "r_psfFluxErr": fluxErr,
420 "i_cmodelFlux": flux[::-1],
421 "i_cmodelFluxError": fluxErr[::-1],
422 "r_cmodelFlux": flux,
423 "r_cmodelFluxError": fluxErr,
424 "i_extendedness": extendedness,
425 "i_extended": extendedness,
426 }
427 bands = ("r", "i")
428 for band in bands:
429 for flag, bits in falseFlags.items():
430 vector = np.zeros(self.size, dtype=bool)
431 for bit in bits:
432 vector[bit] = 1
433 self.data[flag.format(band=band)] = vector
435 for flag, bits in trueFlags.items():
436 vector = np.ones(self.size, dtype=bool)
437 for bit in bits:
438 vector[bit] = 0
439 self.data[flag.format(band=band)] = vector
441 def _checkSchema(self, action, truth):
442 schema = [col for col, colType in action.getInputSchema()]
443 self.assertEqual(sorted(schema), sorted(truth))
445 def testFlagSelector(self):
446 selector = FlagSelector(
447 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
448 )
449 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
450 result = selector(self.data, band="r")
451 truth = np.ones(self.size, dtype=bool)
452 truth[1] = False
453 truth[9] = False
454 np.testing.assert_array_equal(result, truth)
456 def testCoaddPlotFlagSelector(self):
457 # Test defaults
458 # Bands needs to be set to something otherwise it
459 # will crash as the default value looks it up
460 # elsewhere.
461 selector = CoaddPlotFlagSelector(bands=["i"])
462 keys = [
463 "{band}_psfFlux_flag",
464 "{band}_pixelFlags_saturatedCenter",
465 "{band}_extendedness_flag",
466 "xy_flag",
467 "detect_isPatchInner",
468 "detect_isDeblendedSource",
469 ]
470 self._checkSchema(selector, keys)
472 result = selector(self.data)
473 truth = np.ones(self.size, dtype=bool)
474 for bit in (1, 3, 5, 7, 9, 11):
475 truth[bit] = 0
476 np.testing.assert_array_equal(result, truth)
478 # Test bands override
479 selector = CoaddPlotFlagSelector(
480 bands=["i", "r"],
481 selectWhenFalse=["{band}_psfFlux_flag"],
482 selectWhenTrue=["detect_isDeblendedSource"],
483 )
484 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
485 result = selector(self.data)
486 truth = np.ones(self.size, dtype=bool)
487 for bit in (1, 11):
488 truth[bit] = 0
489 np.testing.assert_array_equal(result, truth)
491 def testRangeSelector(self):
492 selector = RangeSelector(vectorKey="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
493 self._checkSchema(selector, ["r_psfFlux"])
494 result = self.data["r_psfFlux"][selector(self.data)]
495 truth = [30, 40]
496 np.testing.assert_array_equal(result, truth)
498 def testSnSelector(self):
499 # test defaults
500 selector = SnSelector()
501 keys = [
502 "{band}_psfFlux",
503 "{band}_psfFluxErr",
504 ]
505 self._checkSchema(selector, keys)
506 result = selector(self.data, bands=["r"])
507 truth = np.ones(self.size, dtype=bool)
508 truth[:6] = 0
509 np.testing.assert_array_equal(result, truth)
511 # test overrides
512 selector = SnSelector(
513 fluxType="{band}_cmodelFlux",
514 threshold=200.0,
515 uncertaintySuffix="Error",
516 bands=["r", "i"],
517 )
518 keys = [
519 "{band}_cmodelFlux",
520 "{band}_cmodelFluxError",
521 ]
522 self._checkSchema(selector, keys)
523 result = selector(self.data)
524 truth = np.ones(self.size, dtype=bool)
525 truth[:3] = 0
526 truth[-3:] = 0
527 np.testing.assert_array_equal(result, truth)
529 def testSkyObjectSelector(self):
530 # Test default
531 selector = SkyObjectSelector()
532 keys = ["{band}_pixelFlags_edge", "sky_object"]
533 self._checkSchema(selector, keys)
534 result = selector(self.data)
535 truth = np.zeros(self.size, dtype=bool)
536 truth[15] = 1
537 truth[17] = 1
538 np.testing.assert_array_equal(result, truth)
540 # Test overrides
541 selector = SkyObjectSelector(bands=["i", "r"])
542 self._checkSchema(selector, keys)
543 result = selector(self.data)
544 truth = np.zeros(self.size, dtype=bool)
545 truth[17] = 1
546 np.testing.assert_array_equal(result, truth)
548 def testStarSelector(self):
549 # test default
550 selector = StarSelector()
551 self._checkSchema(selector, ["{band}_extendedness"])
552 result = selector(self.data, band="i")
553 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
554 np.testing.assert_array_almost_equal(result, truth)
556 # Test overrides
557 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
558 result = selector(self.data, band="i")
559 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
560 np.testing.assert_array_almost_equal(result, truth)
562 def testGalaxySelector(self):
563 # test default
564 selector = GalaxySelector()
565 self._checkSchema(selector, ["{band}_extendedness"])
566 result = selector(self.data, band="i")
567 truth = self.data["i_extendedness"] > 0.5
568 np.testing.assert_array_almost_equal(result, truth)
570 # Test overrides
571 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
572 result = selector(self.data, band="i")
573 truth = self.data["i_extendedness"] > 0.3
574 np.testing.assert_array_almost_equal(result, truth)
576 def testVectorSelector(self):
577 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
578 self._checkSchema(selector, ["{band}_psfFlux_flag"])
579 result = selector(self.data, band="i")
580 truth = np.zeros(self.size, dtype=bool)
581 truth[1] = True
582 np.testing.assert_array_equal(result, truth)
585class TestKeyedDataActions(unittest.TestCase):
586 def testCalcRelativeDistances(self):
587 # To test CalcRelativeDistances, make a matched visit catalog with
588 # objects in a box slightly larger than the annulus used in calculating
589 # relative distances.
590 num_visits = 15
591 scatter_in_degrees = (5 * u.milliarcsecond).to(u.degree).value
592 obj_id = 0
593 visit_id = range(num_visits)
594 all_ras, all_decs, all_objs, all_visits = [], [], [], []
595 for ra in np.linspace(0, 6, 10):
596 for dec in np.linspace(0, 6, 10):
597 ra_degrees = (ra * u.arcmin).to(u.degree).value
598 dec_degrees = (dec * u.arcmin).to(u.degree).value
599 ra_meas = ra_degrees + np.random.rand(num_visits) * scatter_in_degrees
600 dec_meas = dec_degrees + np.random.rand(num_visits) * scatter_in_degrees
601 all_ras.append(ra_meas)
602 all_decs.append(dec_meas)
603 all_objs.append(np.ones(num_visits) * obj_id)
604 all_visits.append(visit_id)
605 obj_id += 1
606 data = pd.DataFrame(
607 {
608 "coord_ra": np.concatenate(all_ras),
609 "coord_dec": np.concatenate(all_decs),
610 "obj_index": np.concatenate(all_objs),
611 "visit": np.concatenate(all_visits),
612 }
613 )
615 task = CalcRelativeDistances()
616 res = task(data)
618 self.assertNotEqual(res["AMx"], np.nan)
619 self.assertNotEqual(res["ADx"], np.nan)
620 self.assertNotEqual(res["AFx"], np.nan)
623if __name__ == "__main__": 623 ↛ 624line 623 didn't jump to line 624, because the condition on line 623 was never true
624 lsst.utils.tests.init()
625 unittest.main()