Coverage for tests/test_actions.py: 15%
305 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 04:02 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 04:02 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import astropy.units as u
25import lsst.utils.tests
26import numpy as np
27import pandas as pd
28from lsst.analysis.tools.actions.keyedData.calcDistances import CalcRelativeDistances
29from lsst.analysis.tools.actions.scalar.scalarActions import (
30 ApproxFloor,
31 CountAction,
32 MeanAction,
33 MedianAction,
34 SigmaMadAction,
35 StdevAction,
36)
37from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction
38from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize
39from lsst.analysis.tools.actions.vector.mathActions import (
40 AddVector,
41 ConstantValue,
42 DivideVector,
43 FractionalDifference,
44 MultiplyVector,
45 SubtractVector,
46)
47from lsst.analysis.tools.actions.vector.selectors import (
48 CoaddPlotFlagSelector,
49 FlagSelector,
50 GalaxySelector,
51 RangeSelector,
52 SkyObjectSelector,
53 SnSelector,
54 StarSelector,
55 VectorSelector,
56)
57from lsst.analysis.tools.actions.vector.vectorActions import (
58 ConvertFluxToMag,
59 DownselectVector,
60 ExtinctionCorrectedMagDiff,
61 LoadVector,
62 MagDiff,
63)
66class TestScalarActions(unittest.TestCase):
67 """ "Test ScalarActions"""
69 def setUp(self):
70 x = np.arange(100, dtype=float)
71 x[31] = np.nan
72 x[41] = np.nan
73 x[59] = np.nan
74 y = x**2
75 self.data = {"r_y": x, "i_y": y}
76 self.mask = np.zeros(100, dtype=bool)
77 self.mask[1:50] = 1
79 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth):
80 action = cls(vectorKey="{band}_y")
81 schema = [col for col, colType in action.getInputSchema()]
82 self.assertEqual(schema, ["{band}_y"])
83 result = action(self.data, band="i")
84 self.assertAlmostEqual(result, truth)
85 result = action(self.data, mask=self.mask, band="i")
86 self.assertAlmostEqual(result, maskedTruth)
88 def testMedianAction(self):
89 self._testScalarActionAlmostEqual(MedianAction, 2500, 576)
91 def testMeanAction(self):
92 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766)
94 def testStdevAction(self):
95 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842)
97 def testSigmaMadAction(self):
98 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682)
100 def testCountAction(self):
101 self._testScalarActionAlmostEqual(CountAction, 97, 47)
103 def testApproxFloorAction(self):
104 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5)
107class TestVectorActions(unittest.TestCase):
108 """Test VectorActions"""
110 def setUp(self):
111 self.size = 5
112 r = np.arange(self.size) + 1
113 i = r**2
114 rFlag = np.ones(self.size)
115 rFlag[1] = 0
116 iFlag = np.ones(self.size)
117 iFlag[3] = 0
118 data = {
119 "r_vector": r,
120 "i_vector": i,
121 "r_flag": rFlag,
122 "i_flag": iFlag,
123 "g_vector": 3 * r,
124 "E(B-V)": r[::-1],
125 "r_ixx": r**2,
126 "r_iyy": r[::-1] ** 2,
127 "r_ixy": r * r[::-1],
128 "g_iixx": r**2,
129 "g_iiyy": r[::-1] ** 2,
130 "g_iixy": r * r[::-1],
131 }
133 self.data = pd.DataFrame.from_dict(data)
135 def _checkSchema(self, action, truth):
136 schema = sorted([col for col, colType in action.getInputSchema()])
137 self.assertEqual(schema, truth)
139 # VectorActions with their own files
141 def testCalcBinnedStats(self):
142 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size)
143 prefix = "a_"
144 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
145 result = stats(self.data)
146 mask = selector(self.data)
147 values = self.data["r_vector"][mask]
148 median = np.median(values)
149 truth = {
150 stats.name_mask: mask,
151 stats.name_median: median,
152 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(values - median)),
153 stats.name_count: np.sum(mask),
154 stats.name_select_maximum: np.max(values),
155 stats.name_select_median: median,
156 stats.name_select_minimum: np.min(values),
157 "range_maximum": self.size,
158 "range_minimum": 0,
159 }
160 self.assertEqual(list(result.keys()), list(truth.keys()))
162 np.testing.assert_array_almost_equal(result[stats.name_sigmaMad], truth[stats.name_sigmaMad])
163 del truth[stats.name_sigmaMad]
165 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask])
166 del truth[stats.name_mask]
168 for key, value in truth.items():
169 self.assertEqual(result[key], value, key)
171 # def testCalcRhoStatistics(self): TODO: implement
173 def testCalcShapeSize(self):
174 xx = self.data["r_ixx"]
175 yy = self.data["r_iyy"]
176 xy = self.data["r_ixy"]
178 # Test determinant with defaults
179 action = CalcShapeSize()
180 result = action(self.data, band="r")
181 schema = [col for col, colType in action.getInputSchema()]
182 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"])
183 truth = 0.25 * (xx * yy - xy**2)
184 np.testing.assert_array_almost_equal(result, truth)
186 # Test trace with columns specified
187 action = CalcShapeSize(
188 colXx="{band}_iixx",
189 colYy="{band}_iiyy",
190 colXy="{band}_iixy",
191 sizeType="trace",
192 )
193 result = action(self.data, band="g")
194 schema = [col for col, colType in action.getInputSchema()]
195 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"])
196 truth = np.sqrt(0.5 * (xx + yy))
197 np.testing.assert_array_almost_equal(result, truth)
199 # def testCalcE(self): TODO: implement
201 # def testCalcEDiff(self): TODO: implement
203 # def testCalcE1(self): TODO: implement
205 # def testCalcE2(self): TODO: implement
207 # MathActions
209 def _testMath(self, ActionType, truth, compare_exact: bool = False):
210 actionA = LoadVector(vectorKey="{band1}_vector")
211 actionB = LoadVector(vectorKey="{band2}_vector")
212 action = ActionType(actionA=actionA, actionB=actionB)
213 result = action(self.data, band1="r", band2="i")
214 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey])
215 if compare_exact:
216 np.testing.assert_array_equal(result, truth)
217 else:
218 np.testing.assert_array_almost_equal(result, truth)
220 def testConstant(self):
221 truth = [42.0]
222 action = ConstantValue(value=truth[0])
223 self._checkSchema(action, [])
224 result = action({})
225 np.testing.assert_array_equal(result, truth)
227 def testAdd(self):
228 truth = [2.0, 6.0, 12.0, 20.0, 30.0]
229 self._testMath(AddVector, truth, True)
231 def testSubtract(self):
232 truth = [0.0, -2.0, -6.0, -12.0, -20.0]
233 self._testMath(SubtractVector, truth, True)
235 def testMultiply(self):
236 truth = [1.0, 8.0, 27.0, 64.0, 125.0]
237 self._testMath(MultiplyVector, truth, False)
239 def testDivide(self):
240 truth = 1 / np.arange(1, 6)
241 self._testMath(DivideVector, truth, False)
243 def testFractionalDifference(self):
244 actionA = LoadVector(vectorKey="{band1}_vector")
245 actionB = LoadVector(vectorKey="{band2}_vector")
246 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8]
247 diff = FractionalDifference(actionA=actionA, actionB=actionB)
248 result = diff(self.data, band1="r", band2="i")
249 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"])
250 np.testing.assert_array_almost_equal(result, truth)
252 # Basic vectorActions
254 # def testLoadVector(self): TODO: implement
256 def testDownselectVector(self):
257 selector = FlagSelector(selectWhenTrue=["{band}_flag"])
258 action = DownselectVector(vectorKey="{band}_vector", selector=selector)
259 result = action(self.data, band="r")
260 self._checkSchema(action, ["{band}_flag", "{band}_vector"])
261 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5]))
263 # def testMultiCriteriaDownselectVector(self): TODO: implement
265 # Astronomical vectorActions
267 # def testCalcSn(self): TODO: implement
269 def testConvertFluxToMag(self):
270 truth = [
271 31.4,
272 29.89485002168,
273 29.0143937264,
274 28.38970004336,
275 27.90514997832,
276 ]
277 action = ConvertFluxToMag(vectorKey="{band}_vector")
278 result = action(self.data, band="i")
279 self._checkSchema(action, ["{band}_vector"])
280 np.testing.assert_array_almost_equal(result, truth)
282 # def testConvertUnits(self): TODO: implement
284 def testMagDiff(self):
285 # Use the same units as the data so that we know the difference exactly
286 # without conversions.
287 # testExtinctionCorrectedMagDiff will test the conversions
288 truth = np.array(2 * self.data["r_vector"]) * u.ABmag
289 action = MagDiff(
290 col1="{band1}_vector",
291 col2="{band2}_vector",
292 fluxUnits1="mag(AB)",
293 fluxUnits2="mag(AB)",
294 returnMillimags=False,
295 )
296 result = action(self.data, band1="g", band2="r")
297 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"])
298 np.testing.assert_array_almost_equal(result, truth.value)
300 def testExtinctionCorrectedMagDiff(self):
301 for returnMillimags in (True, False):
302 # Check that conversions are working properly
303 magDiff = MagDiff(
304 col1="{band1}_vector",
305 col2="{band2}_vector",
306 fluxUnits1="jansky",
307 fluxUnits2="jansky",
308 returnMillimags=returnMillimags,
309 )
310 action = ExtinctionCorrectedMagDiff(
311 magDiff=magDiff,
312 band1="g",
313 band2="r",
314 ebvCol="E(B-V)",
315 extinctionCoeffs={"g": 0.2, "r": 1.5},
316 )
318 result = action(self.data, band1="g", band2="r")
319 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
320 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
321 diff = lhs - rhs
322 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
323 if returnMillimags:
324 diff = diff.to(u.mmag)
325 correction = correction.to(u.mmag)
326 truth = diff - correction
327 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"])
328 np.testing.assert_array_almost_equal(result, truth.value)
330 # Test with hard coded bands
331 magDiff = MagDiff(
332 col1="g_vector",
333 col2="r_vector",
334 fluxUnits1="jansky",
335 fluxUnits2="jansky",
336 returnMillimags=False,
337 )
338 action = ExtinctionCorrectedMagDiff(
339 magDiff=magDiff,
340 ebvCol="E(B-V)",
341 extinctionCoeffs={"g": 0.2, "r": 1.5},
342 )
343 result = action(self.data)
344 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag)
345 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag)
346 diff = lhs - rhs
347 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag
348 truth = diff - correction
349 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"])
350 np.testing.assert_array_almost_equal(result, truth.value)
352 # def testRAcosDec(self): TODO: implement
354 # Statistical vectorActions
356 # def testPerGroupStatistic(self): TODO: implement
358 # def testResidualWithPerGroupStatistic(self): TODO: implement
361class TestVectorSelectors(unittest.TestCase):
362 def setUp(self):
363 self.size = 20
364 falseFlags = {
365 "{band}_psfFlux_flag": [1],
366 "{band}_pixelFlags_saturatedCenter": [3],
367 "{band}_extendedness_flag": [5],
368 "xy_flag": [7],
369 "i_pixelFlags_edge": [13],
370 "r_pixelFlags_edge": [15],
371 "sky_object": [13, 15, 17],
372 }
374 trueFlags = {
375 "detect_isPatchInner": [9],
376 "detect_isDeblendedSource": [11],
377 }
379 flux = np.arange(self.size) * 10
380 fluxErr = np.ones(self.size) * 0.1
381 extendedness = np.arange(20) / 20 - 0.1
383 self.data = {
384 "r_psfFlux": flux,
385 "r_psfFluxErr": fluxErr,
386 "i_cmodelFlux": flux[::-1],
387 "i_cmodelFluxError": fluxErr[::-1],
388 "r_cmodelFlux": flux,
389 "r_cmodelFluxError": fluxErr,
390 "i_extendedness": extendedness,
391 "i_extended": extendedness,
392 }
393 bands = ("r", "i")
394 for band in bands:
395 for flag, bits in falseFlags.items():
396 vector = np.zeros(self.size, dtype=bool)
397 for bit in bits:
398 vector[bit] = 1
399 self.data[flag.format(band=band)] = vector
401 for flag, bits in trueFlags.items():
402 vector = np.ones(self.size, dtype=bool)
403 for bit in bits:
404 vector[bit] = 0
405 self.data[flag.format(band=band)] = vector
407 def _checkSchema(self, action, truth):
408 schema = [col for col, colType in action.getInputSchema()]
409 self.assertEqual(sorted(schema), sorted(truth))
411 def testFlagSelector(self):
412 selector = FlagSelector(
413 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"]
414 )
415 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"])
416 result = selector(self.data, band="r")
417 truth = np.ones(self.size, dtype=bool)
418 truth[1] = False
419 truth[9] = False
420 np.testing.assert_array_equal(result, truth)
422 def testCoaddPlotFlagSelector(self):
423 # Test defaults
424 # Bands needs to be set to something otherwise it
425 # will crash as the default value looks it up
426 # elsewhere.
427 selector = CoaddPlotFlagSelector(bands=["i"])
428 keys = [
429 "{band}_psfFlux_flag",
430 "{band}_pixelFlags_saturatedCenter",
431 "{band}_extendedness_flag",
432 "xy_flag",
433 "detect_isPatchInner",
434 "detect_isDeblendedSource",
435 ]
436 self._checkSchema(selector, keys)
438 result = selector(self.data)
439 truth = np.ones(self.size, dtype=bool)
440 for bit in (1, 3, 5, 7, 9, 11):
441 truth[bit] = 0
442 np.testing.assert_array_equal(result, truth)
444 # Test bands override
445 selector = CoaddPlotFlagSelector(
446 bands=["i", "r"],
447 selectWhenFalse=["{band}_psfFlux_flag"],
448 selectWhenTrue=["detect_isDeblendedSource"],
449 )
450 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"])
451 result = selector(self.data)
452 truth = np.ones(self.size, dtype=bool)
453 for bit in (1, 11):
454 truth[bit] = 0
455 np.testing.assert_array_equal(result, truth)
457 def testRangeSelector(self):
458 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
459 self._checkSchema(selector, ["r_psfFlux"])
460 result = self.data["r_psfFlux"][selector(self.data)]
461 truth = [30, 40]
462 np.testing.assert_array_equal(result, truth)
464 def testSnSelector(self):
465 # test defaults
466 selector = SnSelector()
467 keys = [
468 "{band}_psfFlux",
469 "{band}_psfFluxErr",
470 ]
471 self._checkSchema(selector, keys)
472 result = selector(self.data, bands=["r"])
473 truth = np.ones(self.size, dtype=bool)
474 truth[:6] = 0
475 np.testing.assert_array_equal(result, truth)
477 # test overrides
478 selector = SnSelector(
479 fluxType="{band}_cmodelFlux",
480 threshold=200.0,
481 uncertaintySuffix="Error",
482 bands=["r", "i"],
483 )
484 keys = [
485 "{band}_cmodelFlux",
486 "{band}_cmodelFluxError",
487 ]
488 self._checkSchema(selector, keys)
489 result = selector(self.data)
490 truth = np.ones(self.size, dtype=bool)
491 truth[:3] = 0
492 truth[-3:] = 0
493 np.testing.assert_array_equal(result, truth)
495 def testSkyObjectSelector(self):
496 # Test default
497 selector = SkyObjectSelector()
498 keys = ["{band}_pixelFlags_edge", "sky_object"]
499 self._checkSchema(selector, keys)
500 result = selector(self.data)
501 truth = np.zeros(self.size, dtype=bool)
502 truth[15] = 1
503 truth[17] = 1
504 np.testing.assert_array_equal(result, truth)
506 # Test overrides
507 selector = SkyObjectSelector(bands=["i", "r"])
508 self._checkSchema(selector, keys)
509 result = selector(self.data)
510 truth = np.zeros(self.size, dtype=bool)
511 truth[17] = 1
512 np.testing.assert_array_equal(result, truth)
514 def testStarSelector(self):
515 # test default
516 selector = StarSelector()
517 self._checkSchema(selector, ["{band}_extendedness"])
518 result = selector(self.data, band="i")
519 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5)
520 np.testing.assert_array_almost_equal(result, truth)
522 # Test overrides
523 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3)
524 result = selector(self.data, band="i")
525 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3)
526 np.testing.assert_array_almost_equal(result, truth)
528 def testGalaxySelector(self):
529 # test default
530 selector = GalaxySelector()
531 self._checkSchema(selector, ["{band}_extendedness"])
532 result = selector(self.data, band="i")
533 truth = self.data["i_extendedness"] > 0.5
534 np.testing.assert_array_almost_equal(result, truth)
536 # Test overrides
537 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3)
538 result = selector(self.data, band="i")
539 truth = self.data["i_extendedness"] > 0.3
540 np.testing.assert_array_almost_equal(result, truth)
542 def testVectorSelector(self):
543 selector = VectorSelector(vectorKey="{band}_psfFlux_flag")
544 self._checkSchema(selector, ["{band}_psfFlux_flag"])
545 result = selector(self.data, band="i")
546 truth = np.zeros(self.size, dtype=bool)
547 truth[1] = True
548 np.testing.assert_array_equal(result, truth)
551class TestKeyedDataActions(unittest.TestCase):
552 def testCalcRelativeDistances(self):
553 # To test CalcRelativeDistances, make a matched visit catalog with
554 # objects in a box slightly larger than the annulus used in calculating
555 # relative distances.
556 num_visits = 15
557 scatter_in_degrees = (5 * u.milliarcsecond).to(u.degree).value
558 obj_id = 0
559 visit_id = range(num_visits)
560 all_ras, all_decs, all_objs, all_visits = [], [], [], []
561 for ra in np.linspace(0, 6, 10):
562 for dec in np.linspace(0, 6, 10):
563 ra_degrees = (ra * u.arcmin).to(u.degree).value
564 dec_degrees = (dec * u.arcmin).to(u.degree).value
565 ra_meas = ra_degrees + np.random.rand(num_visits) * scatter_in_degrees
566 dec_meas = dec_degrees + np.random.rand(num_visits) * scatter_in_degrees
567 all_ras.append(ra_meas)
568 all_decs.append(dec_meas)
569 all_objs.append(np.ones(num_visits) * obj_id)
570 all_visits.append(visit_id)
571 obj_id += 1
572 data = pd.DataFrame(
573 {
574 "coord_ra": np.concatenate(all_ras),
575 "coord_dec": np.concatenate(all_decs),
576 "obj_index": np.concatenate(all_objs),
577 "visit": np.concatenate(all_visits),
578 }
579 )
581 task = CalcRelativeDistances()
582 res = task(data)
584 self.assertNotEqual(res["AMx"], np.nan)
585 self.assertNotEqual(res["ADx"], np.nan)
586 self.assertNotEqual(res["AFx"], np.nan)
589if __name__ == "__main__": 589 ↛ 590line 589 didn't jump to line 590, because the condition on line 589 was never true
590 lsst.utils.tests.init()
591 unittest.main()