Coverage for tests/test_isolatedStarAssociation.py: 11%
214 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-08 01:41 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-08 01:41 -0700
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""Test IsolatedStarAssociationTask.
23"""
24import unittest
25import numpy as np
26import pandas as pd
28import lsst.utils.tests
29import lsst.pipe.base
30import lsst.skymap
32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig,
33 IsolatedStarAssociationTask)
34from smatch.matcher import Matcher
37class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase):
38 """Tests of IsolatedStarAssociationTask.
40 These tests bypass the middleware used for accessing data and
41 managing Task execution.
42 """
43 def setUp(self):
44 self.skymap = self._make_skymap()
45 self.tract = 9813
46 self.data_refs = self._make_simdata(self.tract)
47 self.visits = np.arange(len(self.data_refs)) + 1
49 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
50 self.data_refs)}
52 config = IsolatedStarAssociationConfig()
53 config.band_order = ['i', 'r']
54 config.extra_columns = ['extra_column']
55 config.source_selector['science'].doFlags = False
56 config.source_selector['science'].doIsolated = False
58 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config)
60 def _make_skymap(self):
61 """Make a testing skymap."""
62 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig()
63 skymap_config.numRings = 120
64 skymap_config.projection = "TAN"
65 skymap_config.tractOverlap = 1.0/60
66 skymap_config.pixelScale = 0.168
67 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config)
69 def _make_simdata(self,
70 tract,
71 only_neighbors=False,
72 only_out_of_tract=False,
73 only_out_of_inner_tract=False,
74 no_secondary_overlap=False):
75 """Make simulated data tables and references.
77 Parameters
78 ----------
79 only_neighbors : `bool`, optional
80 Only put in neighbors.
81 only_out_of_tract : `bool`, optional
82 All stars are out of the tract.
83 only_out_of_inner_tract : `bool`, optional
84 All stars are out of the inner tract.
85 no_secondary_overlap : `bool`, optional
86 Secondary band has no overlap with the primary band.
88 Returns
89 -------
90 data_refs : `list` [`InMemoryDatasetHandle`]
91 List of mock references.
92 """
93 np.random.seed(12345)
95 n_visit_per_band = 5
96 n_star_both = 50
97 n_star_just_one = 5
99 tract_info = self.skymap[tract]
100 ctr = tract_info.ctr_coord
101 ctr_ra = ctr.getRa().asDegrees()
102 ctr_dec = ctr.getDec().asDegrees()
104 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both)
105 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both)
107 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
108 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one)
109 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
110 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one)
112 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.])
113 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.])
115 # Create the r-band datarefs
116 dtype = [('sourceId', 'i8'),
117 ('ra', 'f8'),
118 ('decl', 'f8'),
119 ('apFlux_12_0_instFlux', 'f4'),
120 ('apFlux_12_0_instFluxErr', 'f4'),
121 ('apFlux_12_0_instFlux_flag', '?'),
122 ('extendedness', 'f4'),
123 ('visit', 'i4'),
124 ('detector', 'i4'),
125 ('physical_filter', 'U10'),
126 ('band', 'U2'),
127 ('extra_column', 'f4')]
129 id_counter = 0
130 visit_counter = 1
132 data_refs = []
133 for band in ['r', 'i']:
134 if band == 'r':
135 filtername = 'R FILTER'
136 ra_just = ra_just_r
137 dec_just = dec_just_r
138 else:
139 filtername = 'I FILTER'
140 ra_just = ra_just_i
141 dec_just = dec_just_i
143 if only_neighbors:
144 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
145 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
146 elif no_secondary_overlap:
147 star_ra = np.concatenate((ra_just,))
148 star_dec = np.concatenate((dec_just,))
149 else:
150 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just))
151 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just))
153 if only_out_of_tract:
154 poly = self.skymap[self.tract].outer_sky_polygon
155 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec))
156 star_ra = star_ra[use]
157 star_dec = star_dec[use]
158 elif only_out_of_inner_tract:
159 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True)
160 use = (inner_tract_ids != self.tract)
161 star_ra = star_ra[use]
162 star_dec = star_dec[use]
164 nstar = len(star_ra)
166 for i in range(n_visit_per_band):
167 ras = np.random.normal(loc=star_ra, scale=0.2/3600.)
168 decs = np.random.normal(loc=star_dec, scale=0.2/3600.)
170 table = np.zeros(nstar, dtype=dtype)
171 table['sourceId'] = np.arange(nstar) + id_counter
172 table['ra'] = ras
173 table['decl'] = decs
174 table['apFlux_12_0_instFlux'] = 100.0
175 table['apFlux_12_0_instFluxErr'] = 1.0
176 table['physical_filter'] = filtername
177 table['band'] = band
178 table['extra_column'] = np.ones(nstar)
179 table['visit'] = visit_counter
180 table['detector'] = 1
182 if i == 0:
183 # Make one star have low s/n
184 table['apFlux_12_0_instFlux'][0] = 1.0
186 df = pd.DataFrame(table)
187 df.set_index('sourceId', inplace=True)
188 data_refs.append(lsst.pipe.base.InMemoryDatasetHandle(df, storageClass="DataFrame"))
190 id_counter += nstar
191 visit_counter += 1
193 self.n_visit_per_band = n_visit_per_band
194 self.n_star_both = n_star_both
195 self.n_star_just_one = n_star_just_one
196 if only_neighbors:
197 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
198 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
199 else:
200 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor))
201 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor))
203 return data_refs
205 def test_compute_unique_ids(self):
206 """Test computation of unique ids."""
207 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
208 9813,
209 10000)
210 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
211 9814,
212 5000)
213 ids = np.concatenate((ids1, ids2))
214 self.assertEqual(len(np.unique(ids)), len(ids))
216 def test_remove_neighbors(self):
217 """Test removing close neighbors."""
218 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'),
219 ('decl', 'f8')])
221 # Put two stars < 2" apart, and across the 0/360 boundary.
222 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0]
223 primary_star_cat['decl'] = [5.0, 5.0, 5.0]
225 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat)
227 self.assertEqual(len(cut_cat), 1)
228 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0])
230 def test_match_primary_stars(self):
231 """Test matching primary stars."""
232 # Stack all the sources; we do not want any cutting here.
233 tables = []
234 for data_ref in self.data_refs:
235 df = data_ref.get()
236 tables.append(df.to_records())
237 source_cat = np.concatenate(tables)
239 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'],
240 source_cat)
242 # Ensure we found the right number of stars in each p
243 test_i = (primary_star_cat['primary_band'] == 'i')
244 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1)
245 test_r = (primary_star_cat['primary_band'] == 'r')
246 self.assertEqual(test_r.sum(), self.n_star_just_one)
248 # Ensure that these stars all match to input stars within 1 arcsec.
249 with Matcher(self.star_ras, self.star_decs) as matcher:
250 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'],
251 primary_star_cat['decl'],
252 1./3600.,
253 return_indices=True)
254 self.assertEqual(i1.size, self.star_ras.size)
256 def test_get_source_table_visit_columns(self):
257 """Test making of source table visit columns."""
258 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names()
260 # Make sure all persisted columns are in all columns.
261 for col in persist_columns:
262 self.assertTrue(col in all_columns)
264 # And make sure extendedness is not in persisted columns.
265 self.assertTrue('extendedness' not in persist_columns)
267 def test_match_sources(self):
268 """Test _match_sources source to primary matching."""
269 # Stack all the sources; we do not want any cutting here.
270 tables = []
271 for data_ref in self.data_refs:
272 df = data_ref.get()
273 tables.append(df.to_records())
274 source_cat = np.concatenate(tables)
276 source_cat = np.lib.recfunctions.append_fields(source_cat,
277 ['obj_index'],
278 [np.zeros(source_cat.size, dtype=np.int32)],
279 dtypes=['i4'],
280 usemask=False)
282 primary_bands = ['i', 'r']
284 primary_cat = np.zeros(self.star_ras.size,
285 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands))
286 primary_cat['ra'] = self.star_ras
287 primary_cat['decl'] = self.star_decs
289 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'],
290 source_cat,
291 primary_cat)
292 # All the star sources should be matched
293 self.assertEqual(source_cat_sorted.size, source_cat.size)
295 # Full index tests are performed in test_run_isolated_star_association_task
297 def test_make_all_star_sources(self):
298 """Test appending all the star sources."""
299 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract],
300 self.data_ref_dict)
302 # Make sure we don't have any low s/n sources.
303 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr'])
304 self.assertGreater(sn_min, 10.0)
306 # And make sure they are all within the tract outer boundary.
307 poly = self.skymap[self.tract].outer_sky_polygon
308 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl']))
309 self.assertEqual(use.sum(), len(source_cat))
311 def test_run_isolated_star_association_task(self):
312 """Test running the full task."""
313 struct = self.isolatedStarAssociationTask.run(self.skymap,
314 self.tract,
315 self.data_ref_dict)
317 star_source_cat = struct.star_source_cat
318 star_cat = struct.star_cat
320 # Check that sources are all unique ids
321 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size)
323 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras,
324 self.star_decs,
325 degrees=True)
326 inner_stars = (inner_tract_ids == self.tract)
328 # There should be the same number of stars in the inner tract region,
329 # taking away the 2 close neighbors.
330 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2)
332 # Check the star indices
333 for i in range(len(star_cat)):
334 all_source_star = star_source_cat[star_cat['source_cat_index'][i]:
335 star_cat['source_cat_index'][i] + star_cat['nsource'][i]]
337 # Check that these all point to the correct object
338 np.testing.assert_array_equal(all_source_star['obj_index'], i)
340 # Check these are all pointing to the same star position
341 with Matcher(np.atleast_1d(star_cat['ra'][i]),
342 np.atleast_1d(star_cat['decl'][i])) as matcher:
343 idx = matcher.query_radius(all_source_star['ra'],
344 all_source_star['decl'],
345 1./3600.)
346 self.assertEqual(len(idx[0]), star_cat['nsource'][i])
348 # Check per band indices
349 for band in ['r', 'i']:
350 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]:
351 star_cat[f'source_cat_index_{band}'][i]
352 + star_cat[f'nsource_{band}'][i]]
353 with Matcher(np.atleast_1d(star_cat['ra'][i]),
354 np.atleast_1d(star_cat['decl'][i])) as matcher:
355 idx = matcher.query_radius(band_source_star['ra'],
356 band_source_star['decl'],
357 1./3600.)
358 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i])
360 def test_run_task_all_neighbors(self):
361 """Test running the task when all the stars are rejected as neighbors."""
362 data_refs = self._make_simdata(self.tract, only_neighbors=True)
363 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
364 data_refs)}
366 struct = self.isolatedStarAssociationTask.run(self.skymap,
367 self.tract,
368 data_ref_dict)
370 # These should ber zero length.
371 self.assertEqual(len(struct.star_source_cat), 0)
372 self.assertEqual(len(struct.star_cat), 0)
373 # And spot-check a couple of expected fields to make sure they have the right type.
374 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
375 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
377 def test_run_task_all_out_of_tract(self):
378 """Test running the task when all the sources are out of the tract."""
379 data_refs = self._make_simdata(self.tract, only_out_of_tract=True)
380 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
381 data_refs)}
383 struct = self.isolatedStarAssociationTask.run(self.skymap,
384 self.tract,
385 data_ref_dict)
387 # These should ber zero length.
388 self.assertEqual(len(struct.star_source_cat), 0)
389 self.assertEqual(len(struct.star_cat), 0)
390 # And spot-check a couple of expected fields to make sure they have the right type.
391 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
392 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
394 def test_run_task_all_out_of_inner_tract(self):
395 """Test running the task when all the sources are out of the inner tract."""
396 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True)
397 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
398 data_refs)}
400 struct = self.isolatedStarAssociationTask.run(self.skymap,
401 self.tract,
402 data_ref_dict)
404 # These should ber zero length.
405 self.assertEqual(len(struct.star_source_cat), 0)
406 self.assertEqual(len(struct.star_cat), 0)
407 # And spot-check a couple of expected fields to make sure they have the right type.
408 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
409 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
411 def test_run_task_secondary_no_overlap(self):
412 """Test running the task when the secondary band has no overlaps.
414 This tests DM-34834.
415 """
416 data_refs = self._make_simdata(self.tract, no_secondary_overlap=True)
417 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
418 data_refs)}
420 struct = self.isolatedStarAssociationTask.run(self.skymap,
421 self.tract,
422 data_ref_dict)
424 # Add a sanity check that we got a catalog out.
425 self.assertGreater(len(struct.star_source_cat), 0)
426 self.assertGreater(len(struct.star_cat), 0)
429class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
430 pass
433def setup_module(module):
434 lsst.utils.tests.init()
437if __name__ == "__main__": 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true
438 lsst.utils.tests.init()
439 unittest.main()