Coverage for tests/test_isolatedStarAssociation.py: 12%
224 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-18 02:44 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-18 02:44 -0800
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""Test IsolatedStarAssociationTask.
23"""
24import unittest
25import numpy as np
26import pandas as pd
28import lsst.utils.tests
29import lsst.daf.butler
30import lsst.skymap
32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig,
33 IsolatedStarAssociationTask)
34from smatch.matcher import Matcher
37class MockSourceTableReference(lsst.daf.butler.DeferredDatasetHandle):
38 """Very simple object that looks like a Gen3 data reference to
39 a sourceTable_visit.
41 Parameters
42 ----------
43 source_table : `pandas.DataFrame`
44 Dataframe of the source table.
45 """
46 def __init__(self, source_table):
47 self.source_table = source_table
49 def get(self, parameters={}, **kwargs):
50 """Retrieve the specified dataset using the API of the Gen3 Butler.
52 Parameters
53 ----------
54 parameters : `dict`, optional
55 Parameter dictionary. Supported key is ``columns``.
57 Returns
58 -------
59 dataframe : `pandas.DataFrame`
60 dataframe, cut to the specified columns.
61 """
62 if 'columns' in parameters:
63 _columns = parameters['columns']
64 if 'sourceId' in parameters['columns']:
65 # Treat the index separately
66 _columns.remove('sourceId')
68 return self.source_table[_columns]
69 else:
70 return self.source_table.copy()
73class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase):
74 """Tests of IsolatedStarAssociationTask.
76 These tests bypass the middleware used for accessing data and
77 managing Task execution.
78 """
79 def setUp(self):
80 self.skymap = self._make_skymap()
81 self.tract = 9813
82 self.data_refs = self._make_simdata(self.tract)
83 self.visits = np.arange(len(self.data_refs)) + 1
85 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
86 self.data_refs)}
88 config = IsolatedStarAssociationConfig()
89 config.band_order = ['i', 'r']
90 config.extra_columns = ['extra_column']
91 config.source_selector['science'].doFlags = False
92 config.source_selector['science'].doIsolated = False
94 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config)
96 def _make_skymap(self):
97 """Make a testing skymap."""
98 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig()
99 skymap_config.numRings = 120
100 skymap_config.projection = "TAN"
101 skymap_config.tractOverlap = 1.0/60
102 skymap_config.pixelScale = 0.168
103 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config)
105 def _make_simdata(self,
106 tract,
107 only_neighbors=False,
108 only_out_of_tract=False,
109 only_out_of_inner_tract=False,
110 no_secondary_overlap=False):
111 """Make simulated data tables and references.
113 Parameters
114 ----------
115 only_neighbors : `bool`, optional
116 Only put in neighbors.
117 only_out_of_tract : `bool`, optional
118 All stars are out of the tract.
119 only_out_of_inner_tract : `bool`, optional
120 All stars are out of the inner tract.
121 no_secondary_overlap : `bool`, optional
122 Secondary band has no overlap with the primary band.
124 Returns
125 -------
126 data_refs : `list` [`MockSourceTableReference`]
127 List of mock references.
128 """
129 np.random.seed(12345)
131 n_visit_per_band = 5
132 n_star_both = 50
133 n_star_just_one = 5
135 tract_info = self.skymap[tract]
136 ctr = tract_info.ctr_coord
137 ctr_ra = ctr.getRa().asDegrees()
138 ctr_dec = ctr.getDec().asDegrees()
140 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both)
141 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both)
143 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
144 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one)
145 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
146 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one)
148 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.])
149 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.])
151 # Create the r-band datarefs
152 dtype = [('sourceId', 'i8'),
153 ('ra', 'f8'),
154 ('decl', 'f8'),
155 ('apFlux_12_0_instFlux', 'f4'),
156 ('apFlux_12_0_instFluxErr', 'f4'),
157 ('apFlux_12_0_instFlux_flag', '?'),
158 ('extendedness', 'f4'),
159 ('visit', 'i4'),
160 ('detector', 'i4'),
161 ('physical_filter', 'U10'),
162 ('band', 'U2'),
163 ('extra_column', 'f4')]
165 id_counter = 0
166 visit_counter = 1
168 data_refs = []
169 for band in ['r', 'i']:
170 if band == 'r':
171 filtername = 'R FILTER'
172 ra_just = ra_just_r
173 dec_just = dec_just_r
174 else:
175 filtername = 'I FILTER'
176 ra_just = ra_just_i
177 dec_just = dec_just_i
179 if only_neighbors:
180 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
181 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
182 elif no_secondary_overlap:
183 star_ra = np.concatenate((ra_just,))
184 star_dec = np.concatenate((dec_just,))
185 else:
186 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just))
187 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just))
189 if only_out_of_tract:
190 poly = self.skymap[self.tract].outer_sky_polygon
191 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec))
192 star_ra = star_ra[use]
193 star_dec = star_dec[use]
194 elif only_out_of_inner_tract:
195 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True)
196 use = (inner_tract_ids != self.tract)
197 star_ra = star_ra[use]
198 star_dec = star_dec[use]
200 nstar = len(star_ra)
202 for i in range(n_visit_per_band):
203 ras = np.random.normal(loc=star_ra, scale=0.2/3600.)
204 decs = np.random.normal(loc=star_dec, scale=0.2/3600.)
206 table = np.zeros(nstar, dtype=dtype)
207 table['sourceId'] = np.arange(nstar) + id_counter
208 table['ra'] = ras
209 table['decl'] = decs
210 table['apFlux_12_0_instFlux'] = 100.0
211 table['apFlux_12_0_instFluxErr'] = 1.0
212 table['physical_filter'] = filtername
213 table['band'] = band
214 table['extra_column'] = np.ones(nstar)
215 table['visit'] = visit_counter
216 table['detector'] = 1
218 if i == 0:
219 # Make one star have low s/n
220 table['apFlux_12_0_instFlux'][0] = 1.0
222 df = pd.DataFrame(table)
223 df.set_index('sourceId', inplace=True)
224 data_refs.append(MockSourceTableReference(df))
226 id_counter += nstar
227 visit_counter += 1
229 self.n_visit_per_band = n_visit_per_band
230 self.n_star_both = n_star_both
231 self.n_star_just_one = n_star_just_one
232 if only_neighbors:
233 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
234 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
235 else:
236 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor))
237 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor))
239 return data_refs
241 def test_compute_unique_ids(self):
242 """Test computation of unique ids."""
243 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
244 9813,
245 10000)
246 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
247 9814,
248 5000)
249 ids = np.concatenate((ids1, ids2))
250 self.assertEqual(len(np.unique(ids)), len(ids))
252 def test_remove_neighbors(self):
253 """Test removing close neighbors."""
254 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'),
255 ('decl', 'f8')])
257 # Put two stars < 2" apart, and across the 0/360 boundary.
258 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0]
259 primary_star_cat['decl'] = [5.0, 5.0, 5.0]
261 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat)
263 self.assertEqual(len(cut_cat), 1)
264 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0])
266 def test_match_primary_stars(self):
267 """Test matching primary stars."""
268 # Stack all the sources; we do not want any cutting here.
269 tables = []
270 for data_ref in self.data_refs:
271 df = data_ref.get()
272 tables.append(df.to_records())
273 source_cat = np.concatenate(tables)
275 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'],
276 source_cat)
278 # Ensure we found the right number of stars in each p
279 test_i = (primary_star_cat['primary_band'] == 'i')
280 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1)
281 test_r = (primary_star_cat['primary_band'] == 'r')
282 self.assertEqual(test_r.sum(), self.n_star_just_one)
284 # Ensure that these stars all match to input stars within 1 arcsec.
285 with Matcher(self.star_ras, self.star_decs) as matcher:
286 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'],
287 primary_star_cat['decl'],
288 1./3600.,
289 return_indices=True)
290 self.assertEqual(i1.size, self.star_ras.size)
292 def test_get_source_table_visit_columns(self):
293 """Test making of source table visit columns."""
294 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names()
296 # Make sure all persisted columns are in all columns.
297 for col in persist_columns:
298 self.assertTrue(col in all_columns)
300 # And make sure extendedness is not in persisted columns.
301 self.assertTrue('extendedness' not in persist_columns)
303 def test_match_sources(self):
304 """Test _match_sources source to primary matching."""
305 # Stack all the sources; we do not want any cutting here.
306 tables = []
307 for data_ref in self.data_refs:
308 df = data_ref.get()
309 tables.append(df.to_records())
310 source_cat = np.concatenate(tables)
312 source_cat = np.lib.recfunctions.append_fields(source_cat,
313 ['obj_index'],
314 [np.zeros(source_cat.size, dtype=np.int32)],
315 dtypes=['i4'],
316 usemask=False)
318 primary_bands = ['i', 'r']
320 primary_cat = np.zeros(self.star_ras.size,
321 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands))
322 primary_cat['ra'] = self.star_ras
323 primary_cat['decl'] = self.star_decs
325 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'],
326 source_cat,
327 primary_cat)
328 # All the star sources should be matched
329 self.assertEqual(source_cat_sorted.size, source_cat.size)
331 # Full index tests are performed in test_run_isolated_star_association_task
333 def test_make_all_star_sources(self):
334 """Test appending all the star sources."""
335 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract],
336 self.data_ref_dict)
338 # Make sure we don't have any low s/n sources.
339 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr'])
340 self.assertGreater(sn_min, 10.0)
342 # And make sure they are all within the tract outer boundary.
343 poly = self.skymap[self.tract].outer_sky_polygon
344 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl']))
345 self.assertEqual(use.sum(), len(source_cat))
347 def test_run_isolated_star_association_task(self):
348 """Test running the full task."""
349 struct = self.isolatedStarAssociationTask.run(self.skymap,
350 self.tract,
351 self.data_ref_dict)
353 star_source_cat = struct.star_source_cat
354 star_cat = struct.star_cat
356 # Check that sources are all unique ids
357 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size)
359 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras,
360 self.star_decs,
361 degrees=True)
362 inner_stars = (inner_tract_ids == self.tract)
364 # There should be the same number of stars in the inner tract region,
365 # taking away the 2 close neighbors.
366 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2)
368 # Check the star indices
369 for i in range(len(star_cat)):
370 all_source_star = star_source_cat[star_cat['source_cat_index'][i]:
371 star_cat['source_cat_index'][i] + star_cat['nsource'][i]]
373 # Check that these all point to the correct object
374 np.testing.assert_array_equal(all_source_star['obj_index'], i)
376 # Check these are all pointing to the same star position
377 with Matcher(np.atleast_1d(star_cat['ra'][i]),
378 np.atleast_1d(star_cat['decl'][i])) as matcher:
379 idx = matcher.query_radius(all_source_star['ra'],
380 all_source_star['decl'],
381 1./3600.)
382 self.assertEqual(len(idx[0]), star_cat['nsource'][i])
384 # Check per band indices
385 for band in ['r', 'i']:
386 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]:
387 star_cat[f'source_cat_index_{band}'][i]
388 + star_cat[f'nsource_{band}'][i]]
389 with Matcher(np.atleast_1d(star_cat['ra'][i]),
390 np.atleast_1d(star_cat['decl'][i])) as matcher:
391 idx = matcher.query_radius(band_source_star['ra'],
392 band_source_star['decl'],
393 1./3600.)
394 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i])
396 def test_run_task_all_neighbors(self):
397 """Test running the task when all the stars are rejected as neighbors."""
398 data_refs = self._make_simdata(self.tract, only_neighbors=True)
399 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
400 data_refs)}
402 struct = self.isolatedStarAssociationTask.run(self.skymap,
403 self.tract,
404 data_ref_dict)
406 # These should ber zero length.
407 self.assertEqual(len(struct.star_source_cat), 0)
408 self.assertEqual(len(struct.star_cat), 0)
409 # And spot-check a couple of expected fields to make sure they have the right type.
410 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
411 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
413 def test_run_task_all_out_of_tract(self):
414 """Test running the task when all the sources are out of the tract."""
415 data_refs = self._make_simdata(self.tract, only_out_of_tract=True)
416 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
417 data_refs)}
419 struct = self.isolatedStarAssociationTask.run(self.skymap,
420 self.tract,
421 data_ref_dict)
423 # These should ber zero length.
424 self.assertEqual(len(struct.star_source_cat), 0)
425 self.assertEqual(len(struct.star_cat), 0)
426 # And spot-check a couple of expected fields to make sure they have the right type.
427 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
428 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
430 def test_run_task_all_out_of_inner_tract(self):
431 """Test running the task when all the sources are out of the inner tract."""
432 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True)
433 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
434 data_refs)}
436 struct = self.isolatedStarAssociationTask.run(self.skymap,
437 self.tract,
438 data_ref_dict)
440 # These should ber zero length.
441 self.assertEqual(len(struct.star_source_cat), 0)
442 self.assertEqual(len(struct.star_cat), 0)
443 # And spot-check a couple of expected fields to make sure they have the right type.
444 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
445 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
447 def test_run_task_secondary_no_overlap(self):
448 """Test running the task when the secondary band has no overlaps.
450 This tests DM-34834.
451 """
452 data_refs = self._make_simdata(self.tract, no_secondary_overlap=True)
453 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
454 data_refs)}
456 struct = self.isolatedStarAssociationTask.run(self.skymap,
457 self.tract,
458 data_ref_dict)
460 # Add a sanity check that we got a catalog out.
461 self.assertGreater(len(struct.star_source_cat), 0)
462 self.assertGreater(len(struct.star_cat), 0)
465class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
466 pass
469def setup_module(module):
470 lsst.utils.tests.init()
473if __name__ == "__main__": 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true
474 lsst.utils.tests.init()
475 unittest.main()