Coverage for tests/test_isolatedStarAssociation.py: 14%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""Test IsolatedStarAssociationTask.
23"""
24import unittest
25import numpy as np
26import pandas as pd
28import lsst.utils.tests
29import lsst.daf.butler
30import lsst.skymap
32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig,
33 IsolatedStarAssociationTask)
34from smatch.matcher import Matcher
37class MockSourceTableReference(lsst.daf.butler.DeferredDatasetHandle):
38 """Very simple object that looks like a Gen3 data reference to
39 a sourceTable_visit.
41 Parameters
42 ----------
43 source_table : `pandas.DataFrame`
44 Dataframe of the source table.
45 """
46 def __init__(self, source_table):
47 self.source_table = source_table
49 def get(self, parameters={}, **kwargs):
50 """Retrieve the specified dataset using the API of the Gen3 Butler.
52 Parameters
53 ----------
54 parameters : `dict`, optional
55 Parameter dictionary. Supported key is ``columns``.
57 Returns
58 -------
59 dataframe : `pandas.DataFrame`
60 dataframe, cut to the specified columns.
61 """
62 if 'columns' in parameters:
63 _columns = parameters['columns']
64 if 'sourceId' in parameters['columns']:
65 # Treat the index separately
66 _columns.remove('sourceId')
68 return self.source_table[_columns]
69 else:
70 return self.source_table.copy()
73class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase):
74 """Tests of IsolatedStarAssociationTask.
76 These tests bypass the middleware used for accessing data and
77 managing Task execution.
78 """
79 def setUp(self):
80 self.skymap = self._make_skymap()
81 self.tract = 9813
82 self.data_refs = self._make_simdata(self.tract)
83 self.visits = np.arange(len(self.data_refs)) + 1
85 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
86 self.data_refs)}
88 config = IsolatedStarAssociationConfig()
89 config.band_order = ['i', 'r']
90 config.extra_columns = ['extra_column']
91 config.source_selector['science'].doFlags = False
92 config.source_selector['science'].doIsolated = False
94 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config)
96 def _make_skymap(self):
97 """Make a testing skymap."""
98 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig()
99 skymap_config.numRings = 120
100 skymap_config.projection = "TAN"
101 skymap_config.tractOverlap = 1.0/60
102 skymap_config.pixelScale = 0.168
103 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config)
105 def _make_simdata(self,
106 tract,
107 only_neighbors=False,
108 only_out_of_tract=False,
109 only_out_of_inner_tract=False):
110 """Make simulated data tables and references.
112 Parameters
113 ----------
114 only_neighbors : `bool`, optional
115 Only put in neighbors.
116 only_out_of_tract : `bool`, optional
117 All stars are out of the tract.
118 only_out_of_inner_tract : `bool`, optional
119 All stars are out of the inner tract.
121 Returns
122 -------
123 data_refs : `list` [`MockSourceTableReference`]
124 List of mock references.
125 """
126 np.random.seed(12345)
128 n_visit_per_band = 5
129 n_star_both = 50
130 n_star_just_one = 5
132 tract_info = self.skymap[tract]
133 ctr = tract_info.ctr_coord
134 ctr_ra = ctr.getRa().asDegrees()
135 ctr_dec = ctr.getDec().asDegrees()
137 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both)
138 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both)
140 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
141 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one)
142 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one)
143 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one)
145 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.])
146 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.])
148 # Create the r-band datarefs
149 dtype = [('sourceId', 'i8'),
150 ('ra', 'f8'),
151 ('decl', 'f8'),
152 ('apFlux_12_0_instFlux', 'f4'),
153 ('apFlux_12_0_instFluxErr', 'f4'),
154 ('apFlux_12_0_instFlux_flag', '?'),
155 ('extendedness', 'f4'),
156 ('visit', 'i4'),
157 ('detector', 'i4'),
158 ('physical_filter', 'U10'),
159 ('band', 'U2'),
160 ('extra_column', 'f4')]
162 id_counter = 0
163 visit_counter = 1
165 data_refs = []
166 for band in ['r', 'i']:
167 if band == 'r':
168 filtername = 'R FILTER'
169 ra_just = ra_just_r
170 dec_just = dec_just_r
171 else:
172 filtername = 'I FILTER'
173 ra_just = ra_just_i
174 dec_just = dec_just_i
176 if only_neighbors:
177 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
178 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
179 else:
180 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just))
181 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just))
183 if only_out_of_tract:
184 poly = self.skymap[self.tract].outer_sky_polygon
185 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec))
186 star_ra = star_ra[use]
187 star_dec = star_dec[use]
188 elif only_out_of_inner_tract:
189 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True)
190 use = (inner_tract_ids != self.tract)
191 star_ra = star_ra[use]
192 star_dec = star_dec[use]
194 nstar = len(star_ra)
196 for i in range(n_visit_per_band):
197 ras = np.random.normal(loc=star_ra, scale=0.2/3600.)
198 decs = np.random.normal(loc=star_dec, scale=0.2/3600.)
200 table = np.zeros(nstar, dtype=dtype)
201 table['sourceId'] = np.arange(nstar) + id_counter
202 table['ra'] = ras
203 table['decl'] = decs
204 table['apFlux_12_0_instFlux'] = 100.0
205 table['apFlux_12_0_instFluxErr'] = 1.0
206 table['physical_filter'] = filtername
207 table['band'] = band
208 table['extra_column'] = np.ones(nstar)
209 table['visit'] = visit_counter
210 table['detector'] = 1
212 if i == 0:
213 # Make one star have low s/n
214 table['apFlux_12_0_instFlux'][0] = 1.0
216 df = pd.DataFrame(table)
217 df.set_index('sourceId', inplace=True)
218 data_refs.append(MockSourceTableReference(df))
220 id_counter += nstar
221 visit_counter += 1
223 self.n_visit_per_band = n_visit_per_band
224 self.n_star_both = n_star_both
225 self.n_star_just_one = n_star_just_one
226 if only_neighbors:
227 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor))
228 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor))
229 else:
230 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor))
231 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor))
233 return data_refs
235 def test_compute_unique_ids(self):
236 """Test computation of unique ids."""
237 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
238 9813,
239 10000)
240 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap,
241 9814,
242 5000)
243 ids = np.concatenate((ids1, ids2))
244 self.assertEqual(len(np.unique(ids)), len(ids))
246 def test_remove_neighbors(self):
247 """Test removing close neighbors."""
248 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'),
249 ('decl', 'f8')])
251 # Put two stars < 2" apart, and across the 0/360 boundary.
252 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0]
253 primary_star_cat['decl'] = [5.0, 5.0, 5.0]
255 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat)
257 self.assertEqual(len(cut_cat), 1)
258 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0])
260 def test_match_primary_stars(self):
261 """Test matching primary stars."""
262 # Stack all the sources; we do not want any cutting here.
263 tables = []
264 for data_ref in self.data_refs:
265 df = data_ref.get()
266 tables.append(df.to_records())
267 source_cat = np.concatenate(tables)
269 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'],
270 source_cat)
272 # Ensure we found the right number of stars in each p
273 test_i = (primary_star_cat['primary_band'] == 'i')
274 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1)
275 test_r = (primary_star_cat['primary_band'] == 'r')
276 self.assertEqual(test_r.sum(), self.n_star_just_one)
278 # Ensure that these stars all match to input stars within 1 arcsec.
279 with Matcher(self.star_ras, self.star_decs) as matcher:
280 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'],
281 primary_star_cat['decl'],
282 1./3600.,
283 return_indices=True)
284 self.assertEqual(i1.size, self.star_ras.size)
286 def test_get_source_table_visit_columns(self):
287 """Test making of source table visit columns."""
288 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names()
290 # Make sure all persisted columns are in all columns.
291 for col in persist_columns:
292 self.assertTrue(col in all_columns)
294 # And make sure extendedness is not in persisted columns.
295 self.assertTrue('extendedness' not in persist_columns)
297 def test_match_sources(self):
298 """Test _match_sources source to primary matching."""
299 # Stack all the sources; we do not want any cutting here.
300 tables = []
301 for data_ref in self.data_refs:
302 df = data_ref.get()
303 tables.append(df.to_records())
304 source_cat = np.concatenate(tables)
306 source_cat = np.lib.recfunctions.append_fields(source_cat,
307 ['obj_index'],
308 [np.zeros(source_cat.size, dtype=np.int32)],
309 dtypes=['i4'],
310 usemask=False)
312 primary_bands = ['i', 'r']
314 primary_cat = np.zeros(self.star_ras.size,
315 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands))
316 primary_cat['ra'] = self.star_ras
317 primary_cat['decl'] = self.star_decs
319 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'],
320 source_cat,
321 primary_cat)
322 # All the star sources should be matched
323 self.assertEqual(source_cat_sorted.size, source_cat.size)
325 # Full index tests are performed in test_run_isolated_star_association_task
327 def test_make_all_star_sources(self):
328 """Test appending all the star sources."""
329 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract],
330 self.data_ref_dict)
332 # Make sure we don't have any low s/n sources.
333 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr'])
334 self.assertGreater(sn_min, 10.0)
336 # And make sure they are all within the tract outer boundary.
337 poly = self.skymap[self.tract].outer_sky_polygon
338 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl']))
339 self.assertEqual(use.sum(), len(source_cat))
341 def test_run_isolated_star_association_task(self):
342 """Test running the full task."""
343 struct = self.isolatedStarAssociationTask.run(self.skymap,
344 self.tract,
345 self.data_ref_dict)
347 star_source_cat = struct.star_source_cat
348 star_cat = struct.star_cat
350 # Check that sources are all unique ids
351 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size)
353 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras,
354 self.star_decs,
355 degrees=True)
356 inner_stars = (inner_tract_ids == self.tract)
358 # There should be the same number of stars in the inner tract region,
359 # taking away the 2 close neighbors.
360 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2)
362 # Check the star indices
363 for i in range(len(star_cat)):
364 all_source_star = star_source_cat[star_cat['source_cat_index'][i]:
365 star_cat['source_cat_index'][i] + star_cat['nsource'][i]]
367 # Check that these all point to the correct object
368 np.testing.assert_array_equal(all_source_star['obj_index'], i)
370 # Check these are all pointing to the same star position
371 with Matcher(np.atleast_1d(star_cat['ra'][i]),
372 np.atleast_1d(star_cat['decl'][i])) as matcher:
373 idx = matcher.query_radius(all_source_star['ra'],
374 all_source_star['decl'],
375 1./3600.)
376 self.assertEqual(len(idx[0]), star_cat['nsource'][i])
378 # Check per band indices
379 for band in ['r', 'i']:
380 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]:
381 star_cat[f'source_cat_index_{band}'][i]
382 + star_cat[f'nsource_{band}'][i]]
383 with Matcher(np.atleast_1d(star_cat['ra'][i]),
384 np.atleast_1d(star_cat['decl'][i])) as matcher:
385 idx = matcher.query_radius(band_source_star['ra'],
386 band_source_star['decl'],
387 1./3600.)
388 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i])
390 def test_run_task_all_neighbors(self):
391 """Test running the task when all the stars are rejected as neighbors."""
392 data_refs = self._make_simdata(self.tract, only_neighbors=True)
393 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
394 data_refs)}
396 struct = self.isolatedStarAssociationTask.run(self.skymap,
397 self.tract,
398 data_ref_dict)
400 # These should ber zero length.
401 self.assertEqual(len(struct.star_source_cat), 0)
402 self.assertEqual(len(struct.star_cat), 0)
403 # And spot-check a couple of expected fields to make sure they have the right type.
404 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
405 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
407 def test_run_task_all_out_of_tract(self):
408 """Test running the task when all the sources are out of the tract."""
409 data_refs = self._make_simdata(self.tract, only_out_of_tract=True)
410 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
411 data_refs)}
413 struct = self.isolatedStarAssociationTask.run(self.skymap,
414 self.tract,
415 data_ref_dict)
417 # These should ber zero length.
418 self.assertEqual(len(struct.star_source_cat), 0)
419 self.assertEqual(len(struct.star_cat), 0)
420 # And spot-check a couple of expected fields to make sure they have the right type.
421 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
422 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
424 def test_run_task_all_out_of_inner_tract(self):
425 """Test running the task when all the sources are out of the inner tract."""
426 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True)
427 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits,
428 data_refs)}
430 struct = self.isolatedStarAssociationTask.run(self.skymap,
431 self.tract,
432 data_ref_dict)
434 # These should ber zero length.
435 self.assertEqual(len(struct.star_source_cat), 0)
436 self.assertEqual(len(struct.star_cat), 0)
437 # And spot-check a couple of expected fields to make sure they have the right type.
438 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names)
439 self.assertTrue('nsource_i' in struct.star_cat.dtype.names)
442class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
443 pass
446def setup_module(module):
447 lsst.utils.tests.init()
450if __name__ == "__main__": 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true
451 lsst.utils.tests.init()
452 unittest.main()