Coverage for tests/nopytest_ingestIndexReferenceCatalog.py : 97%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22# This file is excluded from running through pytest due to concerns about the
23# interaction between multiprocessing as invoked by this code, and the process
24# pool used by pytest.
25#
26# Note that it is invoked independently by SCons, so the tests are still run
27# as part of the build.
29import os.path
30import tempfile
31import unittest
32import unittest.mock
34import numpy as np
36import lsst.daf.persistence as dafPersist
37from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask,
38 LoadIndexedReferenceObjectsConfig)
39from lsst.meas.algorithms.htmIndexer import HtmIndexer
40from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
41from lsst.meas.algorithms.ingestIndexManager import IngestIndexManager
42from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask
43import lsst.utils
45import ingestIndexTestBase
48class TestIngestReferenceCatalogParallel(ingestIndexTestBase.IngestIndexCatalogTestBase,
49 lsst.utils.tests.TestCase):
50 """Test ingesting a refcat with multiprocessing turned on."""
51 def testIngestTwoFilesTwoCores(self):
52 def runTest(withRaDecErr):
53 # Generate a second catalog, with different ids
54 inPath1 = tempfile.mkdtemp()
55 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
56 inPath2 = tempfile.mkdtemp()
57 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
58 # override some field names, and use multiple cores
59 config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
60 withPm=True, withPmErr=True)
61 # use a very small HTM pixelization depth to ensure there will be collisions when
62 # ingesting the files in parallel
63 config.dataset_config.indexer.active.depth = 2
64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
65 config.file_reader.format = 'ascii.commented_header'
66 config.n_processes = 2 # use multiple cores for this test only
67 config.id_name = 'id' # Use the ids from the generated catalogs
68 outpath = os.path.join(self.outPath, "output_multifile_parallel",
69 "_withRaDecErr" if withRaDecErr else "")
70 IngestIndexedReferenceTask.parseAndRun(
71 args=[self.input_dir, "--output", outpath,
72 skyCatalogFile1, skyCatalogFile2], config=config)
74 # Test if we can get back the catalog with a non-standard dataset name
75 butler = dafPersist.Butler(outpath)
76 loaderConfig = LoadIndexedReferenceObjectsConfig()
77 loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
78 self.checkAllRowsInRefcat(loader, skyCatalog1, config)
79 self.checkAllRowsInRefcat(loader, skyCatalog2, config)
81 runTest(withRaDecErr=True)
82 runTest(withRaDecErr=False)
85class TestIngestIndexManager(ingestIndexTestBase.IngestIndexCatalogTestBase,
86 lsst.utils.tests.TestCase):
87 """Unittests of various methods of IngestIndexManager.
89 Uses mocks to force particular behavior regarding e.g. catalogs.
90 """
91 def setUp(self):
92 np.random.seed(10)
94 self.log = lsst.log.Log.getLogger("TestIngestIndexManager")
95 self.config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=True)
96 self.config.id_name = 'id'
97 depth = 2 # very small depth, for as few pixels as possible.
98 self.indexer = HtmIndexer(depth)
99 self.htm = lsst.sphgeom.HtmPixelization(depth)
100 ingester = IngestIndexedReferenceTask(self.config)
101 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
102 ('a', '<f8'), ('a_err', '<f8')]
103 self.schema, self.key_map = ingester.makeSchema(dtype)
104 self.fileReader = ReadTextCatalogTask()
106 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
107 self.matchedPixels = np.array([1, 1, 2, 2, 3])
108 self.path = tempfile.mkdtemp()
109 self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)}
111 self.worker = IngestIndexManager(self.filenames,
112 self.config,
113 self.fileReader,
114 self.indexer,
115 self.schema,
116 self.key_map,
117 self.htm.universe()[0],
118 addRefCatMetadata,
119 self.log)
121 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42):
122 """Create a fake output SimpleCatalog, populated with nOld+nNew elements.
124 Parameters
125 ----------
126 nOld : `int`, optional
127 The number of filled in sources to put in the catalog.
128 nNew : `int`, optional
129 The number of empty sources to put in the catalog.
130 idStart : `int`, optional
131 The start id of the ``nOld`` sources.
133 Returns
134 -------
135 catalog : `lsst.afw.table.SimpleCatalog`
136 A catalog populated with random data and contiguous ids.
137 """
138 catalog = lsst.afw.table.SimpleCatalog(self.schema)
139 catalog.resize(nOld)
140 for x in self.schema:
141 catalog[x.key] = np.random.random(nOld)
142 # do the ids separately, so there are no duplicates
143 catalog['id'] = np.arange(idStart, idStart + nOld)
144 catalog.resize(nOld + nNew) # make space for the elements we will add
145 return catalog.copy(deep=True)
147 def test_doOnePixelNewData(self):
148 """Test that we can add new data to an existing catalog."""
149 pixelId = 1 # the pixel we are going to test
151 nOld = 5
152 nNew = sum(self.matchedPixels == pixelId)
153 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
154 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
156 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
157 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
159 # check that the "pre" catalog is unchanged, exactly
160 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id'])
161 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra'])
162 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec'])
164 # check that the new catalog elements are set correctly
165 newElements = self.fakeInput[self.matchedPixels == pixelId]
166 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id'])
167 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180)
168 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180)
170 def test_doOnePixelNoData(self):
171 """Test that we can put new data into an empty catalog."""
172 pixelId = 2
174 nOld = 0
175 nNew = sum(self.matchedPixels == pixelId)
176 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
177 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
179 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
180 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
182 # check that the new catalog elements are set correctly
183 newElements = self.fakeInput[self.matchedPixels == pixelId]
184 np.testing.assert_equal(newcat['id'], newElements['id'])
185 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180)
186 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180)
188 def test_getCatalog(self):
189 """Test that getCatalog returns a properly expanded new catalog."""
190 pixelId = 3
191 nOld = 10
192 nNewElements = 5
193 # save a catalog to disk that we can check against the getCatalog()'s return
194 catalog = self._createFakeCatalog(nOld=nOld, nNew=0)
195 catalog.writeFits(self.filenames[pixelId])
196 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements)
198 self.assertEqual(len(newcat), nOld + nNewElements)
200 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id'])
201 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra'])
202 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec'])
205class TestMemory(lsst.utils.tests.MemoryTestCase):
206 pass
209def setup_module(module):
210 lsst.utils.tests.init()
213if __name__ == "__main__": 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true
214 lsst.utils.tests.init()
215 unittest.main()