Coverage for tests/nopytest_ingestIndexReferenceCatalog.py: 98%
133 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-28 02:22 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-28 02:22 -0700
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22# This file is excluded from running through pytest due to concerns about the
23# interaction between multiprocessing as invoked by this code, and the process
24# pool used by pytest.
25#
26# Note that it is invoked independently by SCons, so the tests are still run
27# as part of the build.
29import os.path
30import tempfile
31import unittest
32import unittest.mock
34import numpy as np
36import lsst.daf.butler
37from lsst.daf.butler import DatasetType, DeferredDatasetHandle
38from lsst.daf.butler.script import ingest_files
39from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader)
40from lsst.meas.algorithms.htmIndexer import HtmIndexer
41from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
42from lsst.meas.algorithms.convertRefcatManager import ConvertRefcatManager
43from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask
44import lsst.utils
46import ingestIndexTestBase
49class TestConvertReferenceCatalogParallel(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
50 lsst.utils.tests.TestCase):
51 """Test converting a refcat with multiprocessing turned on."""
52 def testIngestTwoFilesTwoCores(self):
53 def runTest(withRaDecErr):
54 # Generate a second catalog, with different ids
55 inTempDir1 = tempfile.TemporaryDirectory()
56 inPath1 = inTempDir1.name
57 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
58 inTempDir2 = tempfile.TemporaryDirectory()
59 inPath2 = inTempDir2.name
60 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
61 # override some field names, and use multiple cores
62 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True,
63 withPm=True, withPmErr=True)
64 # use a very small HTM pixelization depth to ensure there will be collisions when
65 # ingesting the files in parallel
66 depth = 2
67 config.dataset_config.indexer.active.depth = depth
68 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
69 config.file_reader.format = 'ascii.commented_header'
70 config.n_processes = 2 # use multiple cores for this test only
71 config.id_name = 'id' # Use the ids from the generated catalogs
72 repoPath = os.path.join(self.outPath, "output_multifile_parallel",
73 "_withRaDecErr" if withRaDecErr else "_noRaDecErr")
75 # Convert the input data files to our HTM indexed format.
76 dataTempDir = tempfile.TemporaryDirectory()
77 dataPath = dataTempDir.name
78 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
79 converter.run([skyCatalogFile1, skyCatalogFile2])
81 # Make a temporary butler to ingest them into.
82 butler = self.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
83 dimensions = [f"htm{depth}"]
84 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
85 dimensions,
86 "SimpleCatalog",
87 universe=butler.registry.dimensions,
88 isCalibration=False)
89 butler.registry.registerDatasetType(datasetType)
91 # Ingest the files into the new butler.
92 run = "testingRun"
93 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
94 ingest_files(repoPath,
95 config.dataset_config.ref_dataset_name,
96 run,
97 htmTableFile,
98 transfer="auto")
100 # Test if we can get back the catalogs, with a new butler.
101 butler = lsst.daf.butler.Butler(repoPath)
102 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
103 collections=[run]).expanded())
104 handlers = []
105 for dataRef in datasetRefs:
106 handlers.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
107 loaderConfig = ReferenceObjectLoader.ConfigClass()
108 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in datasetRefs],
109 handlers,
110 config=loaderConfig,
111 log=self.logger)
112 self.checkAllRowsInRefcat(loader, skyCatalog1, config)
113 self.checkAllRowsInRefcat(loader, skyCatalog2, config)
115 inTempDir1.cleanup()
116 inTempDir2.cleanup()
117 dataTempDir.cleanup()
119 runTest(withRaDecErr=True)
120 runTest(withRaDecErr=False)
123class TestConvertRefcatManager(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
124 lsst.utils.tests.TestCase):
125 """Unittests of various methods of ConvertRefcatManager.
127 Uses mocks to force particular behavior regarding e.g. catalogs.
128 """
129 def setUp(self):
130 np.random.seed(10)
132 self.tempDir = tempfile.TemporaryDirectory()
133 tempPath = self.tempDir.name
134 self.log = lsst.log.Log.getLogger("lsst.TestIngestIndexManager")
135 self.config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True)
136 self.config.id_name = 'id'
137 self.depth = 2 # very small depth, for as few pixels as possible.
138 self.indexer = HtmIndexer(self.depth)
139 self.htm = lsst.sphgeom.HtmPixelization(self.depth)
140 ingester = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config)
141 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
142 ('a', '<f8'), ('a_err', '<f8')]
143 self.schema, self.key_map = ingester.makeSchema(dtype)
144 self.fileReader = ReadTextCatalogTask()
146 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
147 self.matchedPixels = np.array([1, 1, 2, 2, 3])
148 self.tempDir2 = tempfile.TemporaryDirectory()
149 tempPath = self.tempDir2.name
150 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)}
152 self.worker = ConvertRefcatManager(self.filenames,
153 self.config,
154 self.fileReader,
155 self.indexer,
156 self.schema,
157 self.key_map,
158 self.htm.universe()[0],
159 addRefCatMetadata,
160 self.log)
162 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42):
163 """Create a fake output SimpleCatalog, populated with nOld+nNew elements.
165 Parameters
166 ----------
167 nOld : `int`, optional
168 The number of filled in sources to put in the catalog.
169 nNew : `int`, optional
170 The number of empty sources to put in the catalog.
171 idStart : `int`, optional
172 The start id of the ``nOld`` sources.
174 Returns
175 -------
176 catalog : `lsst.afw.table.SimpleCatalog`
177 A catalog populated with random data and contiguous ids.
178 """
179 catalog = lsst.afw.table.SimpleCatalog(self.schema)
180 catalog.resize(nOld)
181 for x in self.schema:
182 catalog[x.key] = np.random.random(nOld)
183 # do the ids separately, so there are no duplicates
184 catalog['id'] = np.arange(idStart, idStart + nOld)
185 catalog.resize(nOld + nNew) # make space for the elements we will add
186 return catalog.copy(deep=True)
188 def test_doOnePixelNewData(self):
189 """Test that we can add new data to an existing catalog."""
190 pixelId = 1 # the pixel we are going to test
192 nOld = 5
193 nNew = sum(self.matchedPixels == pixelId)
194 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
195 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
197 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
198 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
200 # check that the "pre" catalog is unchanged, exactly
201 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id'])
202 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra'])
203 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec'])
205 # check that the new catalog elements are set correctly
206 newElements = self.fakeInput[self.matchedPixels == pixelId]
207 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id'])
208 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180)
209 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180)
211 def test_doOnePixelNoData(self):
212 """Test that we can put new data into an empty catalog."""
213 pixelId = 2
215 nOld = 0
216 nNew = sum(self.matchedPixels == pixelId)
217 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
218 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
220 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
221 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
223 # check that the new catalog elements are set correctly
224 newElements = self.fakeInput[self.matchedPixels == pixelId]
225 np.testing.assert_equal(newcat['id'], newElements['id'])
226 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180)
227 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180)
229 def test_getCatalog(self):
230 """Test that getCatalog returns a properly expanded new catalog."""
231 pixelId = 3
232 nOld = 10
233 nNewElements = 5
234 # save a catalog to disk that we can check against the getCatalog()'s return
235 catalog = self._createFakeCatalog(nOld=nOld, nNew=0)
236 catalog.writeFits(self.filenames[pixelId])
237 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements)
239 self.assertEqual(len(newcat), nOld + nNewElements)
241 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id'])
242 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra'])
243 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec'])
245 def tearDown(self):
246 self.tempDir.cleanup()
247 self.tempDir2.cleanup()
250class TestMemory(lsst.utils.tests.MemoryTestCase):
251 pass
254def setup_module(module):
255 lsst.utils.tests.init()
258if __name__ == "__main__": 258 ↛ 259line 258 didn't jump to line 259, because the condition on line 258 was never true
259 lsst.utils.tests.init()
260 unittest.main()