Coverage for tests/nopytest_ingestIndexReferenceCatalog.py: 98%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22# This file is excluded from running through pytest due to concerns about the
23# interaction between multiprocessing as invoked by this code, and the process
24# pool used by pytest.
25#
26# Note that it is invoked independently by SCons, so the tests are still run
27# as part of the build.
29import os.path
30import tempfile
31import unittest
32import unittest.mock
34import numpy as np
36import lsst.daf.butler
37from lsst.daf.butler import DatasetType, DeferredDatasetHandle
38from lsst.daf.butler.script import ingest_files
39from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader)
40from lsst.meas.algorithms.htmIndexer import HtmIndexer
41from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata
42from lsst.meas.algorithms.convertRefcatManager import ConvertRefcatManager
43from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask
44import lsst.utils
46import ingestIndexTestBase
49class TestConvertReferenceCatalogParallel(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
50 lsst.utils.tests.TestCase):
51 """Test converting a refcat with multiprocessing turned on."""
52 def testIngestTwoFilesTwoCores(self):
53 def runTest(withRaDecErr):
54 # Generate a second catalog, with different ids
55 inPath1 = tempfile.mkdtemp()
56 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
57 inPath2 = tempfile.mkdtemp()
58 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
59 # override some field names, and use multiple cores
60 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True,
61 withPm=True, withPmErr=True)
62 # use a very small HTM pixelization depth to ensure there will be collisions when
63 # ingesting the files in parallel
64 depth = 2
65 config.dataset_config.indexer.active.depth = depth
66 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
67 config.file_reader.format = 'ascii.commented_header'
68 config.n_processes = 2 # use multiple cores for this test only
69 config.id_name = 'id' # Use the ids from the generated catalogs
70 repoPath = os.path.join(self.outPath, "output_multifile_parallel",
71 "_withRaDecErr" if withRaDecErr else "_noRaDecErr")
73 # Convert the input data files to our HTM indexed format.
74 dataPath = tempfile.mkdtemp()
75 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
76 converter.run([skyCatalogFile1, skyCatalogFile2])
78 # Make a temporary butler to ingest them into.
79 butler = self.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
80 dimensions = [f"htm{depth}"]
81 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
82 dimensions,
83 "SimpleCatalog",
84 universe=butler.registry.dimensions,
85 isCalibration=False)
86 butler.registry.registerDatasetType(datasetType)
88 # Ingest the files into the new butler.
89 run = "testingRun"
90 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
91 ingest_files(repoPath,
92 config.dataset_config.ref_dataset_name,
93 run,
94 htmTableFile,
95 transfer="auto")
97 # Test if we can get back the catalogs, with a new butler.
98 butler = lsst.daf.butler.Butler(repoPath)
99 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
100 collections=[run]).expanded())
101 handlers = []
102 for dataRef in datasetRefs:
103 handlers.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
104 loaderConfig = ReferenceObjectLoader.ConfigClass()
105 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in datasetRefs],
106 handlers,
107 loaderConfig,
108 log=self.logger)
109 self.checkAllRowsInRefcat(loader, skyCatalog1, config)
110 self.checkAllRowsInRefcat(loader, skyCatalog2, config)
112 runTest(withRaDecErr=True)
113 runTest(withRaDecErr=False)
116class TestConvertRefcatManager(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
117 lsst.utils.tests.TestCase):
118 """Unittests of various methods of ConvertRefcatManager.
120 Uses mocks to force particular behavior regarding e.g. catalogs.
121 """
122 def setUp(self):
123 np.random.seed(10)
125 tempPath = tempfile.mkdtemp()
126 self.log = lsst.log.Log.getLogger("lsst.TestIngestIndexManager")
127 self.config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True)
128 self.config.id_name = 'id'
129 self.depth = 2 # very small depth, for as few pixels as possible.
130 self.indexer = HtmIndexer(self.depth)
131 self.htm = lsst.sphgeom.HtmPixelization(self.depth)
132 ingester = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config)
133 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
134 ('a', '<f8'), ('a_err', '<f8')]
135 self.schema, self.key_map = ingester.makeSchema(dtype)
136 self.fileReader = ReadTextCatalogTask()
138 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
139 self.matchedPixels = np.array([1, 1, 2, 2, 3])
140 self.path = tempfile.mkdtemp()
141 self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)}
143 self.worker = ConvertRefcatManager(self.filenames,
144 self.config,
145 self.fileReader,
146 self.indexer,
147 self.schema,
148 self.key_map,
149 self.htm.universe()[0],
150 addRefCatMetadata,
151 self.log)
153 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42):
154 """Create a fake output SimpleCatalog, populated with nOld+nNew elements.
156 Parameters
157 ----------
158 nOld : `int`, optional
159 The number of filled in sources to put in the catalog.
160 nNew : `int`, optional
161 The number of empty sources to put in the catalog.
162 idStart : `int`, optional
163 The start id of the ``nOld`` sources.
165 Returns
166 -------
167 catalog : `lsst.afw.table.SimpleCatalog`
168 A catalog populated with random data and contiguous ids.
169 """
170 catalog = lsst.afw.table.SimpleCatalog(self.schema)
171 catalog.resize(nOld)
172 for x in self.schema:
173 catalog[x.key] = np.random.random(nOld)
174 # do the ids separately, so there are no duplicates
175 catalog['id'] = np.arange(idStart, idStart + nOld)
176 catalog.resize(nOld + nNew) # make space for the elements we will add
177 return catalog.copy(deep=True)
179 def test_doOnePixelNewData(self):
180 """Test that we can add new data to an existing catalog."""
181 pixelId = 1 # the pixel we are going to test
183 nOld = 5
184 nNew = sum(self.matchedPixels == pixelId)
185 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
186 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
188 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
189 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
191 # check that the "pre" catalog is unchanged, exactly
192 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id'])
193 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra'])
194 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec'])
196 # check that the new catalog elements are set correctly
197 newElements = self.fakeInput[self.matchedPixels == pixelId]
198 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id'])
199 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180)
200 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180)
202 def test_doOnePixelNoData(self):
203 """Test that we can put new data into an empty catalog."""
204 pixelId = 2
206 nOld = 0
207 nNew = sum(self.matchedPixels == pixelId)
208 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew)
209 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog)
211 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {})
212 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId])
214 # check that the new catalog elements are set correctly
215 newElements = self.fakeInput[self.matchedPixels == pixelId]
216 np.testing.assert_equal(newcat['id'], newElements['id'])
217 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180)
218 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180)
220 def test_getCatalog(self):
221 """Test that getCatalog returns a properly expanded new catalog."""
222 pixelId = 3
223 nOld = 10
224 nNewElements = 5
225 # save a catalog to disk that we can check against the getCatalog()'s return
226 catalog = self._createFakeCatalog(nOld=nOld, nNew=0)
227 catalog.writeFits(self.filenames[pixelId])
228 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements)
230 self.assertEqual(len(newcat), nOld + nNewElements)
232 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id'])
233 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra'])
234 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec'])
237class TestMemory(lsst.utils.tests.MemoryTestCase):
238 pass
241def setup_module(module):
242 lsst.utils.tests.init()
245if __name__ == "__main__": 245 ↛ 246line 245 didn't jump to line 246, because the condition on line 245 was never true
246 lsst.utils.tests.init()
247 unittest.main()