Coverage for tests/nopytest_ingestIndexReferenceCatalog.py: 98%

133 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-15 03:08 -0700

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22# This file is excluded from running through pytest due to concerns about the 

23# interaction between multiprocessing as invoked by this code, and the process 

24# pool used by pytest. 

25# 

26# Note that it is invoked independently by SCons, so the tests are still run 

27# as part of the build. 

28 

29import os.path 

30import tempfile 

31import unittest 

32import unittest.mock 

33 

34import numpy as np 

35 

36import lsst.daf.butler 

37from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

38from lsst.daf.butler.script import ingest_files 

39from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader) 

40from lsst.meas.algorithms.htmIndexer import HtmIndexer 

41from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

42from lsst.meas.algorithms.convertRefcatManager import ConvertRefcatManager 

43from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask 

44import lsst.utils 

45 

46import ingestIndexTestBase 

47 

48 

49class TestConvertReferenceCatalogParallel(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

50 lsst.utils.tests.TestCase): 

51 """Test converting a refcat with multiprocessing turned on.""" 

52 def testIngestTwoFilesTwoCores(self): 

53 def runTest(withRaDecErr): 

54 # Generate a second catalog, with different ids 

55 inTempDir1 = tempfile.TemporaryDirectory() 

56 inPath1 = inTempDir1.name 

57 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123) 

58 inTempDir2 = tempfile.TemporaryDirectory() 

59 inPath2 = inTempDir2.name 

60 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11) 

61 # override some field names, and use multiple cores 

62 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

63 withPm=True, withPmErr=True) 

64 # use a very small HTM pixelization depth to ensure there will be collisions when 

65 # ingesting the files in parallel 

66 depth = 2 

67 config.dataset_config.indexer.active.depth = depth 

68 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

69 config.file_reader.format = 'ascii.commented_header' 

70 config.n_processes = 2 # use multiple cores for this test only 

71 config.id_name = 'id' # Use the ids from the generated catalogs 

72 repoPath = os.path.join(self.outPath, "output_multifile_parallel", 

73 "_withRaDecErr" if withRaDecErr else "_noRaDecErr") 

74 

75 # Convert the input data files to our HTM indexed format. 

76 dataTempDir = tempfile.TemporaryDirectory() 

77 dataPath = dataTempDir.name 

78 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

79 converter.run([skyCatalogFile1, skyCatalogFile2]) 

80 

81 # Make a temporary butler to ingest them into. 

82 butler = self.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

83 dimensions = [f"htm{depth}"] 

84 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

85 dimensions, 

86 "SimpleCatalog", 

87 universe=butler.registry.dimensions, 

88 isCalibration=False) 

89 butler.registry.registerDatasetType(datasetType) 

90 

91 # Ingest the files into the new butler. 

92 run = "testingRun" 

93 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

94 ingest_files(repoPath, 

95 config.dataset_config.ref_dataset_name, 

96 run, 

97 htmTableFile, 

98 transfer="auto") 

99 

100 # Test if we can get back the catalogs, with a new butler. 

101 butler = lsst.daf.butler.Butler(repoPath) 

102 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

103 collections=[run]).expanded()) 

104 handlers = [] 

105 for dataRef in datasetRefs: 

106 handlers.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

107 loaderConfig = ReferenceObjectLoader.ConfigClass() 

108 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in datasetRefs], 

109 handlers, 

110 name="testrefcat", 

111 config=loaderConfig, 

112 log=self.logger) 

113 self.checkAllRowsInRefcat(loader, skyCatalog1, config) 

114 self.checkAllRowsInRefcat(loader, skyCatalog2, config) 

115 

116 inTempDir1.cleanup() 

117 inTempDir2.cleanup() 

118 dataTempDir.cleanup() 

119 

120 runTest(withRaDecErr=True) 

121 runTest(withRaDecErr=False) 

122 

123 

124class TestConvertRefcatManager(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

125 lsst.utils.tests.TestCase): 

126 """Unittests of various methods of ConvertRefcatManager. 

127 

128 Uses mocks to force particular behavior regarding e.g. catalogs. 

129 """ 

130 def setUp(self): 

131 np.random.seed(10) 

132 

133 self.tempDir = tempfile.TemporaryDirectory() 

134 tempPath = self.tempDir.name 

135 self.log = lsst.log.Log.getLogger("lsst.TestIngestIndexManager") 

136 self.config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True) 

137 self.config.id_name = 'id' 

138 self.depth = 2 # very small depth, for as few pixels as possible. 

139 self.indexer = HtmIndexer(self.depth) 

140 self.htm = lsst.sphgeom.HtmPixelization(self.depth) 

141 ingester = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config) 

142 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'), 

143 ('a', '<f8'), ('a_err', '<f8')] 

144 self.schema, self.key_map = ingester.makeSchema(dtype) 

145 self.fileReader = ReadTextCatalogTask() 

146 

147 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543) 

148 self.matchedPixels = np.array([1, 1, 2, 2, 3]) 

149 self.tempDir2 = tempfile.TemporaryDirectory() 

150 tempPath = self.tempDir2.name 

151 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)} 

152 

153 self.worker = ConvertRefcatManager(self.filenames, 

154 self.config, 

155 self.fileReader, 

156 self.indexer, 

157 self.schema, 

158 self.key_map, 

159 self.htm.universe()[0], 

160 addRefCatMetadata, 

161 self.log) 

162 

163 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42): 

164 """Create a fake output SimpleCatalog, populated with nOld+nNew elements. 

165 

166 Parameters 

167 ---------- 

168 nOld : `int`, optional 

169 The number of filled in sources to put in the catalog. 

170 nNew : `int`, optional 

171 The number of empty sources to put in the catalog. 

172 idStart : `int`, optional 

173 The start id of the ``nOld`` sources. 

174 

175 Returns 

176 ------- 

177 catalog : `lsst.afw.table.SimpleCatalog` 

178 A catalog populated with random data and contiguous ids. 

179 """ 

180 catalog = lsst.afw.table.SimpleCatalog(self.schema) 

181 catalog.resize(nOld) 

182 for x in self.schema: 

183 catalog[x.key] = np.random.random(nOld) 

184 # do the ids separately, so there are no duplicates 

185 catalog['id'] = np.arange(idStart, idStart + nOld) 

186 catalog.resize(nOld + nNew) # make space for the elements we will add 

187 return catalog.copy(deep=True) 

188 

189 def test_doOnePixelNewData(self): 

190 """Test that we can add new data to an existing catalog.""" 

191 pixelId = 1 # the pixel we are going to test 

192 

193 nOld = 5 

194 nNew = sum(self.matchedPixels == pixelId) 

195 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

196 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

197 

198 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

199 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

200 

201 # check that the "pre" catalog is unchanged, exactly 

202 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id']) 

203 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra']) 

204 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec']) 

205 

206 # check that the new catalog elements are set correctly 

207 newElements = self.fakeInput[self.matchedPixels == pixelId] 

208 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id']) 

209 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180) 

210 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180) 

211 

212 def test_doOnePixelNoData(self): 

213 """Test that we can put new data into an empty catalog.""" 

214 pixelId = 2 

215 

216 nOld = 0 

217 nNew = sum(self.matchedPixels == pixelId) 

218 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

219 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

220 

221 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

222 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

223 

224 # check that the new catalog elements are set correctly 

225 newElements = self.fakeInput[self.matchedPixels == pixelId] 

226 np.testing.assert_equal(newcat['id'], newElements['id']) 

227 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180) 

228 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180) 

229 

230 def test_getCatalog(self): 

231 """Test that getCatalog returns a properly expanded new catalog.""" 

232 pixelId = 3 

233 nOld = 10 

234 nNewElements = 5 

235 # save a catalog to disk that we can check against the getCatalog()'s return 

236 catalog = self._createFakeCatalog(nOld=nOld, nNew=0) 

237 catalog.writeFits(self.filenames[pixelId]) 

238 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements) 

239 

240 self.assertEqual(len(newcat), nOld + nNewElements) 

241 

242 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id']) 

243 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra']) 

244 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec']) 

245 

246 def tearDown(self): 

247 self.tempDir.cleanup() 

248 self.tempDir2.cleanup() 

249 

250 

251class TestMemory(lsst.utils.tests.MemoryTestCase): 

252 pass 

253 

254 

255def setup_module(module): 

256 lsst.utils.tests.init() 

257 

258 

259if __name__ == "__main__": 259 ↛ 260line 259 didn't jump to line 260, because the condition on line 259 was never true

260 lsst.utils.tests.init() 

261 unittest.main()