Coverage for tests/nopytest_ingestIndexReferenceCatalog.py: 98%

133 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-06 13:12 -0800

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22# This file is excluded from running through pytest due to concerns about the 

23# interaction between multiprocessing as invoked by this code, and the process 

24# pool used by pytest. 

25# 

26# Note that it is invoked independently by SCons, so the tests are still run 

27# as part of the build. 

28 

29import os.path 

30import tempfile 

31import unittest 

32import unittest.mock 

33 

34import numpy as np 

35 

36import lsst.daf.butler 

37from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

38from lsst.daf.butler.script import ingest_files 

39from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader) 

40from lsst.meas.algorithms.htmIndexer import HtmIndexer 

41from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

42from lsst.meas.algorithms.convertRefcatManager import ConvertRefcatManager 

43from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask 

44import lsst.utils 

45 

46import ingestIndexTestBase 

47 

48 

49class TestConvertReferenceCatalogParallel(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

50 lsst.utils.tests.TestCase): 

51 """Test converting a refcat with multiprocessing turned on.""" 

52 def testIngestTwoFilesTwoCores(self): 

53 def runTest(withRaDecErr): 

54 # Generate a second catalog, with different ids 

55 inTempDir1 = tempfile.TemporaryDirectory() 

56 inPath1 = inTempDir1.name 

57 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123) 

58 inTempDir2 = tempfile.TemporaryDirectory() 

59 inPath2 = inTempDir2.name 

60 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11) 

61 # override some field names, and use multiple cores 

62 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

63 withPm=True, withPmErr=True) 

64 # use a very small HTM pixelization depth to ensure there will be collisions when 

65 # ingesting the files in parallel 

66 depth = 2 

67 config.dataset_config.indexer.active.depth = depth 

68 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

69 config.file_reader.format = 'ascii.commented_header' 

70 config.n_processes = 2 # use multiple cores for this test only 

71 config.id_name = 'id' # Use the ids from the generated catalogs 

72 repoPath = os.path.join(self.outPath, "output_multifile_parallel", 

73 "_withRaDecErr" if withRaDecErr else "_noRaDecErr") 

74 

75 # Convert the input data files to our HTM indexed format. 

76 dataTempDir = tempfile.TemporaryDirectory() 

77 dataPath = dataTempDir.name 

78 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

79 converter.run([skyCatalogFile1, skyCatalogFile2]) 

80 

81 # Make a temporary butler to ingest them into. 

82 butler = self.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

83 dimensions = [f"htm{depth}"] 

84 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

85 dimensions, 

86 "SimpleCatalog", 

87 universe=butler.registry.dimensions, 

88 isCalibration=False) 

89 butler.registry.registerDatasetType(datasetType) 

90 

91 # Ingest the files into the new butler. 

92 run = "testingRun" 

93 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

94 ingest_files(repoPath, 

95 config.dataset_config.ref_dataset_name, 

96 run, 

97 htmTableFile, 

98 transfer="auto") 

99 

100 # Test if we can get back the catalogs, with a new butler. 

101 butler = lsst.daf.butler.Butler(repoPath) 

102 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

103 collections=[run]).expanded()) 

104 handlers = [] 

105 for dataRef in datasetRefs: 

106 handlers.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

107 loaderConfig = ReferenceObjectLoader.ConfigClass() 

108 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in datasetRefs], 

109 handlers, 

110 config=loaderConfig, 

111 log=self.logger) 

112 self.checkAllRowsInRefcat(loader, skyCatalog1, config) 

113 self.checkAllRowsInRefcat(loader, skyCatalog2, config) 

114 

115 inTempDir1.cleanup() 

116 inTempDir2.cleanup() 

117 dataTempDir.cleanup() 

118 

119 runTest(withRaDecErr=True) 

120 runTest(withRaDecErr=False) 

121 

122 

123class TestConvertRefcatManager(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

124 lsst.utils.tests.TestCase): 

125 """Unittests of various methods of ConvertRefcatManager. 

126 

127 Uses mocks to force particular behavior regarding e.g. catalogs. 

128 """ 

129 def setUp(self): 

130 np.random.seed(10) 

131 

132 self.tempDir = tempfile.TemporaryDirectory() 

133 tempPath = self.tempDir.name 

134 self.log = lsst.log.Log.getLogger("lsst.TestIngestIndexManager") 

135 self.config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True) 

136 self.config.id_name = 'id' 

137 self.depth = 2 # very small depth, for as few pixels as possible. 

138 self.indexer = HtmIndexer(self.depth) 

139 self.htm = lsst.sphgeom.HtmPixelization(self.depth) 

140 ingester = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config) 

141 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'), 

142 ('a', '<f8'), ('a_err', '<f8')] 

143 self.schema, self.key_map = ingester.makeSchema(dtype) 

144 self.fileReader = ReadTextCatalogTask() 

145 

146 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543) 

147 self.matchedPixels = np.array([1, 1, 2, 2, 3]) 

148 self.tempDir2 = tempfile.TemporaryDirectory() 

149 tempPath = self.tempDir2.name 

150 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)} 

151 

152 self.worker = ConvertRefcatManager(self.filenames, 

153 self.config, 

154 self.fileReader, 

155 self.indexer, 

156 self.schema, 

157 self.key_map, 

158 self.htm.universe()[0], 

159 addRefCatMetadata, 

160 self.log) 

161 

162 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42): 

163 """Create a fake output SimpleCatalog, populated with nOld+nNew elements. 

164 

165 Parameters 

166 ---------- 

167 nOld : `int`, optional 

168 The number of filled in sources to put in the catalog. 

169 nNew : `int`, optional 

170 The number of empty sources to put in the catalog. 

171 idStart : `int`, optional 

172 The start id of the ``nOld`` sources. 

173 

174 Returns 

175 ------- 

176 catalog : `lsst.afw.table.SimpleCatalog` 

177 A catalog populated with random data and contiguous ids. 

178 """ 

179 catalog = lsst.afw.table.SimpleCatalog(self.schema) 

180 catalog.resize(nOld) 

181 for x in self.schema: 

182 catalog[x.key] = np.random.random(nOld) 

183 # do the ids separately, so there are no duplicates 

184 catalog['id'] = np.arange(idStart, idStart + nOld) 

185 catalog.resize(nOld + nNew) # make space for the elements we will add 

186 return catalog.copy(deep=True) 

187 

188 def test_doOnePixelNewData(self): 

189 """Test that we can add new data to an existing catalog.""" 

190 pixelId = 1 # the pixel we are going to test 

191 

192 nOld = 5 

193 nNew = sum(self.matchedPixels == pixelId) 

194 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

195 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

196 

197 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

198 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

199 

200 # check that the "pre" catalog is unchanged, exactly 

201 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id']) 

202 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra']) 

203 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec']) 

204 

205 # check that the new catalog elements are set correctly 

206 newElements = self.fakeInput[self.matchedPixels == pixelId] 

207 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id']) 

208 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180) 

209 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180) 

210 

211 def test_doOnePixelNoData(self): 

212 """Test that we can put new data into an empty catalog.""" 

213 pixelId = 2 

214 

215 nOld = 0 

216 nNew = sum(self.matchedPixels == pixelId) 

217 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

218 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

219 

220 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

221 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

222 

223 # check that the new catalog elements are set correctly 

224 newElements = self.fakeInput[self.matchedPixels == pixelId] 

225 np.testing.assert_equal(newcat['id'], newElements['id']) 

226 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180) 

227 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180) 

228 

229 def test_getCatalog(self): 

230 """Test that getCatalog returns a properly expanded new catalog.""" 

231 pixelId = 3 

232 nOld = 10 

233 nNewElements = 5 

234 # save a catalog to disk that we can check against the getCatalog()'s return 

235 catalog = self._createFakeCatalog(nOld=nOld, nNew=0) 

236 catalog.writeFits(self.filenames[pixelId]) 

237 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements) 

238 

239 self.assertEqual(len(newcat), nOld + nNewElements) 

240 

241 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id']) 

242 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra']) 

243 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec']) 

244 

245 def tearDown(self): 

246 self.tempDir.cleanup() 

247 self.tempDir2.cleanup() 

248 

249 

250class TestMemory(lsst.utils.tests.MemoryTestCase): 

251 pass 

252 

253 

254def setup_module(module): 

255 lsst.utils.tests.init() 

256 

257 

258if __name__ == "__main__": 258 ↛ 259line 258 didn't jump to line 259, because the condition on line 258 was never true

259 lsst.utils.tests.init() 

260 unittest.main()