Coverage for tests/nopytest_ingestIndexReferenceCatalog.py: 98%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

122 statements  

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22# This file is excluded from running through pytest due to concerns about the 

23# interaction between multiprocessing as invoked by this code, and the process 

24# pool used by pytest. 

25# 

26# Note that it is invoked independently by SCons, so the tests are still run 

27# as part of the build. 

28 

29import os.path 

30import tempfile 

31import unittest 

32import unittest.mock 

33 

34import numpy as np 

35 

36import lsst.daf.butler 

37from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

38from lsst.daf.butler.script import ingest_files 

39from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader) 

40from lsst.meas.algorithms.htmIndexer import HtmIndexer 

41from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

42from lsst.meas.algorithms.convertRefcatManager import ConvertRefcatManager 

43from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask 

44import lsst.utils 

45 

46import ingestIndexTestBase 

47 

48 

49class TestConvertReferenceCatalogParallel(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

50 lsst.utils.tests.TestCase): 

51 """Test converting a refcat with multiprocessing turned on.""" 

52 def testIngestTwoFilesTwoCores(self): 

53 def runTest(withRaDecErr): 

54 # Generate a second catalog, with different ids 

55 inPath1 = tempfile.mkdtemp() 

56 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123) 

57 inPath2 = tempfile.mkdtemp() 

58 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11) 

59 # override some field names, and use multiple cores 

60 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

61 withPm=True, withPmErr=True) 

62 # use a very small HTM pixelization depth to ensure there will be collisions when 

63 # ingesting the files in parallel 

64 depth = 2 

65 config.dataset_config.indexer.active.depth = depth 

66 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

67 config.file_reader.format = 'ascii.commented_header' 

68 config.n_processes = 2 # use multiple cores for this test only 

69 config.id_name = 'id' # Use the ids from the generated catalogs 

70 repoPath = os.path.join(self.outPath, "output_multifile_parallel", 

71 "_withRaDecErr" if withRaDecErr else "_noRaDecErr") 

72 

73 # Convert the input data files to our HTM indexed format. 

74 dataPath = tempfile.mkdtemp() 

75 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

76 converter.run([skyCatalogFile1, skyCatalogFile2]) 

77 

78 # Make a temporary butler to ingest them into. 

79 butler = self.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

80 dimensions = [f"htm{depth}"] 

81 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

82 dimensions, 

83 "SimpleCatalog", 

84 universe=butler.registry.dimensions, 

85 isCalibration=False) 

86 butler.registry.registerDatasetType(datasetType) 

87 

88 # Ingest the files into the new butler. 

89 run = "testingRun" 

90 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

91 ingest_files(repoPath, 

92 config.dataset_config.ref_dataset_name, 

93 run, 

94 htmTableFile, 

95 transfer="auto") 

96 

97 # Test if we can get back the catalogs, with a new butler. 

98 butler = lsst.daf.butler.Butler(repoPath) 

99 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

100 collections=[run]).expanded()) 

101 handlers = [] 

102 for dataRef in datasetRefs: 

103 handlers.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

104 loaderConfig = ReferenceObjectLoader.ConfigClass() 

105 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in datasetRefs], 

106 handlers, 

107 loaderConfig, 

108 log=self.logger) 

109 self.checkAllRowsInRefcat(loader, skyCatalog1, config) 

110 self.checkAllRowsInRefcat(loader, skyCatalog2, config) 

111 

112 runTest(withRaDecErr=True) 

113 runTest(withRaDecErr=False) 

114 

115 

116class TestConvertRefcatManager(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

117 lsst.utils.tests.TestCase): 

118 """Unittests of various methods of ConvertRefcatManager. 

119 

120 Uses mocks to force particular behavior regarding e.g. catalogs. 

121 """ 

122 def setUp(self): 

123 np.random.seed(10) 

124 

125 tempPath = tempfile.mkdtemp() 

126 self.log = lsst.log.Log.getLogger("lsst.TestIngestIndexManager") 

127 self.config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True) 

128 self.config.id_name = 'id' 

129 self.depth = 2 # very small depth, for as few pixels as possible. 

130 self.indexer = HtmIndexer(self.depth) 

131 self.htm = lsst.sphgeom.HtmPixelization(self.depth) 

132 ingester = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config) 

133 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'), 

134 ('a', '<f8'), ('a_err', '<f8')] 

135 self.schema, self.key_map = ingester.makeSchema(dtype) 

136 self.fileReader = ReadTextCatalogTask() 

137 

138 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543) 

139 self.matchedPixels = np.array([1, 1, 2, 2, 3]) 

140 self.path = tempfile.mkdtemp() 

141 self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)} 

142 

143 self.worker = ConvertRefcatManager(self.filenames, 

144 self.config, 

145 self.fileReader, 

146 self.indexer, 

147 self.schema, 

148 self.key_map, 

149 self.htm.universe()[0], 

150 addRefCatMetadata, 

151 self.log) 

152 

153 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42): 

154 """Create a fake output SimpleCatalog, populated with nOld+nNew elements. 

155 

156 Parameters 

157 ---------- 

158 nOld : `int`, optional 

159 The number of filled in sources to put in the catalog. 

160 nNew : `int`, optional 

161 The number of empty sources to put in the catalog. 

162 idStart : `int`, optional 

163 The start id of the ``nOld`` sources. 

164 

165 Returns 

166 ------- 

167 catalog : `lsst.afw.table.SimpleCatalog` 

168 A catalog populated with random data and contiguous ids. 

169 """ 

170 catalog = lsst.afw.table.SimpleCatalog(self.schema) 

171 catalog.resize(nOld) 

172 for x in self.schema: 

173 catalog[x.key] = np.random.random(nOld) 

174 # do the ids separately, so there are no duplicates 

175 catalog['id'] = np.arange(idStart, idStart + nOld) 

176 catalog.resize(nOld + nNew) # make space for the elements we will add 

177 return catalog.copy(deep=True) 

178 

179 def test_doOnePixelNewData(self): 

180 """Test that we can add new data to an existing catalog.""" 

181 pixelId = 1 # the pixel we are going to test 

182 

183 nOld = 5 

184 nNew = sum(self.matchedPixels == pixelId) 

185 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

186 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

187 

188 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

189 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

190 

191 # check that the "pre" catalog is unchanged, exactly 

192 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id']) 

193 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra']) 

194 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec']) 

195 

196 # check that the new catalog elements are set correctly 

197 newElements = self.fakeInput[self.matchedPixels == pixelId] 

198 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id']) 

199 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180) 

200 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180) 

201 

202 def test_doOnePixelNoData(self): 

203 """Test that we can put new data into an empty catalog.""" 

204 pixelId = 2 

205 

206 nOld = 0 

207 nNew = sum(self.matchedPixels == pixelId) 

208 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

209 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

210 

211 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

212 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

213 

214 # check that the new catalog elements are set correctly 

215 newElements = self.fakeInput[self.matchedPixels == pixelId] 

216 np.testing.assert_equal(newcat['id'], newElements['id']) 

217 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180) 

218 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180) 

219 

220 def test_getCatalog(self): 

221 """Test that getCatalog returns a properly expanded new catalog.""" 

222 pixelId = 3 

223 nOld = 10 

224 nNewElements = 5 

225 # save a catalog to disk that we can check against the getCatalog()'s return 

226 catalog = self._createFakeCatalog(nOld=nOld, nNew=0) 

227 catalog.writeFits(self.filenames[pixelId]) 

228 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements) 

229 

230 self.assertEqual(len(newcat), nOld + nNewElements) 

231 

232 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id']) 

233 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra']) 

234 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec']) 

235 

236 

237class TestMemory(lsst.utils.tests.MemoryTestCase): 

238 pass 

239 

240 

241def setup_module(module): 

242 lsst.utils.tests.init() 

243 

244 

245if __name__ == "__main__": 245 ↛ 246line 245 didn't jump to line 246, because the condition on line 245 was never true

246 lsst.utils.tests.init() 

247 unittest.main()