Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22# This file is excluded from running through pytest due to concerns about the 

23# interaction between multiprocessing as invoked by this code, and the process 

24# pool used by pytest. 

25# 

26# Note that it is invoked independently by SCons, so the tests are still run 

27# as part of the build. 

28 

29import os.path 

30import tempfile 

31import unittest 

32import unittest.mock 

33 

34import numpy as np 

35 

36import lsst.daf.persistence as dafPersist 

37from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask, 

38 LoadIndexedReferenceObjectsConfig) 

39from lsst.meas.algorithms.htmIndexer import HtmIndexer 

40from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

41from lsst.meas.algorithms.ingestIndexManager import IngestIndexManager 

42from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask 

43import lsst.utils 

44 

45import ingestIndexTestBase 

46 

47 

48class TestIngestReferenceCatalogParallel(ingestIndexTestBase.IngestIndexCatalogTestBase, 

49 lsst.utils.tests.TestCase): 

50 """Test ingesting a refcat with multiprocessing turned on.""" 

51 def testIngestTwoFilesTwoCores(self): 

52 def runTest(withRaDecErr): 

53 # Generate a second catalog, with different ids 

54 inPath1 = tempfile.mkdtemp() 

55 skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123) 

56 inPath2 = tempfile.mkdtemp() 

57 skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11) 

58 # override some field names, and use multiple cores 

59 config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

60 withPm=True, withPmErr=True) 

61 # use a very small HTM pixelization depth to ensure there will be collisions when 

62 # ingesting the files in parallel 

63 config.dataset_config.indexer.active.depth = 2 

64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

65 config.file_reader.format = 'ascii.commented_header' 

66 config.n_processes = 2 # use multiple cores for this test only 

67 config.id_name = 'id' # Use the ids from the generated catalogs 

68 outpath = os.path.join(self.outPath, "output_multifile_parallel", 

69 "_withRaDecErr" if withRaDecErr else "") 

70 IngestIndexedReferenceTask.parseAndRun( 

71 args=[self.input_dir, "--output", outpath, 

72 skyCatalogFile1, skyCatalogFile2], config=config) 

73 

74 # Test if we can get back the catalog with a non-standard dataset name 

75 butler = dafPersist.Butler(outpath) 

76 loaderConfig = LoadIndexedReferenceObjectsConfig() 

77 loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) 

78 self.checkAllRowsInRefcat(loader, skyCatalog1, config) 

79 self.checkAllRowsInRefcat(loader, skyCatalog2, config) 

80 

81 runTest(withRaDecErr=True) 

82 runTest(withRaDecErr=False) 

83 

84 

85class TestIngestIndexManager(ingestIndexTestBase.IngestIndexCatalogTestBase, 

86 lsst.utils.tests.TestCase): 

87 """Unittests of various methods of IngestIndexManager. 

88 

89 Uses mocks to force particular behavior regarding e.g. catalogs. 

90 """ 

91 def setUp(self): 

92 np.random.seed(10) 

93 

94 self.log = lsst.log.Log.getLogger("TestIngestIndexManager") 

95 self.config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=True) 

96 self.config.id_name = 'id' 

97 depth = 2 # very small depth, for as few pixels as possible. 

98 self.indexer = HtmIndexer(depth) 

99 self.htm = lsst.sphgeom.HtmPixelization(depth) 

100 ingester = IngestIndexedReferenceTask(self.config) 

101 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'), 

102 ('a', '<f8'), ('a_err', '<f8')] 

103 self.schema, self.key_map = ingester.makeSchema(dtype) 

104 self.fileReader = ReadTextCatalogTask() 

105 

106 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543) 

107 self.matchedPixels = np.array([1, 1, 2, 2, 3]) 

108 self.path = tempfile.mkdtemp() 

109 self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)} 

110 

111 self.worker = IngestIndexManager(self.filenames, 

112 self.config, 

113 self.fileReader, 

114 self.indexer, 

115 self.schema, 

116 self.key_map, 

117 self.htm.universe()[0], 

118 addRefCatMetadata, 

119 self.log) 

120 

121 def _createFakeCatalog(self, nOld=5, nNew=0, idStart=42): 

122 """Create a fake output SimpleCatalog, populated with nOld+nNew elements. 

123 

124 Parameters 

125 ---------- 

126 nOld : `int`, optional 

127 The number of filled in sources to put in the catalog. 

128 nNew : `int`, optional 

129 The number of empty sources to put in the catalog. 

130 idStart : `int`, optional 

131 The start id of the ``nOld`` sources. 

132 

133 Returns 

134 ------- 

135 catalog : `lsst.afw.table.SimpleCatalog` 

136 A catalog populated with random data and contiguous ids. 

137 """ 

138 catalog = lsst.afw.table.SimpleCatalog(self.schema) 

139 catalog.resize(nOld) 

140 for x in self.schema: 

141 catalog[x.key] = np.random.random(nOld) 

142 # do the ids separately, so there are no duplicates 

143 catalog['id'] = np.arange(idStart, idStart + nOld) 

144 catalog.resize(nOld + nNew) # make space for the elements we will add 

145 return catalog.copy(deep=True) 

146 

147 def test_doOnePixelNewData(self): 

148 """Test that we can add new data to an existing catalog.""" 

149 pixelId = 1 # the pixel we are going to test 

150 

151 nOld = 5 

152 nNew = sum(self.matchedPixels == pixelId) 

153 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

154 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

155 

156 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

157 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

158 

159 # check that the "pre" catalog is unchanged, exactly 

160 np.testing.assert_equal(newcat[:nOld]['id'], catalog[:nOld]['id']) 

161 self.assertFloatsEqual(newcat[:nOld]['coord_ra'], catalog[:nOld]['coord_ra']) 

162 self.assertFloatsEqual(newcat[:nOld]['coord_dec'], catalog[:nOld]['coord_dec']) 

163 

164 # check that the new catalog elements are set correctly 

165 newElements = self.fakeInput[self.matchedPixels == pixelId] 

166 np.testing.assert_equal(newcat[nOld:]['id'], newElements['id']) 

167 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_ra'], newElements['ra_icrs']*np.pi/180) 

168 self.assertFloatsAlmostEqual(newcat[nOld:]['coord_dec'], newElements['dec_icrs']*np.pi/180) 

169 

170 def test_doOnePixelNoData(self): 

171 """Test that we can put new data into an empty catalog.""" 

172 pixelId = 2 

173 

174 nOld = 0 

175 nNew = sum(self.matchedPixels == pixelId) 

176 catalog = self._createFakeCatalog(nOld=nOld, nNew=nNew) 

177 self.worker.getCatalog = unittest.mock.Mock(self.worker.getCatalog, return_value=catalog) 

178 

179 self.worker._doOnePixel(self.fakeInput, self.matchedPixels, pixelId, {}, {}) 

180 newcat = lsst.afw.table.SimpleCatalog.readFits(self.filenames[pixelId]) 

181 

182 # check that the new catalog elements are set correctly 

183 newElements = self.fakeInput[self.matchedPixels == pixelId] 

184 np.testing.assert_equal(newcat['id'], newElements['id']) 

185 self.assertFloatsAlmostEqual(newcat['coord_ra'], newElements['ra_icrs']*np.pi/180) 

186 self.assertFloatsAlmostEqual(newcat['coord_dec'], newElements['dec_icrs']*np.pi/180) 

187 

188 def test_getCatalog(self): 

189 """Test that getCatalog returns a properly expanded new catalog.""" 

190 pixelId = 3 

191 nOld = 10 

192 nNewElements = 5 

193 # save a catalog to disk that we can check against the getCatalog()'s return 

194 catalog = self._createFakeCatalog(nOld=nOld, nNew=0) 

195 catalog.writeFits(self.filenames[pixelId]) 

196 newcat = self.worker.getCatalog(pixelId, self.schema, nNewElements) 

197 

198 self.assertEqual(len(newcat), nOld + nNewElements) 

199 

200 np.testing.assert_equal(newcat[:len(catalog)]['id'], catalog['id']) 

201 self.assertFloatsEqual(newcat[:len(catalog)]['coord_ra'], catalog['coord_ra']) 

202 self.assertFloatsEqual(newcat[:len(catalog)]['coord_dec'], catalog['coord_dec']) 

203 

204 

205class TestMemory(lsst.utils.tests.MemoryTestCase): 

206 pass 

207 

208 

209def setup_module(module): 

210 lsst.utils.tests.init() 

211 

212 

213if __name__ == "__main__": 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true

214 lsst.utils.tests.init() 

215 unittest.main()