Coverage for tests/test_driver.py: 40%

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import functools 

25import os 

26import shutil 

27import tempfile 

28import unittest.mock 

29 

30import lsst.utils.tests 

31from lsst.obs.base import RawIngestTask, DefineVisitsTask 

32from lsst.ap.verify import pipeline_driver 

33from lsst.ap.verify.testUtils import DataTestCase 

34from lsst.ap.verify import Dataset, WorkspaceGen3 

35 

36 

37TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40def _getDataIds(butler): 

41 return list(butler.registry.queryDataIds({"instrument", "visit", "detector"}, datasets="raw")) 

42 

43 

44def patchApPipeGen3(method): 

45 """Shortcut decorator for consistently patching AP code. 

46 """ 

47 @functools.wraps(method) 

48 def wrapper(self, *args, **kwargs): 

49 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

50 patchedMethod = dbPatcher(method) 

51 return patchedMethod(self, *args, **kwargs) 

52 return wrapper 

53 

54 

55class PipelineDriverTestSuiteGen3(DataTestCase): 

56 

57 @classmethod 

58 def setUpClass(cls): 

59 super().setUpClass() 

60 

61 cls.dataset = Dataset(cls.testDataset) 

62 

63 def setUp(self): 

64 super().setUp() 

65 

66 self._testDir = tempfile.mkdtemp() 

67 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

68 

69 self.workspace = WorkspaceGen3(self._testDir) 

70 self.dataset.makeCompatibleRepoGen3(self.workspace.repo) 

71 raws = [os.path.join(self.dataset.rawLocation, "lsst_a_204595_R11_S01_i.fits")] 

72 rawIngest = RawIngestTask(butler=self.workspace.workButler, config=RawIngestTask.ConfigClass()) 

73 rawIngest.run(raws, run=None) 

74 defineVisit = DefineVisitsTask(butler=self.workspace.workButler, 

75 config=DefineVisitsTask.ConfigClass()) 

76 defineVisit.run(self.workspace.workButler.registry.queryDataIds("exposure", datasets="raw")) 

77 ids = _getDataIds(self.workspace.workButler) 

78 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

79 ["--data-query", f"instrument = '{ids[0]['instrument']}' AND visit = {ids[0]['visit']}", 

80 "--pipeline", os.path.join(TESTDIR, "MockApPipe.yaml")]) 

81 

82 def testrunApPipeGen3Steps(self): 

83 """Test that runApPipeGen3 runs the entire pipeline. 

84 """ 

85 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

86 

87 # Use datasets as a proxy for pipeline completion 

88 id = _getDataIds(self.workspace.analysisButler)[0] 

89 self.assertTrue(self.workspace.analysisButler.exists("calexp", id)) 

90 self.assertTrue(self.workspace.analysisButler.exists("src", id)) 

91 self.assertTrue(self.workspace.analysisButler.exists("goodSeeingDiff_differenceExp", id)) 

92 self.assertTrue(self.workspace.analysisButler.exists("goodSeeingDiff_diaSrc", id)) 

93 self.assertTrue(self.workspace.analysisButler.exists("apdb_marker", id)) 

94 self.assertTrue(self.workspace.analysisButler.exists("goodSeeingDiff_assocDiaSrc", id)) 

95 

96 def _getCmdLineArgs(self, parseAndRunArgs): 

97 if parseAndRunArgs[0]: 

98 return parseAndRunArgs[0][0] 

99 elif "args" in parseAndRunArgs[1]: 

100 return parseAndRunArgs[1]["args"] 

101 else: 

102 self.fail("No command-line args passed to parseAndRun!") 

103 

104 @patchApPipeGen3 

105 def testrunApPipeGen3WorkspaceDb(self, mockDb): 

106 """Test that runApPipeGen3 places a database in the workspace location by default. 

107 """ 

108 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

109 

110 # Test the call to make_apdb.py 

111 mockDb.assert_called_once() 

112 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

113 self.assertIn("db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

114 

115 # Test the call to the AP pipeline 

116 id = _getDataIds(self.workspace.analysisButler)[0] 

117 apdbConfig = self.workspace.analysisButler.get("apdb_marker", id) 

118 self.assertEqual(apdbConfig.db_url, "sqlite:///" + self.workspace.dbLocation) 

119 

120 @patchApPipeGen3 

121 def testrunApPipeGen3WorkspaceCustom(self, mockDb): 

122 """Test that runApPipeGen3 places a database in the specified location. 

123 """ 

124 self.apPipeArgs.db = "postgresql://somebody@pgdb.misc.org/custom_db" 

125 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

126 

127 # Test the call to make_apdb.py 

128 mockDb.assert_called_once() 

129 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

130 self.assertIn("db_url=" + self.apPipeArgs.db, cmdLineArgs) 

131 

132 # Test the call to the AP pipeline 

133 id = _getDataIds(self.workspace.analysisButler)[0] 

134 apdbConfig = self.workspace.analysisButler.get("apdb_marker", id) 

135 self.assertEqual(apdbConfig.db_url, self.apPipeArgs.db) 

136 

137 def testrunApPipeGen3Reuse(self): 

138 """Test that runApPipeGen3 does not run the pipeline at all (not even with 

139 --skip-existing) if --skip-pipeline is provided. 

140 """ 

141 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

142 pipeline_driver.runApPipeGen3(self.workspace, skipArgs) 

143 

144 # Use datasets as a proxy for pipeline completion. 

145 # Depending on the overall test setup, the dataset may or may not be 

146 # registered if the pipeline didn't run; check both cases. 

147 id = _getDataIds(self.workspace.analysisButler)[0] 

148 calexpQuery = set(self.workspace.analysisButler.registry.queryDatasetTypes("calexp")) 

149 calexpExists = len(calexpQuery) > 0 

150 self.assertFalse(calexpExists and self.workspace.analysisButler.exists("calexp", id)) 

151 

152 

153class MemoryTester(lsst.utils.tests.MemoryTestCase): 

154 pass 

155 

156 

157def setup_module(module): 

158 lsst.utils.tests.init() 

159 

160 

161if __name__ == "__main__": 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true

162 lsst.utils.tests.init() 

163 unittest.main()