Coverage for tests/test_driver.py: 35%
86 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:39 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-08 23:39 -0800
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import functools
25import os
26import shutil
27import tempfile
28import unittest.mock
30import lsst.utils.tests
31from lsst.obs.base import RawIngestTask, DefineVisitsTask
32from lsst.ap.verify import pipeline_driver
33from lsst.ap.verify.testUtils import DataTestCase
34from lsst.ap.verify import Dataset, WorkspaceGen3
37TESTDIR = os.path.abspath(os.path.dirname(__file__))
40def _getDataIds(butler):
41 return list(butler.registry.queryDataIds({"instrument", "visit", "detector"}, datasets="raw"))
44def patchApPipeGen3(method):
45 """Shortcut decorator for consistently patching AP code.
46 """
47 @functools.wraps(method)
48 def wrapper(self, *args, **kwargs):
49 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb")
50 patchedMethod = dbPatcher(method)
51 return patchedMethod(self, *args, **kwargs)
52 return wrapper
55class PipelineDriverTestSuiteGen3(DataTestCase):
57 @classmethod
58 def setUpClass(cls):
59 super().setUpClass()
61 cls.dataset = Dataset(cls.testDataset)
63 def setUp(self):
64 super().setUp()
66 self._testDir = tempfile.mkdtemp()
67 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True)
69 self.workspace = WorkspaceGen3(self._testDir)
70 self.dataset.makeCompatibleRepoGen3(self.workspace.repo)
71 raws = [os.path.join(self.dataset.rawLocation, "lsst_a_204595_R11_S01_i.fits")]
72 rawIngest = RawIngestTask(butler=self.workspace.workButler, config=RawIngestTask.ConfigClass())
73 rawIngest.run(raws, run=None)
74 defineVisit = DefineVisitsTask(butler=self.workspace.workButler,
75 config=DefineVisitsTask.ConfigClass())
76 defineVisit.run(self.workspace.workButler.registry.queryDataIds("exposure", datasets="raw"))
77 ids = _getDataIds(self.workspace.workButler)
78 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args(
79 ["--data-query", f"instrument = '{ids[0]['instrument']}' AND visit = {ids[0]['visit']}",
80 "--pipeline", os.path.join(TESTDIR, "MockApPipe.yaml")])
82 def testrunApPipeGen3Steps(self):
83 """Test that runApPipeGen3 runs the entire pipeline.
84 """
85 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs)
87 # Use datasets as a proxy for pipeline completion
88 id = _getDataIds(self.workspace.analysisButler)[0]
89 self.assertTrue(self.workspace.analysisButler.datasetExists("calexp", id))
90 self.assertTrue(self.workspace.analysisButler.datasetExists("src", id))
91 self.assertTrue(self.workspace.analysisButler.datasetExists("goodSeeingDiff_differenceExp", id))
92 self.assertTrue(self.workspace.analysisButler.datasetExists("goodSeeingDiff_diaSrc", id))
93 self.assertTrue(self.workspace.analysisButler.datasetExists("apdb_marker", id))
94 self.assertTrue(self.workspace.analysisButler.datasetExists("goodSeeingDiff_assocDiaSrc", id))
96 def _getCmdLineArgs(self, parseAndRunArgs):
97 if parseAndRunArgs[0]:
98 return parseAndRunArgs[0][0]
99 elif "args" in parseAndRunArgs[1]:
100 return parseAndRunArgs[1]["args"]
101 else:
102 self.fail("No command-line args passed to parseAndRun!")
104 @patchApPipeGen3
105 def testrunApPipeGen3WorkspaceDb(self, mockDb):
106 """Test that runApPipeGen3 places a database in the workspace location by default.
107 """
108 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs)
110 # Test the call to make_apdb.py
111 mockDb.assert_called_once()
112 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args)
113 self.assertIn("db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs)
115 # Test the call to the AP pipeline
116 id = _getDataIds(self.workspace.analysisButler)[0]
117 apdbConfig = self.workspace.analysisButler.get("apdb_marker", id)
118 self.assertEqual(apdbConfig.db_url, "sqlite:///" + self.workspace.dbLocation)
120 @patchApPipeGen3
121 def testrunApPipeGen3WorkspaceCustom(self, mockDb):
122 """Test that runApPipeGen3 places a database in the specified location.
123 """
124 self.apPipeArgs.db = "postgresql://somebody@pgdb.misc.org/custom_db"
125 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs)
127 # Test the call to make_apdb.py
128 mockDb.assert_called_once()
129 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args)
130 self.assertIn("db_url=" + self.apPipeArgs.db, cmdLineArgs)
132 # Test the call to the AP pipeline
133 id = _getDataIds(self.workspace.analysisButler)[0]
134 apdbConfig = self.workspace.analysisButler.get("apdb_marker", id)
135 self.assertEqual(apdbConfig.db_url, self.apPipeArgs.db)
137 def testrunApPipeGen3Reuse(self):
138 """Test that runApPipeGen3 does not run the pipeline at all (not even with
139 --skip-existing) if --skip-pipeline is provided.
140 """
141 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"])
142 pipeline_driver.runApPipeGen3(self.workspace, skipArgs)
144 # Use datasets as a proxy for pipeline completion.
145 # Depending on the overall test setup, the dataset may or may not be
146 # registered if the pipeline didn't run; check both cases.
147 id = _getDataIds(self.workspace.analysisButler)[0]
148 calexpQuery = set(self.workspace.analysisButler.registry.queryDatasetTypes("calexp"))
149 calexpExists = len(calexpQuery) > 0
150 self.assertFalse(calexpExists and self.workspace.analysisButler.datasetExists("calexp", id))
153class MemoryTester(lsst.utils.tests.MemoryTestCase):
154 pass
157def setup_module(module):
158 lsst.utils.tests.init()
161if __name__ == "__main__": 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true
162 lsst.utils.tests.init()
163 unittest.main()