Coverage for tests/test_pipelines.py: 35%

39 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-08 12:12 +0000

1# This file is part of ap_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import glob 

23import os.path 

24import tempfile 

25import unittest 

26 

27# need to import pyproj to prevent file handle leakage 

28import pyproj # noqa: F401 

29 

30import lsst.daf.butler.tests as butlerTests 

31import lsst.pipe.base 

32from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester # Can't use fully-qualified name 

33import lsst.utils 

34import lsst.utils.tests 

35 

36 

37class PipelineDefintionsTestSuite(lsst.utils.tests.TestCase): 

38 """Tests of the self-consistency of our pipeline definitions. 

39 """ 

40 def setUp(self): 

41 self.path = os.path.join(lsst.utils.getPackageDir("ap_pipe"), "pipelines") 

42 

43 def test_graph_build(self): 

44 """Test that each pipeline definition file in `_ingredients/` can be 

45 used to build a graph. 

46 """ 

47 files = glob.glob(os.path.join(self.path, "_ingredients/*.yaml")) 

48 for file in files: 

49 if "ApTemplate" in file: 

50 # Our ApTemplate definition cannot be tested here because it 

51 # depends on drp_tasks, which we cannot make a dependency here. 

52 continue 

53 with self.subTest(file): 

54 pipeline = lsst.pipe.base.Pipeline.from_uri(file) 

55 pipeline.addConfigOverride("parameters", "apdb_config", "some/file/path.yaml") 

56 # If this fails, it will produce a useful error message. 

57 pipeline.to_graph() 

58 

59 def test_datasets(self): 

60 files = glob.glob(os.path.join(self.path, "_ingredients/*.yaml")) 

61 for file in files: 

62 if "ApTemplate" in file: 

63 # Our ApTemplate definition cannot be tested here because it 

64 # depends on drp_tasks, which we cannot make a dependency here. 

65 continue 

66 with self.subTest(file): 

67 tester = PipelineStepTester( 

68 filename=file, 

69 step_suffixes=[""], # Test full pipeline 

70 initial_dataset_types=[("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

71 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

72 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

73 ], 

74 expected_inputs={ 

75 # ISR 

76 "raw", "camera", "crosstalk", "crosstalkSources", "bias", "dark", "flat", "ptc", 

77 "fringe", "straylightData", "bfKernel", "newBFKernel", "defects", "linearizer", 

78 "opticsTransmission", "filterTransmission", "atmosphereTransmission", 

79 "illumMaskedImage", "deferredChargeCalib", 

80 # Everything else 

81 "skyMap", "gaia_dr3_20230707", "gaia_dr2_20200414", "ps1_pv3_3pi_20170110", 

82 "goodSeeingCoadd", "pretrainedModelPackage", 

83 }, 

84 # Pipeline outputs highly in flux, don't test 

85 expected_outputs=set(), 

86 pipeline_patches={"parameters:apdb_config": "some/file/path.yaml", 

87 }, 

88 ) 

89 # Tester modifies Butler registry, so need a fresh repo every time 

90 with tempfile.TemporaryDirectory() as tempRepo: 

91 butler = butlerTests.makeTestRepo(tempRepo) 

92 tester.run(butler, self) 

93 

94 

95class MemoryTester(lsst.utils.tests.MemoryTestCase): 

96 pass 

97 

98 

99def setup_module(module): 

100 lsst.utils.tests.init() 

101 

102 

103if __name__ == "__main__": 103 ↛ 104line 103 didn't jump to line 104, because the condition on line 103 was never true

104 lsst.utils.tests.init() 

105 unittest.main()