Coverage for tests/test_graphBuilder.py: 13%

87 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-22 13:28 -0800

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Tests of things related to the GraphBuilder class.""" 

23 

24import io 

25import logging 

26import unittest 

27 

28import lsst.utils.tests 

29from lsst.daf.butler.registry import UserExpressionError 

30from lsst.pipe.base import QuantumGraph 

31from lsst.pipe.base.graphBuilder import DatasetQueryConstraintVariant 

32from lsst.pipe.base.tests import simpleQGraph 

33from lsst.utils.tests import temporaryDirectory 

34 

35_LOG = logging.getLogger(__name__) 

36 

37 

38class GraphBuilderTestCase(unittest.TestCase): 

39 def testDefault(self): 

40 """Simple test to verify makeSimpleQGraph can be used to make a Quantum 

41 Graph.""" 

42 with temporaryDirectory() as root: 

43 # makeSimpleQGraph calls GraphBuilder. 

44 butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root) 

45 # by default makeSimpleQGraph makes a graph with 5 nodes 

46 self.assertEqual(len(qgraph), 5) 

47 constraint = DatasetQueryConstraintVariant.OFF 

48 _, qgraph2 = simpleQGraph.makeSimpleQGraph( 

49 butler=butler, datasetQueryConstraint=constraint, callPopulateButler=False 

50 ) 

51 self.assertEqual(len(qgraph2), 5) 

52 self.assertEqual(qgraph, qgraph2) 

53 constraint = DatasetQueryConstraintVariant.fromExpression("add_dataset0") 

54 _, qgraph3 = simpleQGraph.makeSimpleQGraph( 

55 butler=butler, datasetQueryConstraint=constraint, callPopulateButler=False 

56 ) 

57 self.assertEqual(qgraph2, qgraph3) 

58 

59 def testAddInstrumentMismatch(self): 

60 """Verify that a RuntimeError is raised if the instrument in the user 

61 query does not match the instrument in the pipeline.""" 

62 with temporaryDirectory() as root: 

63 pipeline = simpleQGraph.makeSimplePipeline( 

64 nQuanta=5, instrument="lsst.pipe.base.tests.simpleQGraph.SimpleInstrument" 

65 ) 

66 with self.assertRaises(UserExpressionError): 

67 simpleQGraph.makeSimpleQGraph(root=root, pipeline=pipeline, userQuery="instrument = 'foo'") 

68 

69 def testUserQueryBind(self): 

70 """Verify that bind values work for user query.""" 

71 pipeline = simpleQGraph.makeSimplePipeline( 

72 nQuanta=5, instrument="lsst.pipe.base.tests.simpleQGraph.SimpleInstrument" 

73 ) 

74 instr = simpleQGraph.SimpleInstrument.getName() 

75 # With a literal in the user query 

76 with temporaryDirectory() as root: 

77 simpleQGraph.makeSimpleQGraph(root=root, pipeline=pipeline, userQuery=f"instrument = '{instr}'") 

78 # With a bind for the user query 

79 with temporaryDirectory() as root: 

80 simpleQGraph.makeSimpleQGraph( 

81 root=root, pipeline=pipeline, userQuery="instrument = instr", bind={"instr": instr} 

82 ) 

83 

84 def test_datastore_records(self): 

85 """Test for generating datastore records.""" 

86 with temporaryDirectory() as root: 

87 # need FileDatastore for this tests 

88 butler, qgraph1 = simpleQGraph.makeSimpleQGraph( 

89 root=root, inMemory=False, makeDatastoreRecords=True 

90 ) 

91 

92 # save and reload 

93 buffer = io.BytesIO() 

94 qgraph1.save(buffer) 

95 buffer.seek(0) 

96 qgraph2 = QuantumGraph.load(buffer, universe=butler.dimensions) 

97 del buffer 

98 

99 for qgraph in (qgraph1, qgraph2): 

100 self.assertEqual(len(qgraph), 5) 

101 for i, qnode in enumerate(qgraph): 

102 quantum = qnode.quantum 

103 self.assertIsNotNone(quantum.datastore_records) 

104 # only the first quantum has a pre-existing input 

105 if i == 0: 

106 datastore_name = "FileDatastore@<butlerRoot>" 

107 self.assertEqual(set(quantum.datastore_records.keys()), {datastore_name}) 

108 records_data = quantum.datastore_records[datastore_name] 

109 records = dict(records_data.records) 

110 self.assertEqual(len(records), 1) 

111 _, records = records.popitem() 

112 records = records["file_datastore_records"] 

113 self.assertEqual( 

114 [record.path for record in records], 

115 ["test/add_dataset0/add_dataset0_INSTR_det0_test.pickle"], 

116 ) 

117 else: 

118 self.assertEqual(quantum.datastore_records, {}) 

119 

120 def testResolveRefs(self): 

121 """Test for GraphBuilder with resolveRefs=True.""" 

122 

123 with temporaryDirectory() as root: 

124 # Make a regular graph, only inputs are resolved. 

125 _, qgraph = simpleQGraph.makeSimpleQGraph(root=root) 

126 self.assertEqual(len(qgraph), 5) 

127 for node in qgraph: 

128 quantum = node.quantum 

129 for datasetType, refs in quantum.inputs.items(): 

130 if datasetType.name == "add_dataset0": 

131 self.assertTrue(all(ref.id is not None for ref in refs)) 

132 else: 

133 self.assertTrue(all(ref.id is None for ref in refs)) 

134 for datasetType, refs in quantum.outputs.items(): 

135 self.assertTrue(all(ref.id is None for ref in refs)) 

136 

137 with temporaryDirectory() as root: 

138 # Make a resolved graph, all refs should be resolved. 

139 _, qgraph = simpleQGraph.makeSimpleQGraph(root=root, resolveRefs=True) 

140 self.assertEqual(len(qgraph), 5) 

141 for node in qgraph: 

142 quantum = node.quantum 

143 for datasetType, refs in quantum.inputs.items(): 

144 self.assertTrue(all(ref.id is not None for ref in refs)) 

145 for datasetType, refs in quantum.outputs.items(): 

146 self.assertTrue(all(ref.id is not None for ref in refs)) 

147 for taskDef in qgraph.iterTaskGraph(): 

148 if (refs := qgraph.initInputRefs(taskDef)) is not None: 

149 self.assertTrue(all(ref.id is not None for ref in refs)) 

150 if (refs := qgraph.initOutputRefs(taskDef)) is not None: 

151 self.assertTrue(all(ref.id is not None for ref in refs)) 

152 

153 

154if __name__ == "__main__": 154 ↛ 155line 154 didn't jump to line 155, because the condition on line 154 was never true

155 lsst.utils.tests.init() 

156 unittest.main()