Coverage for tests/test_cliCmdQueryDataIds.py: 30%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for daf_butler CLI query-collections command.""" 

29 

30import os 

31import unittest 

32 

33from astropy.table import Table as AstropyTable 

34from numpy import array 

35 

36from lsst.daf.butler import Butler, DatasetType, script 

37from lsst.daf.butler.direct_butler import DirectButler 

38from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, removeTestTempDir 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43class QueryDataIdsTest(unittest.TestCase, ButlerTestHelper): 

44 """Test the query-data-ids command-line.""" 

45 

46 mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDataIds" 

47 

48 @staticmethod 

49 def _queryDataIds(repo, dimensions=(), collections=(), datasets=None, where=""): 

50 """Call script.queryDataIds, allowing for default values.""" 

51 return script.queryDataIds( 

52 repo=repo, 

53 dimensions=dimensions, 

54 collections=collections, 

55 datasets=datasets, 

56 where=where, 

57 order_by=None, 

58 limit=0, 

59 offset=0, 

60 ) 

61 

62 def setUp(self): 

63 self.root = makeTestTempDir(TESTDIR) 

64 self.repo = Butler.makeRepo(self.root) 

65 

66 def tearDown(self): 

67 removeTestTempDir(self.root) 

68 

69 def loadData(self, *filenames: str) -> Butler: 

70 """Load registry test data from ``TESTDIR/data/registry/<filename>``, 

71 which should be a YAML import/export file. 

72 """ 

73 butler = Butler.from_config(self.repo, writeable=True) 

74 assert isinstance(butler, DirectButler), "Test expects DirectButler" 

75 for filename in filenames: 

76 butler.import_( 

77 filename=f"resource://lsst.daf.butler/tests/registry_data/{filename}", 

78 without_datastore=True, 

79 ) 

80 return butler 

81 

82 def testDimensions(self): 

83 """Test getting a dimension.""" 

84 self.loadData("base.yaml") 

85 res, msg = self._queryDataIds(self.root, dimensions=("detector",)) 

86 expected = AstropyTable( 

87 array((("Cam1", 1), ("Cam1", 2), ("Cam1", 3), ("Cam1", 4))), names=("instrument", "detector") 

88 ) 

89 self.assertFalse(msg) 

90 self.assertAstropyTablesEqual(res, expected) 

91 

92 def testNoDimensions(self): 

93 """Test asking for no dimensions.""" 

94 res, msg = self._queryDataIds(self.root) 

95 self.assertIsNone(res, msg) 

96 self.assertEqual( 

97 msg, "Result has one logical row but no columns because no dimensions were requested." 

98 ) 

99 

100 def testNoResultsEasy(self): 

101 """Test getting no results in a way that's detectable without having 

102 to execute the full query. 

103 """ 

104 self.loadData("base.yaml", "spatial.yaml") 

105 res, msg = self._queryDataIds( 

106 self.root, 

107 dimensions=("visit", "tract"), 

108 where="instrument='Cam1' AND skymap='SkyMap1' AND visit=1 AND tract=1", 

109 ) 

110 self.assertIsNone(res, msg) 

111 self.assertEqual(msg, "") 

112 

113 def testNoResultsHard(self): 

114 """Test getting no results in a way that can't be detected unless we 

115 run the whole query. 

116 """ 

117 self.loadData("base.yaml", "spatial.yaml") 

118 res, msg = self._queryDataIds( 

119 self.root, 

120 dimensions=("visit", "tract"), 

121 where="instrument='Cam1' AND skymap='SkyMap1' AND visit=1 AND tract=0 AND patch=5", 

122 ) 

123 self.assertIsNone(res, msg) 

124 self.assertIn("Post-query region filtering removed all rows", msg) 

125 

126 def testWhere(self): 

127 """Test with a WHERE constraint.""" 

128 self.loadData("base.yaml") 

129 res, msg = self._queryDataIds( 

130 self.root, dimensions=("detector",), where="instrument='Cam1' AND detector=2" 

131 ) 

132 expected = AstropyTable( 

133 array((("Cam1", 2),)), 

134 names=( 

135 "instrument", 

136 "detector", 

137 ), 

138 ) 

139 self.assertAstropyTablesEqual(res, expected) 

140 self.assertIsNone(msg) 

141 

142 def testDatasetsAndCollections(self): 

143 """Test constraining via datasets and collections.""" 

144 butler = self.loadData("base.yaml", "datasets.yaml") 

145 # See that the data IDs returned are constrained by that collection's 

146 # contents. 

147 res, msg = self._queryDataIds( 

148 repo=self.root, dimensions=("detector",), collections=("imported_g",), datasets="bias" 

149 ) 

150 expected = AstropyTable( 

151 array((("Cam1", 1), ("Cam1", 2), ("Cam1", 3))), 

152 names=( 

153 "instrument", 

154 "detector", 

155 ), 

156 ) 

157 self.assertAstropyTablesEqual(res, expected) 

158 self.assertIsNone(msg) 

159 

160 # Check that the dimensions are inferred when not provided. 

161 with self.assertLogs("lsst.daf.butler.script.queryDataIds", "INFO") as cm: 

162 res, msg = self._queryDataIds(repo=self.root, collections=("imported_g",), datasets="bias") 

163 self.assertIn("Determined dimensions", "\n".join(cm.output)) 

164 self.assertAstropyTablesEqual(res, expected) 

165 self.assertIsNone(msg) 

166 

167 # Check that we get a reason if no dimensions can be inferred. 

168 new_dataset_type = DatasetType( 

169 "test_metric_dimensionless", 

170 (), 

171 "StructuredDataDict", 

172 universe=butler.dimensions, 

173 ) 

174 butler.registry.registerDatasetType(new_dataset_type) 

175 res, msg = self._queryDataIds(repo=self.root, collections=("imported_g",), datasets=...) 

176 self.assertIsNone(res) 

177 self.assertIn("No dimensions in common", msg) 

178 

179 # Check that we get a reason returned if no dataset is found in 

180 # collection. 

181 res, msg = self._queryDataIds( 

182 repo=self.root, 

183 dimensions=("detector",), 

184 collections=("imported_g",), 

185 datasets="test_metric_dimensionless", 

186 ) 

187 self.assertIsNone(res) 

188 self.assertIn("No datasets of type 'test_metric_dimensionless'", msg) 

189 

190 

191if __name__ == "__main__": 

192 unittest.main()