Coverage for python/lsst/obs/base/mapper_tests.py: 23%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

86 statements  

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import abc 

23import collections 

24import inspect 

25import os 

26 

27import lsst.afw.geom 

28import lsst.afw.image 

29import lsst.daf.persistence 

30import lsst.utils.tests 

31 

32__all__ = ["MapperTests"] 

33 

34 

35class MapperTests(metaclass=abc.ABCMeta): 

36 """ 

37 Generic tests of obs_* package mapper functionality. 

38 

39 In the subclasses's setUp(): 

40 * Call setUp_mapper() to fill in required parameters. 

41 """ 

42 

43 def setUp_mapper( 

44 self, 

45 output=None, 

46 path_to_raw=None, 

47 keys=None, 

48 query_format=None, 

49 queryMetadata=None, 

50 metadata_output_path=None, 

51 map_python_type=None, 

52 map_python_std_type=None, 

53 map_cpp_type=None, 

54 map_storage_name=None, 

55 raw_filename=None, 

56 default_level=None, 

57 raw_levels=None, 

58 test_config_metadata=True, 

59 ): 

60 """ 

61 Set up the necessary variables for mapper tests. 

62 

63 Parameters 

64 ---------- 

65 

66 output : `str` 

67 full path to output repository (can be the same as data_dir input 

68 repository) 

69 path_to_raw : `str` 

70 full path to the raw file referenced by dataIds['raw'] 

71 keys : `set` 

72 dictionary keys that this mapper should contain 

73 query_format : `list` 

74 format list for the results portion of queryMetadata 

75 queryMetadata : `tuple` of (`dict`, `tuple`) 

76 dataIds and the results of calling them in queryMetadata 

77 metadata_output_path : `str` 

78 path to metadata output associated with dataIds['raw'] 

79 map_python_type : `type` 

80 full python type specification returned by the mapper for 

81 dataIds['raw'] 

82 map_python_std_type : `type` 

83 full python type specification returned by the mapper for 

84 dataIds['raw'] after standardization 

85 map_cpp_type : `str` 

86 C++ type specification returned by the mapper for dataIds['raw'] 

87 map_storage_name : `str` 

88 butler name for the storage type dataIds['raw'] 

89 raw_filename : `str` 

90 Name of the raw files returned by the mapper for dataIds['raw'] 

91 default_level : `str` 

92 value returned from mapper.getDefaultLevel 

93 raw_levels : `tuple` of (`str`, `set` of `str`) 

94 (level, expect) level and expected mapper return for 

95 mapper.getKeys('raw', level) 

96 test_config_metadata : `bool` 

97 Test persisted config and metadata? These tests may not be 

98 appropriate for test stand data. Defaults to True. 

99 """ 

100 fields = [ 

101 "output", 

102 "path_to_raw", 

103 "keys", 

104 "query_format", 

105 "queryMetadata", 

106 "metadata_output_path", 

107 "map_python_type", 

108 "map_python_std_type", 

109 "map_cpp_type", 

110 "map_storage_name", 

111 "raw_filename", 

112 "default_level", 

113 "raw_levels", 

114 "test_config_metadata", 

115 ] 

116 MapperData = collections.namedtuple("MapperData", fields) 

117 self.mapper_data = MapperData( 

118 output=output, 

119 path_to_raw=path_to_raw, 

120 keys=keys, 

121 query_format=query_format, 

122 queryMetadata=queryMetadata, 

123 metadata_output_path=metadata_output_path, 

124 map_python_type=map_python_type, 

125 map_python_std_type=map_python_std_type, 

126 map_cpp_type=map_cpp_type, 

127 map_storage_name=map_storage_name, 

128 raw_filename=raw_filename, 

129 default_level=default_level, 

130 raw_levels=raw_levels, 

131 test_config_metadata=test_config_metadata, 

132 ) 

133 

134 def test_map_config_data(self): 

135 if not self.mapper_data.test_config_metadata: 

136 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

137 dataId = self.dataIds["raw"] 

138 butlerLocation = self.mapper.map("processCcd_config_filename", dataId) 

139 self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig") 

140 self.assertEqual(butlerLocation.getCppType(), "Config") 

141 self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage") 

142 processCcd_path = os.path.join("config", "processCcd.py") 

143 self.assertEqual(self.mapper.root, butlerLocation.getStorage().root) 

144 self.assertEqual(butlerLocation.getLocations(), [processCcd_path]) 

145 for k, v in dataId.items(): 

146 self.assertEqual( 

147 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k) 

148 ) 

149 

150 def test_map_metadata_data(self): 

151 if not self.mapper_data.test_config_metadata: 

152 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

153 dataId = self.dataIds["raw"] 

154 butlerLocation = self.mapper.map_processCcd_metadata(dataId) 

155 self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet") 

156 self.assertEqual(butlerLocation.getCppType(), "PropertySet") 

157 self.assertEqual(butlerLocation.getStorageName(), "YamlStorage") 

158 self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path]) 

159 for k, v in dataId.items(): 

160 self.assertEqual( 

161 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k) 

162 ) 

163 

164 def test_keys(self): 

165 self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys) 

166 

167 def test_get_dataset_types(self): 

168 if not self.mapper_data.test_config_metadata: 

169 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name)) 

170 someKeys = set(["raw", "processCcd_config", "processCcd_metadata"]) 

171 self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys)) 

172 

173 def test_get_keys_raw(self): 

174 for level, expect in self.mapper_data.raw_levels: 

175 result = self.mapper.getKeys("raw", level) 

176 self.assertEqual(set(result), expect, msg="Failed for level={}".format(level)) 

177 

178 def test_get_default_level(self): 

179 self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level) 

180 

181 def _test_map(self, butlerLocation, dataId): 

182 self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type) 

183 self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type) 

184 self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name) 

185 locationList = butlerLocation.getLocations() 

186 self.assertEqual(len(locationList), 1) 

187 fileName = os.path.basename(locationList[0]) 

188 self.assertEqual(fileName, self.mapper_data.raw_filename) 

189 for k, v in dataId.items(): 

190 self.assertEqual( 

191 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k) 

192 ) 

193 

194 def test_map(self): 

195 dataId = self.dataIds["raw"] 

196 location = self.mapper.map_raw(dataId) 

197 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite): 

198 self._test_map(location, dataId) 

199 else: 

200 self.log.warning( 

201 """ButlerComposite datasets are not tested for mapper functions. Though 

202ButlerComposites duck type as ButlerLocations in some ways, they do not 

203share enough methods to be usefully tested by the same function. Note 

204there are tests of the objects in the package in which they are implemented.""" 

205 ) 

206 # This should be the same as above. Testing that both the generic 

207 # and specific interface work for mapping the raw. 

208 location = self.mapper.map("raw", dataId) 

209 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite): 

210 self._test_map(location, dataId) 

211 

212 def test_query_metadata(self): 

213 """ 

214 Test expansion of incomplete information of the available data in this 

215 obs package's testdata repo. 

216 """ 

217 for query, expect in self.mapper_data.queryMetadata: 

218 # queryMetadata returns tuples of available items of the 2nd 

219 # parameter. 

220 result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query) 

221 self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query)) 

222 

223 def test_can_standardize(self): 

224 self.assertTrue(self.mapper.canStandardize("raw")) 

225 self.assertFalse(self.mapper.canStandardize("camera")) 

226 if not self.mapper_data.test_config_metadata: 

227 self.assertFalse(self.mapper.canStandardize("processCcd_config")) 

228 self.assertFalse(self.mapper.canStandardize("processCcd_metadata")) 

229 

230 def _test_validate(self, dataId): 

231 self.assertEqual(self.mapper.validate(dataId), dataId) 

232 

233 def test_validate(self): 

234 self._test_validate({"visit": 1, "filter": "g"}) 

235 self._test_validate({"visit": 2, "filter": "r"}) 

236 self._test_validate({"visit": 3, "filter": "g", "tract": 4}) 

237 # NOTE: when DM-7909 is completed, add assertRaises test here. 

238 # visit must be an integers