Coverage for python/lsst/obs/base/mapper_tests.py: 21%

86 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 01:53 -0800

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import abc 

23import os 

24import inspect 

25 

26import lsst.afw.geom 

27import lsst.utils.tests 

28import lsst.daf.persistence 

29import lsst.afw.image 

30import collections 

31 

32__all__ = ["MapperTests"] 

33 

34 

35class MapperTests(metaclass=abc.ABCMeta): 

36 """ 

37 Generic tests of obs_* package mapper functionality. 

38 

39 In the subclasses's setUp(): 

40 * Call setUp_mapper() to fill in required parameters. 

41 """ 

42 

43 def setUp_mapper(self, 

44 output=None, 

45 path_to_raw=None, 

46 keys=None, 

47 query_format=None, 

48 queryMetadata=None, 

49 metadata_output_path=None, 

50 map_python_type=None, 

51 map_python_std_type=None, 

52 map_cpp_type=None, 

53 map_storage_name=None, 

54 raw_filename=None, 

55 default_level=None, 

56 raw_levels=None, 

57 test_config_metadata=True, 

58 ): 

59 """ 

60 Set up the necessary variables for mapper tests. 

61 

62 Parameters 

63 ---------- 

64 

65 output : `str` 

66 full path to output repository (can be the same as data_dir input 

67 repository) 

68 path_to_raw : `str` 

69 full path to the raw file referenced by dataIds['raw'] 

70 keys : `set` 

71 dictionary keys that this mapper should contain 

72 query_format : `list` 

73 format list for the results portion of queryMetadata 

74 queryMetadata : `tuple` of (`dict`, `tuple`) 

75 dataIds and the results of calling them in queryMetadata 

76 metadata_output_path : `str` 

77 path to metadata output associated with dataIds['raw'] 

78 map_python_type : `type` 

79 full python type specification returned by the mapper for 

80 dataIds['raw'] 

81 map_python_std_type : `type` 

82 full python type specification returned by the mapper for 

83 dataIds['raw'] after standardization 

84 map_cpp_type : `str` 

85 C++ type specification returned by the mapper for dataIds['raw'] 

86 map_storage_name : `str` 

87 butler name for the storage type dataIds['raw'] 

88 raw_filename : `str` 

89 Name of the raw files returned by the mapper for dataIds['raw'] 

90 default_level : `str` 

91 value returned from mapper.getDefaultLevel 

92 raw_levels : `tuple` of (`str`, `set` of `str`) 

93 (level, expect) level and expected mapper return for 

94 mapper.getKeys('raw', level) 

95 test_config_metadata : `bool` 

96 Test persisted config and metadata? These tests may not be 

97 appropriate for test stand data. Defaults to True. 

98 """ 

99 fields = ['output', 

100 'path_to_raw', 

101 'keys', 

102 'query_format', 

103 'queryMetadata', 

104 'metadata_output_path', 

105 'map_python_type', 

106 'map_python_std_type', 

107 'map_cpp_type', 

108 'map_storage_name', 

109 'raw_filename', 

110 'default_level', 

111 'raw_levels', 

112 'test_config_metadata', 

113 ] 

114 MapperData = collections.namedtuple("MapperData", fields) 

115 self.mapper_data = MapperData(output=output, 

116 path_to_raw=path_to_raw, 

117 keys=keys, 

118 query_format=query_format, 

119 queryMetadata=queryMetadata, 

120 metadata_output_path=metadata_output_path, 

121 map_python_type=map_python_type, 

122 map_python_std_type=map_python_std_type, 

123 map_cpp_type=map_cpp_type, 

124 map_storage_name=map_storage_name, 

125 raw_filename=raw_filename, 

126 default_level=default_level, 

127 raw_levels=raw_levels, 

128 test_config_metadata=test_config_metadata, 

129 ) 

130 

131 def test_map_config_data(self): 

132 if not self.mapper_data.test_config_metadata: 

133 self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name)) 

134 dataId = self.dataIds['raw'] 

135 butlerLocation = self.mapper.map("processCcd_config_filename", dataId) 

136 self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig") 

137 self.assertEqual(butlerLocation.getCppType(), "Config") 

138 self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage") 

139 processCcd_path = os.path.join("config", "processCcd.py") 

140 self.assertEqual(self.mapper.root, butlerLocation.getStorage().root) 

141 self.assertEqual(butlerLocation.getLocations(), [processCcd_path]) 

142 for k, v in dataId.items(): 

143 self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v, 

144 msg="Failed for key={}".format(k)) 

145 

146 def test_map_metadata_data(self): 

147 if not self.mapper_data.test_config_metadata: 

148 self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name)) 

149 dataId = self.dataIds['raw'] 

150 butlerLocation = self.mapper.map_processCcd_metadata(dataId) 

151 self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet") 

152 self.assertEqual(butlerLocation.getCppType(), "PropertySet") 

153 self.assertEqual(butlerLocation.getStorageName(), "YamlStorage") 

154 self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path]) 

155 for k, v in dataId.items(): 

156 self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v, 

157 msg="Failed for key={}".format(k)) 

158 

159 def test_keys(self): 

160 self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys) 

161 

162 def test_get_dataset_types(self): 

163 if not self.mapper_data.test_config_metadata: 

164 self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name)) 

165 someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata']) 

166 self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys)) 

167 

168 def test_get_keys_raw(self): 

169 for level, expect in self.mapper_data.raw_levels: 

170 result = self.mapper.getKeys("raw", level) 

171 self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level)) 

172 

173 def test_get_default_level(self): 

174 self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level) 

175 

176 def _test_map(self, butlerLocation, dataId): 

177 self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type) 

178 self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type) 

179 self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name) 

180 locationList = butlerLocation.getLocations() 

181 self.assertEqual(len(locationList), 1) 

182 fileName = os.path.basename(locationList[0]) 

183 self.assertEqual(fileName, self.mapper_data.raw_filename) 

184 for k, v in dataId.items(): 

185 self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v, 

186 msg="Failed for key={}".format(k)) 

187 

188 def test_map(self): 

189 dataId = self.dataIds['raw'] 

190 location = self.mapper.map_raw(dataId) 

191 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite): 

192 self._test_map(location, dataId) 

193 else: 

194 self.log.warning("""ButlerComposite datasets are not tested for mapper functions. Though 

195ButlerComposites duck type as ButlerLocations in some ways, they do not 

196share enough methods to be usefully tested by the same function. Note 

197there are tests of the objects in the package in which they are implemented.""") 

198 # This should be the same as above. Testing that both the generic 

199 # and specific interface work for mapping the raw. 

200 location = self.mapper.map("raw", dataId) 

201 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite): 

202 self._test_map(location, dataId) 

203 

204 def test_query_metadata(self): 

205 """ 

206 Test expansion of incomplete information of the available data in this 

207 obs package's testdata repo. 

208 """ 

209 for query, expect in self.mapper_data.queryMetadata: 

210 # queryMetadata returns tuples of available items of the 2nd 

211 # parameter. 

212 result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query) 

213 self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query)) 

214 

215 def test_can_standardize(self): 

216 self.assertTrue(self.mapper.canStandardize("raw")) 

217 self.assertFalse(self.mapper.canStandardize("camera")) 

218 if not self.mapper_data.test_config_metadata: 

219 self.assertFalse(self.mapper.canStandardize("processCcd_config")) 

220 self.assertFalse(self.mapper.canStandardize("processCcd_metadata")) 

221 

222 def _test_validate(self, dataId): 

223 self.assertEqual(self.mapper.validate(dataId), dataId) 

224 

225 def test_validate(self): 

226 self._test_validate({'visit': 1, 'filter': 'g'}) 

227 self._test_validate({'visit': 2, 'filter': 'r'}) 

228 self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4}) 

229 # NOTE: when DM-7909 is completed, add assertRaises test here. 

230 # visit must be an integers