Coverage for python/lsst/obs/base/mapper_tests.py: 19%
101 statements
« prev ^ index » next coverage.py v6.5.0, created at 2024-01-25 04:31 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2024-01-25 04:31 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import abc
23import collections
24import inspect
25import os
27import lsst.afw.geom
28import lsst.afw.image
29import lsst.daf.persistence
30import lsst.utils.tests
32__all__ = ["MapperTests"]
35class MapperTests(metaclass=abc.ABCMeta):
36 """
37 Generic tests of obs_* package mapper functionality.
39 In the subclasses's setUp():
40 * Call setUp_mapper() to fill in required parameters.
41 """
43 def setUp_mapper(
44 self,
45 output=None,
46 path_to_raw=None,
47 keys=None,
48 query_format=None,
49 queryMetadata=None,
50 metadata_output_path=None,
51 map_python_type=None,
52 map_python_std_type=None,
53 map_cpp_type=None,
54 map_storage_name=None,
55 raw_filename=None,
56 default_level=None,
57 raw_levels=None,
58 test_config_metadata=True,
59 ):
60 """
61 Set up the necessary variables for mapper tests.
63 Parameters
64 ----------
66 output : `str`
67 full path to output repository (can be the same as data_dir input
68 repository)
69 path_to_raw : `str`
70 full path to the raw file referenced by dataIds['raw']
71 keys : `set`
72 dictionary keys that this mapper should contain
73 query_format : `list`
74 format list for the results portion of queryMetadata
75 queryMetadata : `tuple` of (`dict`, `tuple`)
76 dataIds and the results of calling them in queryMetadata
77 metadata_output_path : `str`
78 path to metadata output associated with dataIds['raw']
79 map_python_type : `type`
80 full python type specification returned by the mapper for
81 dataIds['raw']
82 map_python_std_type : `type`
83 full python type specification returned by the mapper for
84 dataIds['raw'] after standardization
85 map_cpp_type : `str`
86 C++ type specification returned by the mapper for dataIds['raw']
87 map_storage_name : `str`
88 butler name for the storage type dataIds['raw']
89 raw_filename : `str`
90 Name of the raw files returned by the mapper for dataIds['raw']
91 default_level : `str`
92 value returned from mapper.getDefaultLevel
93 raw_levels : `tuple` of (`str`, `set` of `str`)
94 (level, expect) level and expected mapper return for
95 mapper.getKeys('raw', level)
96 test_config_metadata : `bool`
97 Test persisted config and metadata? These tests may not be
98 appropriate for test stand data. Defaults to True.
99 """
100 fields = [
101 "output",
102 "path_to_raw",
103 "keys",
104 "query_format",
105 "queryMetadata",
106 "metadata_output_path",
107 "map_python_type",
108 "map_python_std_type",
109 "map_cpp_type",
110 "map_storage_name",
111 "raw_filename",
112 "default_level",
113 "raw_levels",
114 "test_config_metadata",
115 ]
116 MapperData = collections.namedtuple("MapperData", fields)
117 self.mapper_data = MapperData(
118 output=output,
119 path_to_raw=path_to_raw,
120 keys=keys,
121 query_format=query_format,
122 queryMetadata=queryMetadata,
123 metadata_output_path=metadata_output_path,
124 map_python_type=map_python_type,
125 map_python_std_type=map_python_std_type,
126 map_cpp_type=map_cpp_type,
127 map_storage_name=map_storage_name,
128 raw_filename=raw_filename,
129 default_level=default_level,
130 raw_levels=raw_levels,
131 test_config_metadata=test_config_metadata,
132 )
134 def _check_mapper(self):
135 if self.mapper is None:
136 self.skipTest("No mapper defined. Likely a gen3 test environment.")
138 def test_map_config_data(self):
139 self._check_mapper()
140 if not self.mapper_data.test_config_metadata:
141 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
142 dataId = self.dataIds["raw"]
143 butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
144 self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
145 self.assertEqual(butlerLocation.getCppType(), "Config")
146 self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
147 processCcd_path = os.path.join("config", "processCcd.py")
148 self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
149 self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
150 for k, v in dataId.items():
151 self.assertEqual(
152 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
153 )
155 def test_map_metadata_data(self):
156 self._check_mapper()
157 if not self.mapper_data.test_config_metadata:
158 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
159 dataId = self.dataIds["raw"]
160 butlerLocation = self.mapper.map_processCcd_metadata(dataId)
161 self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
162 self.assertEqual(butlerLocation.getCppType(), "PropertySet")
163 self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
164 self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
165 for k, v in dataId.items():
166 self.assertEqual(
167 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
168 )
170 def test_keys(self):
171 self._check_mapper()
172 self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
174 def test_get_dataset_types(self):
175 self._check_mapper()
176 if not self.mapper_data.test_config_metadata:
177 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
178 someKeys = set(["raw", "processCcd_config", "processCcd_metadata"])
179 self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
181 def test_get_keys_raw(self):
182 self._check_mapper()
183 for level, expect in self.mapper_data.raw_levels:
184 result = self.mapper.getKeys("raw", level)
185 self.assertEqual(set(result), expect, msg="Failed for level={}".format(level))
187 def test_get_default_level(self):
188 self._check_mapper()
189 self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
191 def _test_map(self, butlerLocation, dataId):
192 self._check_mapper()
193 self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
194 self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
195 self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
196 locationList = butlerLocation.getLocations()
197 self.assertEqual(len(locationList), 1)
198 fileName = os.path.basename(locationList[0])
199 self.assertEqual(fileName, self.mapper_data.raw_filename)
200 for k, v in dataId.items():
201 self.assertEqual(
202 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
203 )
205 def test_map(self):
206 self._check_mapper()
207 dataId = self.dataIds["raw"]
208 location = self.mapper.map_raw(dataId)
209 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
210 self._test_map(location, dataId)
211 else:
212 self.log.warning(
213 """ButlerComposite datasets are not tested for mapper functions. Though
214ButlerComposites duck type as ButlerLocations in some ways, they do not
215share enough methods to be usefully tested by the same function. Note
216there are tests of the objects in the package in which they are implemented."""
217 )
218 # This should be the same as above. Testing that both the generic
219 # and specific interface work for mapping the raw.
220 location = self.mapper.map("raw", dataId)
221 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
222 self._test_map(location, dataId)
224 def test_query_metadata(self):
225 """
226 Test expansion of incomplete information of the available data in this
227 obs package's testdata repo.
228 """
229 self._check_mapper()
230 for query, expect in self.mapper_data.queryMetadata:
231 # queryMetadata returns tuples of available items of the 2nd
232 # parameter.
233 result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
234 self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
236 def test_can_standardize(self):
237 self._check_mapper()
238 self.assertTrue(self.mapper.canStandardize("raw"))
239 self.assertFalse(self.mapper.canStandardize("camera"))
240 if not self.mapper_data.test_config_metadata:
241 self.assertFalse(self.mapper.canStandardize("processCcd_config"))
242 self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
244 def _test_validate(self, dataId):
245 self._check_mapper()
246 self.assertEqual(self.mapper.validate(dataId), dataId)
248 def test_validate(self):
249 self._check_mapper()
250 self._test_validate({"visit": 1, "filter": "g"})
251 self._test_validate({"visit": 2, "filter": "r"})
252 self._test_validate({"visit": 3, "filter": "g", "tract": 4})
253 # NOTE: when DM-7909 is completed, add assertRaises test here.
254 # visit must be an integers