Coverage for python/lsst/obs/base/mapper_tests.py: 23%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import abc
23import collections
24import inspect
25import os
27import lsst.afw.geom
28import lsst.afw.image
29import lsst.daf.persistence
30import lsst.utils.tests
32__all__ = ["MapperTests"]
35class MapperTests(metaclass=abc.ABCMeta):
36 """
37 Generic tests of obs_* package mapper functionality.
39 In the subclasses's setUp():
40 * Call setUp_mapper() to fill in required parameters.
41 """
43 def setUp_mapper(
44 self,
45 output=None,
46 path_to_raw=None,
47 keys=None,
48 query_format=None,
49 queryMetadata=None,
50 metadata_output_path=None,
51 map_python_type=None,
52 map_python_std_type=None,
53 map_cpp_type=None,
54 map_storage_name=None,
55 raw_filename=None,
56 default_level=None,
57 raw_levels=None,
58 test_config_metadata=True,
59 ):
60 """
61 Set up the necessary variables for mapper tests.
63 Parameters
64 ----------
66 output : `str`
67 full path to output repository (can be the same as data_dir input
68 repository)
69 path_to_raw : `str`
70 full path to the raw file referenced by dataIds['raw']
71 keys : `set`
72 dictionary keys that this mapper should contain
73 query_format : `list`
74 format list for the results portion of queryMetadata
75 queryMetadata : `tuple` of (`dict`, `tuple`)
76 dataIds and the results of calling them in queryMetadata
77 metadata_output_path : `str`
78 path to metadata output associated with dataIds['raw']
79 map_python_type : `type`
80 full python type specification returned by the mapper for
81 dataIds['raw']
82 map_python_std_type : `type`
83 full python type specification returned by the mapper for
84 dataIds['raw'] after standardization
85 map_cpp_type : `str`
86 C++ type specification returned by the mapper for dataIds['raw']
87 map_storage_name : `str`
88 butler name for the storage type dataIds['raw']
89 raw_filename : `str`
90 Name of the raw files returned by the mapper for dataIds['raw']
91 default_level : `str`
92 value returned from mapper.getDefaultLevel
93 raw_levels : `tuple` of (`str`, `set` of `str`)
94 (level, expect) level and expected mapper return for
95 mapper.getKeys('raw', level)
96 test_config_metadata : `bool`
97 Test persisted config and metadata? These tests may not be
98 appropriate for test stand data. Defaults to True.
99 """
100 fields = [
101 "output",
102 "path_to_raw",
103 "keys",
104 "query_format",
105 "queryMetadata",
106 "metadata_output_path",
107 "map_python_type",
108 "map_python_std_type",
109 "map_cpp_type",
110 "map_storage_name",
111 "raw_filename",
112 "default_level",
113 "raw_levels",
114 "test_config_metadata",
115 ]
116 MapperData = collections.namedtuple("MapperData", fields)
117 self.mapper_data = MapperData(
118 output=output,
119 path_to_raw=path_to_raw,
120 keys=keys,
121 query_format=query_format,
122 queryMetadata=queryMetadata,
123 metadata_output_path=metadata_output_path,
124 map_python_type=map_python_type,
125 map_python_std_type=map_python_std_type,
126 map_cpp_type=map_cpp_type,
127 map_storage_name=map_storage_name,
128 raw_filename=raw_filename,
129 default_level=default_level,
130 raw_levels=raw_levels,
131 test_config_metadata=test_config_metadata,
132 )
134 def test_map_config_data(self):
135 if not self.mapper_data.test_config_metadata:
136 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
137 dataId = self.dataIds["raw"]
138 butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
139 self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
140 self.assertEqual(butlerLocation.getCppType(), "Config")
141 self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
142 processCcd_path = os.path.join("config", "processCcd.py")
143 self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
144 self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
145 for k, v in dataId.items():
146 self.assertEqual(
147 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
148 )
150 def test_map_metadata_data(self):
151 if not self.mapper_data.test_config_metadata:
152 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
153 dataId = self.dataIds["raw"]
154 butlerLocation = self.mapper.map_processCcd_metadata(dataId)
155 self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
156 self.assertEqual(butlerLocation.getCppType(), "PropertySet")
157 self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
158 self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
159 for k, v in dataId.items():
160 self.assertEqual(
161 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
162 )
164 def test_keys(self):
165 self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
167 def test_get_dataset_types(self):
168 if not self.mapper_data.test_config_metadata:
169 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
170 someKeys = set(["raw", "processCcd_config", "processCcd_metadata"])
171 self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
173 def test_get_keys_raw(self):
174 for level, expect in self.mapper_data.raw_levels:
175 result = self.mapper.getKeys("raw", level)
176 self.assertEqual(set(result), expect, msg="Failed for level={}".format(level))
178 def test_get_default_level(self):
179 self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
181 def _test_map(self, butlerLocation, dataId):
182 self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
183 self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
184 self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
185 locationList = butlerLocation.getLocations()
186 self.assertEqual(len(locationList), 1)
187 fileName = os.path.basename(locationList[0])
188 self.assertEqual(fileName, self.mapper_data.raw_filename)
189 for k, v in dataId.items():
190 self.assertEqual(
191 butlerLocation.getAdditionalData().getScalar(k), v, msg="Failed for key={}".format(k)
192 )
194 def test_map(self):
195 dataId = self.dataIds["raw"]
196 location = self.mapper.map_raw(dataId)
197 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
198 self._test_map(location, dataId)
199 else:
200 self.log.warning(
201 """ButlerComposite datasets are not tested for mapper functions. Though
202ButlerComposites duck type as ButlerLocations in some ways, they do not
203share enough methods to be usefully tested by the same function. Note
204there are tests of the objects in the package in which they are implemented."""
205 )
206 # This should be the same as above. Testing that both the generic
207 # and specific interface work for mapping the raw.
208 location = self.mapper.map("raw", dataId)
209 if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
210 self._test_map(location, dataId)
212 def test_query_metadata(self):
213 """
214 Test expansion of incomplete information of the available data in this
215 obs package's testdata repo.
216 """
217 for query, expect in self.mapper_data.queryMetadata:
218 # queryMetadata returns tuples of available items of the 2nd
219 # parameter.
220 result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
221 self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
223 def test_can_standardize(self):
224 self.assertTrue(self.mapper.canStandardize("raw"))
225 self.assertFalse(self.mapper.canStandardize("camera"))
226 if not self.mapper_data.test_config_metadata:
227 self.assertFalse(self.mapper.canStandardize("processCcd_config"))
228 self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
230 def _test_validate(self, dataId):
231 self.assertEqual(self.mapper.validate(dataId), dataId)
233 def test_validate(self):
234 self._test_validate({"visit": 1, "filter": "g"})
235 self._test_validate({"visit": 2, "filter": "r"})
236 self._test_validate({"visit": 3, "filter": "g", "tract": 4})
237 # NOTE: when DM-7909 is completed, add assertRaises test here.
238 # visit must be an integers