lsst.obs.base  16.0-14-g71e547a+9
mapper_tests.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import abc
23 import os
24 
25 import lsst.afw.geom
26 import lsst.utils.tests
27 import lsst.daf.persistence
28 import lsst.afw.image
29 import collections
30 
31 __all__ = ["MapperTests"]
32 
33 
34 class MapperTests(metaclass=abc.ABCMeta):
35  """
36  Generic tests of obs_* package mapper functionality.
37 
38  In the subclasses's setUp():
39  * Call setUp_mapper() to fill in required parameters.
40  """
41 
42  def setUp_mapper(self,
43  output=None,
44  path_to_raw=None,
45  keys=None,
46  query_format=None,
47  queryMetadata=None,
48  metadata_output_path=None,
49  map_python_type=None,
50  map_cpp_type=None,
51  map_storage_name=None,
52  raw_filename=None,
53  default_level=None,
54  raw_levels=None,
55  ):
56  """
57  Set up the necessary variables for mapper tests.
58 
59  Parameters
60  ----------
61 
62  output : `str`
63  full path to output repository (can be the same as data_dir input repository)
64  path_to_raw : `str`
65  full path to the raw file referenced by dataIds['raw']
66  keys : `set`
67  dictionary keys that this mapper should contain
68  query_format : `list`
69  format list for the results portion of queryMetadata
70  queryMetadata : `tuple` of (`dict`, `tuple`)
71  dataIds and the results of calling them in queryMetadata
72  metadata_output_path : `str`
73  path to metadata output associated with dataIds['raw']
74  map_python_type : `str`
75  full python type specification returned by the mapper for dataIds['raw']
76  map_cpp_type : `str`
77  C++ type specification returned by the mapper for dataIds['raw']
78  map_storage_name : `str`
79  butler name for the storage type dataIds['raw']
80  raw_filename : `str`
81  Name of the raw files returned by the mapper for dataIds['raw']
82  default_level : `str`
83  value returned from mapper.getDefaultLevel
84  raw_levels : `tuple` of (`str`, `set` of `str`)
85  (level, expect) level and expected mapper return for mapper.getKeys('raw', level)
86  """
87  fields = ['output',
88  'path_to_raw',
89  'keys',
90  'query_format',
91  'queryMetadata',
92  'metadata_output_path',
93  'map_python_type',
94  'map_cpp_type',
95  'map_storage_name',
96  'raw_filename',
97  'default_level',
98  'raw_levels',
99  ]
100  MapperData = collections.namedtuple("MapperData", fields)
101  self.mapper_data = MapperData(output=output,
102  path_to_raw=path_to_raw,
103  keys=keys,
104  query_format=query_format,
105  queryMetadata=queryMetadata,
106  metadata_output_path=metadata_output_path,
107  map_python_type=map_python_type,
108  map_cpp_type=map_cpp_type,
109  map_storage_name=map_storage_name,
110  raw_filename=raw_filename,
111  default_level=default_level,
112  raw_levels=raw_levels,
113  )
114 
116  dataId = self.dataIds['raw']
117  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
118  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
119  self.assertEqual(butlerLocation.getCppType(), "Config")
120  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
121  processCcd_path = os.path.join("config", "processCcd.py")
122  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
123  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
124  for k, v in dataId.items():
125  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
126  msg="Failed for key={}".format(k))
127 
129  dataId = self.dataIds['raw']
130  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
131  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
132  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
133  self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
134  self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
135  for k, v in dataId.items():
136  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
137  msg="Failed for key={}".format(k))
138 
139  def test_keys(self):
140  self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
141 
143  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
144  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
145 
146  def test_get_keys_raw(self):
147  for level, expect in self.mapper_data.raw_levels:
148  result = self.mapper.getKeys("raw", level)
149  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
150 
152  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
153 
154  def _test_map(self, butlerLocation, dataId):
155  self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
156  self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
157  self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
158  locationList = butlerLocation.getLocations()
159  self.assertEqual(len(locationList), 1)
160  fileName = os.path.basename(locationList[0])
161  self.assertEqual(fileName, self.mapper_data.raw_filename)
162  for k, v in dataId.items():
163  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
164  msg="Failed for key={}".format(k))
165 
166  def test_map(self):
167  dataId = self.dataIds['raw']
168  self._test_map(self.mapper.map_raw(dataId), dataId)
169  self._test_map(self.mapper.map("raw", dataId), dataId)
170 
172  """
173  Test expansion of incomplete information of the available data in this
174  obs package's testdata repo.
175  """
176  for query, expect in self.mapper_data.queryMetadata:
177  # queryMetadata returns tuples of available items of the 2nd parameter.
178  result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
179  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
180 
182  self.assertTrue(self.mapper.canStandardize("raw"))
183  self.assertFalse(self.mapper.canStandardize("camera"))
184  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
185  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
186 
188  rawImage = lsst.afw.image.DecoratedImageU(self.mapper_data.path_to_raw)
189  stdImage = self.mapper.standardize("raw", rawImage, self.dataIds['raw'])
190  self.assertIsInstance(stdImage, lsst.afw.image.ExposureU)
191 
192  def _test_validate(self, dataId):
193  self.assertEqual(self.mapper.validate(dataId), dataId)
194 
195  def test_validate(self):
196  self._test_validate({'visit': 1, 'filter': 'g'})
197  self._test_validate({'visit': 2, 'filter': 'r'})
198  self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
199  # NOTE: when DM-7909 is completed, add assertRaises test here.
200  # visit must be an integers
def _test_map(self, butlerLocation, dataId)
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None)
Definition: mapper_tests.py:55