lsst.obs.base  14.0-22-gc48c03f+2
mapper_tests.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from future.utils import with_metaclass
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 import abc
25 import os
26 
27 import lsst.afw.geom
28 import lsst.utils.tests
29 import lsst.daf.persistence
30 import lsst.afw.image
31 import collections
32 
33 
34 class MapperTests(with_metaclass(abc.ABCMeta)):
35  """
36  Generic tests of obs_* package mapper functionality.
37 
38  In the subclasses's setUp():
39  * Call setUp_mapper() to fill in required parameters.
40  """
41 
42  def setUp_mapper(self,
43  output=None,
44  path_to_raw=None,
45  keys=None,
46  query_format=None,
47  queryMetadata=None,
48  metadata_output_path=None,
49  map_python_type=None,
50  map_cpp_type=None,
51  map_storage_name=None,
52  raw_filename=None,
53  default_level=None,
54  raw_levels=None,
55  ):
56  """
57  Set up the necessary variables for mapper tests.
58 
59  Parameters
60  ----------
61 
62  output : `str`
63  full path to output repository (can be the same as data_dir input repository)
64  path_to_raw : `str`
65  full path to the raw file referenced by dataIds['raw']
66  keys : `set`
67  dictionary keys that this mapper should contain
68  query_format : `list`
69  format list for the results portion of queryMetadata
70  queryMetadata : `tuple` of (`dict`, `tuple`)
71  dataIds and the results of calling them in queryMetadata
72  metadata_output_path : `str`
73  path to metadata output associated with dataIds['raw']
74  map_python_type : `str`
75  full python type specification returned by the mapper for dataIds['raw']
76  map_cpp_type : `str`
77  C++ type specification returned by the mapper for dataIds['raw']
78  map_storage_name : `str`
79  butler name for the storage type dataIds['raw']
80  raw_filename : `str`
81  Name of the raw files returned by the mapper for dataIds['raw']
82  default_level : `str`
83  value returned from mapper.getDefaultLevel
84  raw_levels : `tuple` of (`str`, `set` of `str`)
85  (level, expect) level and expected mapper return for mapper.getKeys('raw', level)
86  """
87  fields = ['output',
88  'path_to_raw',
89  'keys',
90  'query_format',
91  'queryMetadata',
92  'metadata_output_path',
93  'map_python_type',
94  'map_cpp_type',
95  'map_storage_name',
96  'raw_filename',
97  'default_level',
98  'raw_levels',
99  ]
100  MapperData = collections.namedtuple("MapperData", fields)
101  self.mapper_data = MapperData(output=output,
102  path_to_raw=path_to_raw,
103  keys=keys,
104  query_format=query_format,
105  queryMetadata=queryMetadata,
106  metadata_output_path=metadata_output_path,
107  map_python_type=map_python_type,
108  map_cpp_type=map_cpp_type,
109  map_storage_name=map_storage_name,
110  raw_filename=raw_filename,
111  default_level=default_level,
112  raw_levels=raw_levels,
113  )
114 
116  dataId = self.dataIds['raw']
117  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
118  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
119  self.assertEqual(butlerLocation.getCppType(), "Config")
120  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
121  processCcd_path = os.path.join("config", "processCcd.py")
122  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
123  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
124  for k, v in dataId.items():
125  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
126 
128  dataId = self.dataIds['raw']
129  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
130  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
131  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
132  self.assertEqual(butlerLocation.getStorageName(), "BoostStorage")
133  self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
134  for k, v in dataId.items():
135  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
136 
137  def test_keys(self):
138  self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
139 
141  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
142  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
143 
144  def test_get_keys_raw(self):
145  for level, expect in self.mapper_data.raw_levels:
146  result = self.mapper.getKeys("raw", level)
147  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
148 
150  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
151 
152  def _test_map(self, butlerLocation, dataId):
153  self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
154  self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
155  self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
156  locationList = butlerLocation.getLocations()
157  self.assertEqual(len(locationList), 1)
158  fileName = os.path.basename(locationList[0])
159  self.assertEqual(fileName, self.mapper_data.raw_filename)
160  for k, v in dataId.items():
161  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
162 
163  def test_map(self):
164  dataId = self.dataIds['raw']
165  self._test_map(self.mapper.map_raw(dataId), dataId)
166  self._test_map(self.mapper.map("raw", dataId), dataId)
167 
169  """
170  Test expansion of incomplete information of the available data in this
171  obs package's testdata repo.
172  """
173  for query, expect in self.mapper_data.queryMetadata:
174  # queryMetadata returns tuples of available items of the 2nd parameter.
175  result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
176  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
177 
179  self.assertTrue(self.mapper.canStandardize("raw"))
180  self.assertFalse(self.mapper.canStandardize("camera"))
181  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
182  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
183 
185  rawImage = lsst.afw.image.DecoratedImageU(self.mapper_data.path_to_raw)
186  stdImage = self.mapper.standardize("raw", rawImage, self.dataIds['raw'])
187  self.assertIsInstance(stdImage, lsst.afw.image.ExposureU)
188 
189  def _test_validate(self, dataId):
190  self.assertEqual(self.mapper.validate(dataId), dataId)
191 
192  def test_validate(self):
193  self._test_validate({'visit': 1, 'filter': 'g'})
194  self._test_validate({'visit': 2, 'filter': 'r'})
195  self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
196  # NOTE: when DM-7909 is completed, add assertRaises test here.
197  # visit must be an integers
def _test_map(self, butlerLocation, dataId)
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None)
Definition: mapper_tests.py:55