lsst.obs.base  15.0-8-g0cd0e28
mapper_tests.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from future.utils import with_metaclass
3 #
4 # LSST Data Management System
5 # Copyright 2016 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 import abc
25 import os
26 
27 import lsst.afw.geom
28 import lsst.utils.tests
29 import lsst.daf.persistence
30 import lsst.afw.image
31 import collections
32 
33 __all__ = ["MapperTests"]
34 
35 
36 class MapperTests(with_metaclass(abc.ABCMeta)):
37  """
38  Generic tests of obs_* package mapper functionality.
39 
40  In the subclasses's setUp():
41  * Call setUp_mapper() to fill in required parameters.
42  """
43 
44  def setUp_mapper(self,
45  output=None,
46  path_to_raw=None,
47  keys=None,
48  query_format=None,
49  queryMetadata=None,
50  metadata_output_path=None,
51  map_python_type=None,
52  map_cpp_type=None,
53  map_storage_name=None,
54  raw_filename=None,
55  default_level=None,
56  raw_levels=None,
57  ):
58  """
59  Set up the necessary variables for mapper tests.
60 
61  Parameters
62  ----------
63 
64  output : `str`
65  full path to output repository (can be the same as data_dir input repository)
66  path_to_raw : `str`
67  full path to the raw file referenced by dataIds['raw']
68  keys : `set`
69  dictionary keys that this mapper should contain
70  query_format : `list`
71  format list for the results portion of queryMetadata
72  queryMetadata : `tuple` of (`dict`, `tuple`)
73  dataIds and the results of calling them in queryMetadata
74  metadata_output_path : `str`
75  path to metadata output associated with dataIds['raw']
76  map_python_type : `str`
77  full python type specification returned by the mapper for dataIds['raw']
78  map_cpp_type : `str`
79  C++ type specification returned by the mapper for dataIds['raw']
80  map_storage_name : `str`
81  butler name for the storage type dataIds['raw']
82  raw_filename : `str`
83  Name of the raw files returned by the mapper for dataIds['raw']
84  default_level : `str`
85  value returned from mapper.getDefaultLevel
86  raw_levels : `tuple` of (`str`, `set` of `str`)
87  (level, expect) level and expected mapper return for mapper.getKeys('raw', level)
88  """
89  fields = ['output',
90  'path_to_raw',
91  'keys',
92  'query_format',
93  'queryMetadata',
94  'metadata_output_path',
95  'map_python_type',
96  'map_cpp_type',
97  'map_storage_name',
98  'raw_filename',
99  'default_level',
100  'raw_levels',
101  ]
102  MapperData = collections.namedtuple("MapperData", fields)
103  self.mapper_data = MapperData(output=output,
104  path_to_raw=path_to_raw,
105  keys=keys,
106  query_format=query_format,
107  queryMetadata=queryMetadata,
108  metadata_output_path=metadata_output_path,
109  map_python_type=map_python_type,
110  map_cpp_type=map_cpp_type,
111  map_storage_name=map_storage_name,
112  raw_filename=raw_filename,
113  default_level=default_level,
114  raw_levels=raw_levels,
115  )
116 
118  dataId = self.dataIds['raw']
119  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
120  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
121  self.assertEqual(butlerLocation.getCppType(), "Config")
122  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
123  processCcd_path = os.path.join("config", "processCcd.py")
124  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
125  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
126  for k, v in dataId.items():
127  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
128 
130  dataId = self.dataIds['raw']
131  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
132  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
133  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
134  self.assertEqual(butlerLocation.getStorageName(), "BoostStorage")
135  self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
136  for k, v in dataId.items():
137  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
138 
139  def test_keys(self):
140  self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
141 
143  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
144  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
145 
146  def test_get_keys_raw(self):
147  for level, expect in self.mapper_data.raw_levels:
148  result = self.mapper.getKeys("raw", level)
149  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
150 
152  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
153 
154  def _test_map(self, butlerLocation, dataId):
155  self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
156  self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
157  self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
158  locationList = butlerLocation.getLocations()
159  self.assertEqual(len(locationList), 1)
160  fileName = os.path.basename(locationList[0])
161  self.assertEqual(fileName, self.mapper_data.raw_filename)
162  for k, v in dataId.items():
163  self.assertEqual(butlerLocation.getAdditionalData().get(k), v, msg="Failed for key={}".format(k))
164 
165  def test_map(self):
166  dataId = self.dataIds['raw']
167  self._test_map(self.mapper.map_raw(dataId), dataId)
168  self._test_map(self.mapper.map("raw", dataId), dataId)
169 
171  """
172  Test expansion of incomplete information of the available data in this
173  obs package's testdata repo.
174  """
175  for query, expect in self.mapper_data.queryMetadata:
176  # queryMetadata returns tuples of available items of the 2nd parameter.
177  result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
178  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
179 
181  self.assertTrue(self.mapper.canStandardize("raw"))
182  self.assertFalse(self.mapper.canStandardize("camera"))
183  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
184  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
185 
187  rawImage = lsst.afw.image.DecoratedImageU(self.mapper_data.path_to_raw)
188  stdImage = self.mapper.standardize("raw", rawImage, self.dataIds['raw'])
189  self.assertIsInstance(stdImage, lsst.afw.image.ExposureU)
190 
191  def _test_validate(self, dataId):
192  self.assertEqual(self.mapper.validate(dataId), dataId)
193 
194  def test_validate(self):
195  self._test_validate({'visit': 1, 'filter': 'g'})
196  self._test_validate({'visit': 2, 'filter': 'r'})
197  self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
198  # NOTE: when DM-7909 is completed, add assertRaises test here.
199  # visit must be an integers
def _test_map(self, butlerLocation, dataId)
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None)
Definition: mapper_tests.py:57