Coverage for python/lsst/obs/base/butler_tests.py: 15%
104 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 18:43 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 18:43 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import abc
23import collections
24import inspect
25import unittest
27from lsst.daf.butler import Butler
28from lsst.daf.butler.registry import DataIdValueError
30__all__ = ["ButlerGetTests"]
33class ButlerGetTests(metaclass=abc.ABCMeta):
34 """Tests of obs_* Butler get() functionality.
36 In the subclasses's setUp():
37 * Call setUp_butler_get() to fill in required parameters.
38 """
40 def setUp_butler_get(
41 self,
42 ccdExposureId_bits=None,
43 exposureIds=None,
44 filters=None,
45 exptimes=None,
46 detectorIds=None,
47 detector_names=None,
48 detector_serials=None,
49 dimensions=None,
50 sky_origin=None,
51 raw_subsets=None,
52 good_detectorIds=None,
53 bad_detectorIds=None,
54 linearizer_type=None,
55 raw_header_wcs=None,
56 ):
57 """
58 Set up the necessary variables for butlerGet tests.
60 All "exposure name" entries below should correspond to an entry in
61 self.dataIds.
63 Parameters
64 ----------
66 ccdExposureId_bits : `int`
67 expected value of ccdExposureId_bits
68 exposureIds : `dict`
69 dict of exposure name : ccdExposureId (the number as returned by
70 the butler)
71 filters : `dict`
72 dict of exposure name : filter name
73 exptimes : `dict`
74 dict of exposure name : exposure time
75 detector_names : `dict`
76 dict of exposure name : detector name
77 detectorIds : `dict`
78 dict of exposure name : detectorId
79 detector_serials : `dict`
80 dict of exposure name : detector serial
81 dimensions : `dict`
82 dict of exposure name : dimensions (as a geom.Extent2I)
83 sky_origin : `tuple` of `float`
84 Longitude, Latitude of 'raw' exposure
85 raw_subsets : `tuple` of (kwargs, `int`)
86 keyword args and expected number of subsets for
87 ``butler.subset('raw', **kwargs)``
88 good_detectorIds : `list` of `int`
89 list of valid ccd numbers
90 bad_detectorIds : `list` of `int`
91 list of invalid ccd numbers
92 linearizer_type : `dict`
93 dict of detectorId (usually `int`): LinearizerType
94 (e.g. lsst.ip.isr.LinearizeLookupTable.LinearityType),
95 or unittest.SkipTest to skip all linearizer tests.
96 raw_header_wcs : `lsst.afw.geom.SkyWcs`
97 The SkyWcs object that should be returned by
98 ``butler.get("raw_header_wcs", dataId=self.dataIds["raw"])``
99 """
101 fields = [
102 "ccdExposureId_bits",
103 "exposureIds",
104 "filters",
105 "exptimes",
106 "detector_names",
107 "detectorIds",
108 "detector_serials",
109 "dimensions",
110 "sky_origin",
111 "raw_subsets",
112 "good_detectorIds",
113 "bad_detectorIds",
114 "linearizer_type",
115 "raw_header_wcs",
116 ]
117 ButlerGet = collections.namedtuple("ButlerGetData", fields)
119 self.butler_get_data = ButlerGet(
120 ccdExposureId_bits=ccdExposureId_bits,
121 exposureIds=exposureIds,
122 filters=filters,
123 exptimes=exptimes,
124 detectorIds=detectorIds,
125 detector_names=detector_names,
126 detector_serials=detector_serials,
127 dimensions=dimensions,
128 sky_origin=sky_origin,
129 raw_subsets=raw_subsets,
130 good_detectorIds=good_detectorIds,
131 bad_detectorIds=bad_detectorIds,
132 linearizer_type=linearizer_type,
133 raw_header_wcs=raw_header_wcs,
134 )
136 def _require_gen2(self):
137 if isinstance(self.butler, Butler):
138 self.skipTest("This test requires daf_persistence Butler, not daf_butler Butler.")
140 def _is_gen3(self):
141 if isinstance(self.butler, Butler):
142 return True
143 return False
145 def test_exposureId_bits(self):
146 self._require_gen2()
147 bits = self.butler.get("ccdExposureId_bits")
148 self.assertEqual(bits, self.butler_get_data.ccdExposureId_bits)
150 def _test_exposure(self, name):
151 if self.dataIds[name] is unittest.SkipTest:
152 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
153 exp = self.butler.get(name, self.dataIds[name])
155 md_component = ".metadata" if self._is_gen3() else "_md"
156 exp_md = self.butler.get(name + md_component, self.dataIds[name])
157 self.assertEqual(type(exp_md), type(exp.getMetadata()))
159 self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name])
160 detector = exp.detector
161 # Some calibration files are missing the detector.
162 if detector:
163 self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name])
164 self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name])
165 self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name])
166 # obs_test does not have physical filters, so include a fallback
167 exposureFilter = exp.getFilter()
168 if exposureFilter:
169 if exposureFilter.hasPhysicalLabel():
170 filterName = exposureFilter.physicalLabel
171 else:
172 filterName = exposureFilter.bandLabel
173 else:
174 filterName = "_unknown_"
175 self.assertEqual(filterName, self.butler_get_data.filters[name])
176 if not self._is_gen3():
177 exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name])
178 self.assertEqual(exposureId, self.butler_get_data.exposureIds[name])
179 self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name])
180 return exp
182 def test_raw(self):
183 exp = self._test_exposure("raw")
184 # We only test the existence of WCS in the raw files, since it's only
185 # well-defined for raw, and other exposure types could have or not
186 # have a WCS depending on various implementation details.
187 # Even for raw, there are data that do not have a WCS, e.g. teststand
188 # data
189 if self.butler_get_data.sky_origin is not unittest.SkipTest:
190 self.assertEqual(exp.hasWcs(), True)
191 origin = exp.getWcs().getSkyOrigin()
192 self.assertAlmostEqual(origin.getLongitude().asDegrees(), self.butler_get_data.sky_origin[0])
193 self.assertAlmostEqual(origin.getLatitude().asDegrees(), self.butler_get_data.sky_origin[1])
195 def test_bias(self):
196 self._test_exposure("bias")
198 def test_dark(self):
199 self._test_exposure("dark")
201 def test_flat(self):
202 self._test_exposure("flat")
204 def test_raw_header_wcs(self):
205 """Test that `raw_header_wcs` returns the unmodified header of the raw
206 image."""
207 if self.butler_get_data.raw_header_wcs is None:
208 self.skipTest("Skipping raw header WCS test since no reference provided.")
209 # Gen3 will not understand this at the moment (DM-35031).
210 wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"])
211 self.assertEqual(wcs, self.butler_get_data.raw_header_wcs)
213 def test_subset_raw(self):
214 for kwargs, expect in self.butler_get_data.raw_subsets:
215 if self._is_gen3():
216 # If one of the keyword args looks like a dimension record
217 # subquery, pull it out into the WHERE clause.
218 where = []
219 bind = {}
220 for k, v in list(kwargs.items()):
221 if "." in k:
222 bindval = k.replace(".", "_")
223 where.append(f"{k} = {bindval}")
224 bind[bindval] = v
225 del kwargs[k]
226 try:
227 subset = set(
228 self.butler.registry.queryDatasets(
229 "raw", **kwargs, bind=bind, where=" AND ".join(where)
230 )
231 )
232 except DataIdValueError:
233 # This means one of the dataId values does not exist.
234 subset = {}
235 else:
236 subset = self.butler.subset("raw", **kwargs)
238 self.assertEqual(len(subset), expect, msg="Failed for kwargs: {}".format(kwargs))
240 def test_get_linearizer(self):
241 """Test that we can get a linearizer for good detectorIds."""
242 if self.butler_get_data.linearizer_type is unittest.SkipTest:
243 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
245 camera = self.butler.get("camera")
246 for detectorId in self.butler_get_data.good_detectorIds:
247 detector = camera[detectorId]
248 if self._is_gen3():
249 kwargs = {"detector": detectorId}
250 else:
251 kwargs = {"dataId": dict(ccd=detectorId), "immediate": True}
252 linearizer = self.butler.get("linearizer", **kwargs)
253 self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId])
254 linearizer.checkDetector(detector)
256 def test_get_linearizer_bad_detectorIds(self):
257 """Do bad detectorIds raise?"""
258 if self.butler_get_data.linearizer_type is unittest.SkipTest:
259 self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
261 for badccd in self.butler_get_data.bad_detectorIds:
262 if self._is_gen3():
263 kwargs = {"detector": badccd}
264 else:
265 kwargs = {"dataId": dict(ccd=badccd), "immediate": True}
266 with self.assertRaises(RuntimeError):
267 self.butler.get("linearizer", **kwargs)