Coverage for python/lsst/daf/butler/tests/utils.py : 26%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
23import astropy
24from astropy.table import Table as AstropyTable
25from astropy.utils.diff import report_diff_values
26import io
27import os
30from .. import (
31 Butler,
32 Config,
33 StorageClassFactory,
34)
35from ..tests import addDatasetType, MetricsExample
36from ..registry import CollectionType
39class ButlerTestHelper:
40 """Mixin with helpers for unit tests."""
42 def assertAstropyTablesEqual(self, tables, expectedTables):
43 """Verify that a list of astropy tables matches a list of expected
44 astropy tables.
46 Parameters
47 ----------
48 tables : `astropy.table.Table` or iterable [`astropy.table.Table`]
49 The table or tables that should match the expected tables.
50 expectedTables : `astropy.table.Table`
51 or iterable [`astropy.table.Table`]
52 The tables with expected values to which the tables under test will
53 be compared.
54 """
55 # If a single table is passed in for tables or expectedTables, put it
56 # in a list.
57 if isinstance(tables, AstropyTable):
58 tables = [tables]
59 if isinstance(expectedTables, AstropyTable):
60 expectedTables = [expectedTables]
61 diff = io.StringIO()
62 self.assertEqual(len(tables), len(expectedTables))
63 for table, expected in zip(tables, expectedTables):
64 # Assert that we are testing what we think we are testing:
65 self.assertIsInstance(table, AstropyTable)
66 self.assertIsInstance(expected, AstropyTable)
67 # Assert that they match:
68 self.assertTrue(report_diff_values(table, expected, fileobj=diff), msg="\n" + diff.getvalue())
71def readTable(textTable):
72 """Read an astropy table from formatted text.
74 Contains formatting that causes the astropy table to print an empty string
75 instead of "--" for missing/unpopulated values in the text table.
78 Parameters
79 ----------
80 textTable : `str`
81 The text version of the table to read.
83 Returns
84 -------
85 table : `astropy.table.Table`
86 The table as an astropy table.
87 """
88 return AstropyTable.read(textTable,
89 format="ascii",
90 data_start=2, # skip the header row and the header row underlines.
91 fill_values=[("", 0, "")])
94class MetricTestRepo:
95 """Creates and manage a test repository on disk with datasets that
96 may be queried and modified for unit tests.
98 Parameters
99 ----------
100 root : `str`
101 The location of the repository, to pass to ``Butler.makeRepo``.
102 configFile : `str`
103 The path to the config file, to pass to ``Butler.makeRepo``.
104 """
106 @staticmethod
107 def _makeExampleMetrics():
108 """Make an object to put into the repository.
109 """
110 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
111 {"a": [1, 2, 3],
112 "b": {"blue": 5, "red": "green"}},
113 [563, 234, 456.7, 752, 8, 9, 27])
115 @staticmethod
116 def _makeDimensionData(id, name, datetimeBegin=None, datetimeEnd=None):
117 """Make a dict of dimensional data with default values to insert into
118 the registry.
119 """
120 data = dict(instrument="DummyCamComp",
121 id=id,
122 name=name,
123 physical_filter="d-r",
124 visit_system=1)
125 if datetimeBegin:
126 data["datetime_begin"] = datetimeBegin
127 data["datetime_end"] = datetimeEnd
128 return data
130 def __init__(self, root, configFile):
131 self.root = root
132 Butler.makeRepo(self.root, config=Config(configFile))
133 butlerConfigFile = os.path.join(self.root, "butler.yaml")
134 self.storageClassFactory = StorageClassFactory()
135 self.storageClassFactory.addFromConfig(butlerConfigFile)
137 # New datasets will be added to run and tag, but we will only look in
138 # tag when looking up datasets.
139 run = "ingest/run"
140 tag = "ingest"
141 self.butler = Butler(butlerConfigFile, run=run, collections=[tag], tags=[tag])
143 # Create and register a DatasetType
144 self.datasetType = addDatasetType(self.butler, "test_metric_comp", ("instrument", "visit"),
145 "StructuredCompositeReadComp")
147 # Add needed Dimensions
148 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"})
149 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp",
150 "name": "d-r",
151 "band": "R"})
152 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp",
153 "id": 1,
154 "name": "default"})
155 visitStart = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai")
156 visitEnd = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai")
157 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp",
158 id=423,
159 name="fourtwentythree",
160 physical_filter="d-r",
161 visit_system=1,
162 datetimeBegin=visitStart,
163 datetimeEnd=visitEnd))
164 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp",
165 id=424,
166 name="fourtwentyfour",
167 physical_filter="d-r",
168 visit_system=1))
170 self.addDataset({"instrument": "DummyCamComp", "visit": 423})
171 self.addDataset({"instrument": "DummyCamComp", "visit": 424})
173 def addDataset(self, dataId, run=None, datasetType=None):
174 """Create a new example metric and add it to the named run with the
175 given dataId.
177 Overwrites tags, so this does not try to associate the new dataset with
178 existing tags. (If/when tags are needed this can be added to the
179 arguments of this function.)
181 Parameters
182 ----------
183 dataId : `dict`
184 The dataId for the new metric.
185 run : `str`, optional
186 The name of the run to create and add a dataset to. If `None`, the
187 dataset will be added to the root butler.
188 datasetType : ``DatasetType``, optional
189 The dataset type of the added dataset. If `None`, will use the
190 default dataset type.
191 """
192 if run:
193 self.butler.registry.registerCollection(run, type=CollectionType.RUN)
194 metric = self._makeExampleMetrics()
195 self.butler.put(metric,
196 self.datasetType if datasetType is None else datasetType,
197 dataId,
198 run=run,
199 tags=())