Coverage for tests/test_ingestion.py: 31%
101 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-13 03:57 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-13 03:57 -0700
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import os
25import pickle
26import re
27import shutil
28import tempfile
29import unittest
31import lsst.utils.tests
32from lsst.daf.butler import CollectionType
33from lsst.ap.verify import ingestion
34from lsst.ap.verify.testUtils import DataTestCase
35from lsst.ap.verify.dataset import Dataset
36from lsst.ap.verify.workspace import WorkspaceGen3
39class IngestionTestSuiteGen3(DataTestCase):
41 @classmethod
42 def setUpClass(cls):
43 super().setUpClass()
45 cls.dataset = Dataset(cls.testDataset)
47 cls.INSTRUMENT = cls.dataset.instrument.getName()
48 cls.VISIT_ID = 204595
49 cls.DETECTOR_ID = 37
51 cls.rawData = [{'type': 'raw', 'file': 'lsst_a_204595_R11_S01_i.fits',
52 'exposure': cls.VISIT_ID, 'detector': cls.DETECTOR_ID,
53 'instrument': cls.INSTRUMENT},
54 ]
56 cls.calibData = [{'type': 'bias', 'file': 'bias-R11-S01-det037_2022-01-01.fits.gz',
57 'detector': cls.DETECTOR_ID, 'instrument': cls.INSTRUMENT},
58 {'type': 'flat', 'file': 'flat_i-R11-S01-det037_2022-08-06.fits.gz',
59 'detector': cls.DETECTOR_ID, 'instrument': cls.INSTRUMENT,
60 'physical_filter': 'i_sim_1.4'},
61 ]
63 @classmethod
64 def makeTestConfig(cls):
65 instrument = cls.dataset.instrument
66 config = ingestion.Gen3DatasetIngestConfig()
67 instrument.applyConfigOverrides(ingestion.Gen3DatasetIngestTask._DefaultName, config)
68 return config
70 def setUp(self):
71 super().setUp()
73 self.config = self.makeTestConfig()
74 self.config.validate()
75 self.config.freeze()
77 self.root = tempfile.mkdtemp()
78 self.addCleanup(shutil.rmtree, self.root, ignore_errors=True)
79 self.workspace = WorkspaceGen3(self.root)
80 self.task = ingestion.Gen3DatasetIngestTask(config=self.config,
81 dataset=self.dataset, workspace=self.workspace)
83 self.butler = self.workspace.workButler
85 def assertIngestedDataFiles(self, data, collection):
86 """Test that data have been loaded into a specific collection.
88 Parameters
89 ----------
90 data : `collections.abc.Iterable` [`collections.abc.Mapping`]
91 An iterable of mappings, each representing the properties of a
92 single input dataset. Each mapping must contain a `"type"` key
93 that maps to the dataset's Gen 3 type.
94 collection
95 Any valid :ref:`collection expression <daf_butler_collection_expressions>`
96 for the collection expected to contain the data.
97 """
98 for datum in data:
99 dataId = datum.copy()
100 dataId.pop("type", None)
101 dataId.pop("file", None)
103 matches = [x for x in self.butler.registry.queryDatasets(datum['type'],
104 collections=collection,
105 dataId=dataId)]
106 self.assertNotEqual(matches, [])
108 def testDataIngest(self):
109 """Test that ingesting science images given specific files adds them to a repository.
110 """
111 files = [os.path.join(self.dataset.rawLocation, datum['file']) for datum in self.rawData]
112 self.task._ingestRaws(files, processes=1)
113 self.assertIngestedDataFiles(self.rawData, self.dataset.instrument.makeDefaultRawIngestRunName())
115 def testDataDoubleIngest(self):
116 """Test that re-ingesting science images raises RuntimeError.
117 """
118 files = [os.path.join(self.dataset.rawLocation, datum['file']) for datum in self.rawData]
119 self.task._ingestRaws(files, processes=1)
120 with self.assertRaises(RuntimeError):
121 self.task._ingestRaws(files, processes=1)
123 def testDataIngestDriver(self):
124 """Test that ingesting science images starting from an abstract dataset adds them to a repository.
125 """
126 self.task._ensureRaws(processes=1)
127 self.assertIngestedDataFiles(self.rawData, self.dataset.instrument.makeDefaultRawIngestRunName())
129 def testCalibIngestDriver(self):
130 """Test that ingesting calibrations starting from an abstract dataset adds them to a repository.
131 """
132 self.task._ensureRaws(processes=1) # Should not affect calibs, but would be run
133 # queryDatasets cannot (yet) search CALIBRATION collections, so we
134 # instead search the RUN-type collections that calibrations are
135 # ingested into first before being associated with a validity range.
136 calibrationRunPattern = re.compile(
137 re.escape(self.dataset.instrument.makeCollectionName("calib") + "/") + ".+"
138 )
139 calibrationRuns = list(
140 self.butler.registry.queryCollections(
141 calibrationRunPattern,
142 collectionTypes={CollectionType.RUN},
143 )
144 )
145 self.assertIngestedDataFiles(self.calibData, calibrationRuns)
147 def testNoFileIngest(self):
148 """Test that attempts to ingest nothing raise an exception.
149 """
150 with self.assertRaises(RuntimeError):
151 self.task._ingestRaws([], processes=1)
153 def testVisitDefinition(self):
154 """Test that the final repository supports indexing by visit.
155 """
156 self.task._ensureRaws(processes=1)
157 self.task._defineVisits(processes=1)
159 testId = {"visit": self.VISIT_ID, "instrument": self.INSTRUMENT, }
160 exposures = list(self.butler.registry.queryDataIds("exposure", dataId=testId))
161 self.assertEqual(len(exposures), 1)
162 self.assertEqual(exposures[0]["exposure"], self.VISIT_ID)
164 def testVisitDoubleDefinition(self):
165 """Test that re-defining visits is guarded against.
166 """
167 self.task._ensureRaws(processes=1)
168 self.task._defineVisits(processes=1)
169 self.task._defineVisits(processes=1) # must not raise
171 testId = {"visit": self.VISIT_ID, "instrument": self.INSTRUMENT, }
172 exposures = list(self.butler.registry.queryDataIds("exposure", dataId=testId))
173 self.assertEqual(len(exposures), 1)
175 def testVisitsUndefinable(self):
176 """Test that attempts to define visits with no exposures raise an exception.
177 """
178 with self.assertRaises(RuntimeError):
179 self.task._defineVisits(processes=1)
181 def testCopyConfigs(self):
182 """Test that "ingesting" configs stores them in the workspace for later reference.
183 """
184 self.task._copyConfigs()
185 self.assertTrue(os.path.exists(self.workspace.configDir))
186 self.assertTrue(os.path.exists(self.workspace.pipelineDir))
187 self.assertTrue(os.path.exists(os.path.join(self.workspace.pipelineDir, "ApVerify.yaml")))
189 def testPickling(self):
190 """Test that a Gen3DatasetIngestTask can be pickled correctly.
192 This is needed for multiprocessing support.
193 """
194 stream = pickle.dumps(self.task)
195 copy = pickle.loads(stream)
196 self.assertEqual(self.task.getFullName(), copy.getFullName())
197 self.assertEqual(self.task.log.name, copy.log.name)
198 # Equality for config ill-behaved; skip testing it
199 self.assertEqual(self.task.dataset, copy.dataset)
200 self.assertEqual(self.task.workspace, copy.workspace)
203class MemoryTester(lsst.utils.tests.MemoryTestCase):
204 pass
207def setup_module(module):
208 lsst.utils.tests.init()
211if __name__ == "__main__": 211 ↛ 212line 211 didn't jump to line 212, because the condition on line 211 was never true
212 lsst.utils.tests.init()
213 unittest.main()