Coverage for tests/test_ingestion.py: 32%

2# This file is part of ap_verify.

4# Developed for the LSST Data Management System.

5# This product includes software developed by the LSST Project

6# (http://www.lsst.org).

7# See the COPYRIGHT file at the top-level directory of this distribution

8# for details of code ownership.

10# This program is free software: you can redistribute it and/or modify

11# it under the terms of the GNU General Public License as published by

12# the Free Software Foundation, either version 3 of the License, or

13# (at your option) any later version.

14#

15# This program is distributed in the hope that it will be useful,

16# but WITHOUT ANY WARRANTY; without even the implied warranty of

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

18# GNU General Public License for more details.

19#

20# You should have received a copy of the GNU General Public License

21# along with this program. If not, see <http://www.gnu.org/licenses/>.

22#

24import os

25import pickle

26import shutil

27import tempfile

28import unittest

30import lsst.utils.tests

31from lsst.daf.butler import CollectionType

32from lsst.ap.verify import ingestion

33from lsst.ap.verify.testUtils import DataTestCase

34from lsst.ap.verify.dataset import Dataset

35from lsst.ap.verify.workspace import WorkspaceGen3

38class IngestionTestSuiteGen3(DataTestCase):

40 @classmethod

41 def setUpClass(cls):

42 super().setUpClass()

44 cls.dataset = Dataset(cls.testDataset)

46 cls.INSTRUMENT = cls.dataset.instrument.getName()

47 cls.VISIT_ID = 204595

48 cls.DETECTOR_ID = 37

50 cls.rawData = [{'type': 'raw', 'file': 'lsst_a_204595_R11_S01_i.fits',

51 'exposure': cls.VISIT_ID, 'detector': cls.DETECTOR_ID,

52 'instrument': cls.INSTRUMENT},

53 ]

55 cls.calibData = [{'type': 'bias', 'file': 'bias-R11-S01-det037_2022-01-01.fits.gz',

56 'detector': cls.DETECTOR_ID, 'instrument': cls.INSTRUMENT},

57 {'type': 'flat', 'file': 'flat_i-R11-S01-det037_2022-08-06.fits.gz',

58 'detector': cls.DETECTOR_ID, 'instrument': cls.INSTRUMENT,

59 'physical_filter': 'i_sim_1.4'},

60 ]

62 @classmethod

63 def makeTestConfig(cls):

64 instrument = cls.dataset.instrument

65 config = ingestion.Gen3DatasetIngestConfig()

66 instrument.applyConfigOverrides(ingestion.Gen3DatasetIngestTask._DefaultName, config)

67 return config

69 def setUp(self):

70 super().setUp()

72 self.config = self.makeTestConfig()

73 self.config.validate()

74 self.config.freeze()

76 self.root = tempfile.mkdtemp()

77 self.addCleanup(shutil.rmtree, self.root, ignore_errors=True)

78 self.workspace = WorkspaceGen3(self.root)

79 self.task = ingestion.Gen3DatasetIngestTask(config=self.config,

80 dataset=self.dataset, workspace=self.workspace)

82 self.butler = self.workspace.workButler

84 def assertIngestedDataFiles(self, data, collection):

85 """Test that data have been loaded into a specific collection.

87 Parameters

88 ----------

89 data : `collections.abc.Iterable` [`collections.abc.Mapping`]

90 An iterable of mappings, each representing the properties of a

91 single input dataset. Each mapping must contain a `"type"` key

92 that maps to the dataset's Gen 3 type.

93 collection

94 Any valid :ref:`collection expression <daf_butler_collection_expressions>`

95 for the collection expected to contain the data.

96 """

97 for datum in data:

98 dataId = datum.copy()

99 dataId.pop("type", None)

100 dataId.pop("file", None)

101

102 matches = [x for x in self.butler.registry.queryDatasets(datum['type'],

103 collections=collection,

104 dataId=dataId)]

105 self.assertNotEqual(matches, [])

106

107 def testDataIngest(self):

108 """Test that ingesting science images given specific files adds them to a repository.

109 """

110 files = [os.path.join(self.dataset.rawLocation, datum['file']) for datum in self.rawData]

111 self.task._ingestRaws(files, processes=1)

112 self.assertIngestedDataFiles(self.rawData, self.dataset.instrument.makeDefaultRawIngestRunName())

113

114 def testDataDoubleIngest(self):

115 """Test that re-ingesting science images raises RuntimeError.

116 """

117 files = [os.path.join(self.dataset.rawLocation, datum['file']) for datum in self.rawData]

118 self.task._ingestRaws(files, processes=1)

119 with self.assertRaises(RuntimeError):

120 self.task._ingestRaws(files, processes=1)

121

122 def testDataIngestDriver(self):

123 """Test that ingesting science images starting from an abstract dataset adds them to a repository.

124 """

125 self.task._ensureRaws(processes=1)

126 self.assertIngestedDataFiles(self.rawData, self.dataset.instrument.makeDefaultRawIngestRunName())

127

128 def testCalibIngestDriver(self):

129 """Test that ingesting calibrations starting from an abstract dataset adds them to a repository.

130 """

131 self.task._ensureRaws(processes=1) # Should not affect calibs, but would be run

132 # queryDatasets cannot (yet) search CALIBRATION collections, so we

133 # instead search the RUN-type collections that calibrations are

134 # ingested into first before being associated with a validity range.

135 calibrationRunPattern = self.dataset.instrument.makeCollectionName("calib") + "/*"

136 calibrationRuns = list(

137 self.butler.registry.queryCollections(

138 calibrationRunPattern,

139 collectionTypes={CollectionType.RUN},

140 )

141 )

142 self.assertIngestedDataFiles(self.calibData, calibrationRuns)

143

144 def testNoFileIngest(self):

145 """Test that attempts to ingest nothing raise an exception.

146 """

147 with self.assertRaises(RuntimeError):

148 self.task._ingestRaws([], processes=1)

149

150 def testVisitDefinition(self):

151 """Test that the final repository supports indexing by visit.

152 """

153 self.task._ensureRaws(processes=1)

154 self.task._defineVisits(processes=1)

155

156 testId = {"visit": self.VISIT_ID, "instrument": self.INSTRUMENT, }

157 exposures = list(self.butler.registry.queryDataIds("exposure", dataId=testId))

158 self.assertEqual(len(exposures), 1)

159 self.assertEqual(exposures[0]["exposure"], self.VISIT_ID)

160

161 def testVisitDoubleDefinition(self):

162 """Test that re-defining visits is guarded against.

163 """

164 self.task._ensureRaws(processes=1)

165 self.task._defineVisits(processes=1)

166 self.task._defineVisits(processes=1) # must not raise

167

168 testId = {"visit": self.VISIT_ID, "instrument": self.INSTRUMENT, }

169 exposures = list(self.butler.registry.queryDataIds("exposure", dataId=testId))

170 self.assertEqual(len(exposures), 1)

171

172 def testVisitsUndefinable(self):

173 """Test that attempts to define visits with no exposures raise an exception.

174 """

175 with self.assertRaises(RuntimeError):

176 self.task._defineVisits(processes=1)

177

178 def testCopyConfigs(self):

179 """Test that "ingesting" configs stores them in the workspace for later reference.

180 """

181 self.task._copyConfigs()

182 self.assertTrue(os.path.exists(self.workspace.configDir))

183 self.assertTrue(os.path.exists(self.workspace.pipelineDir))

184 self.assertTrue(os.path.exists(os.path.join(self.workspace.pipelineDir, "ApVerify.yaml")))

185

186 def testPickling(self):

187 """Test that a Gen3DatasetIngestTask can be pickled correctly.

188

189 This is needed for multiprocessing support.

190 """

191 stream = pickle.dumps(self.task)

192 copy = pickle.loads(stream)

193 self.assertEqual(self.task.getFullName(), copy.getFullName())

194 self.assertEqual(self.task.log.name, copy.log.name)

195 # Equality for config ill-behaved; skip testing it

196 self.assertEqual(self.task.dataset, copy.dataset)

197 self.assertEqual(self.task.workspace, copy.workspace)

198

199

200class MemoryTester(lsst.utils.tests.MemoryTestCase):

201 pass

202

203

204def setup_module(module):

205 lsst.utils.tests.init()

206

207

208if __name__ == "__main__": 208 ↛ 209line 208 didn't jump to line 209 because the condition on line 208 was never true

209 lsst.utils.tests.init()

210 unittest.main()

Coverage for tests / test_ingestion.py: 32%

100 statements