Coverage for tests/test_testPipeline.py: 20%
137 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-02 06:35 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-02 06:35 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import shutil
25import tempfile
26import unittest
28import pandas
30import lsst.utils.tests
31import lsst.geom
32import lsst.afw.image as afwImage
33import lsst.afw.math as afwMath
34import lsst.afw.table as afwTable
35import lsst.skymap
36import lsst.daf.butler.tests as butlerTests
37import lsst.pipe.base.testUtils as pipelineTests
38from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \
39 MockCalibrateTask, MockImageDifferenceTask, MockTransformDiaSourceCatalogTask, \
40 MockDiaPipelineTask
43class MockTaskTestSuite(unittest.TestCase):
44 """Test that mock tasks have the correct inputs and outputs for the task
45 they are replacing.
47 These tests assume that the mock tasks use real config and connection
48 classes, and therefore out-of-date mocks won't match their connections.
49 """
51 @classmethod
52 def setUpClass(cls):
53 super().setUpClass()
55 repoDir = tempfile.mkdtemp()
56 cls.addClassCleanup(shutil.rmtree, repoDir, ignore_errors=True)
57 cls.repo = butlerTests.makeTestRepo(repoDir)
59 INSTRUMENT = "notACam"
60 VISIT = 42
61 CCD = 101
62 HTM = 42
63 SKYMAP = "TreasureMap"
64 TRACT = 28
65 PATCH = 4
66 BAND = 'k'
67 PHYSICAL = 'k2022'
68 SUB_FILTER = 9
69 # Mock instrument by hand, because some tasks care about parameters
70 instrumentRecord = cls.repo.registry.dimensions["instrument"].RecordClass(
71 name=INSTRUMENT, visit_max=256, exposure_max=256, detector_max=128)
72 cls.repo.registry.syncDimensionData("instrument", instrumentRecord)
73 butlerTests.addDataIdValue(cls.repo, "physical_filter", PHYSICAL, band=BAND)
74 butlerTests.addDataIdValue(cls.repo, "subfilter", SUB_FILTER)
75 butlerTests.addDataIdValue(cls.repo, "exposure", VISIT)
76 butlerTests.addDataIdValue(cls.repo, "visit", VISIT)
77 butlerTests.addDataIdValue(cls.repo, "detector", CCD)
78 butlerTests.addDataIdValue(cls.repo, "skymap", SKYMAP)
79 butlerTests.addDataIdValue(cls.repo, "tract", TRACT)
80 butlerTests.addDataIdValue(cls.repo, "patch", PATCH)
82 cls.exposureId = cls.repo.registry.expandDataId(
83 {"instrument": INSTRUMENT, "exposure": VISIT, "detector": CCD})
84 cls.visitId = cls.repo.registry.expandDataId(
85 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD})
86 cls.visitOnlyId = cls.repo.registry.expandDataId(
87 {"instrument": INSTRUMENT, "visit": VISIT})
88 cls.skymapId = cls.repo.registry.expandDataId({"skymap": SKYMAP})
89 cls.skymapVisitId = cls.repo.registry.expandDataId(
90 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD, "skymap": SKYMAP})
91 cls.patchId = cls.repo.registry.expandDataId(
92 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND})
93 cls.subfilterId = cls.repo.registry.expandDataId(
94 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND, "subfilter": SUB_FILTER})
95 cls.htmId = cls.repo.registry.expandDataId({"htm7": HTM})
97 butlerTests.addDatasetType(cls.repo, "postISRCCD", cls.exposureId.keys(), "Exposure")
98 butlerTests.addDatasetType(cls.repo, "icExp", cls.visitId.keys(), "ExposureF")
99 butlerTests.addDatasetType(cls.repo, "icSrc", cls.visitId.keys(), "SourceCatalog")
100 butlerTests.addDatasetType(cls.repo, "icExpBackground", cls.visitId.keys(), "Background")
101 butlerTests.addDatasetType(cls.repo, "gaia_dr2_20200414", cls.htmId.keys(), "SimpleCatalog")
102 butlerTests.addDatasetType(cls.repo, "ps1_pv3_3pi_20170110", cls.htmId.keys(), "SimpleCatalog")
103 butlerTests.addDatasetType(cls.repo, "calexp", cls.visitId.keys(), "ExposureF")
104 butlerTests.addDatasetType(cls.repo, "src", cls.visitId.keys(), "SourceCatalog")
105 butlerTests.addDatasetType(cls.repo, "calexpBackground", cls.visitId.keys(), "Background")
106 butlerTests.addDatasetType(cls.repo, "srcMatch", cls.visitId.keys(), "Catalog")
107 butlerTests.addDatasetType(cls.repo, "srcMatchFull", cls.visitId.keys(), "Catalog")
108 butlerTests.addDatasetType(cls.repo, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
109 cls.skymapId.keys(), "SkyMap")
110 butlerTests.addDatasetType(cls.repo, "deepCoadd", cls.patchId.keys(), "ExposureF")
111 butlerTests.addDatasetType(cls.repo, "dcrCoadd", cls.subfilterId.keys(), "ExposureF")
112 butlerTests.addDatasetType(cls.repo, "deepDiff_differenceExp", cls.visitId.keys(), "ExposureF")
113 butlerTests.addDatasetType(cls.repo, "deepDiff_scoreExp", cls.visitId.keys(), "ExposureF")
114 butlerTests.addDatasetType(cls.repo, "deepDiff_templateExp", cls.visitId.keys(), "ExposureF")
115 butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.keys(), "ExposureF")
116 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.keys(), "SourceCatalog")
117 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.keys(), "DataFrame")
118 butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.keys(), "DataFrame")
119 butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.keys(), "Config")
120 butlerTests.addDatasetType(cls.repo, "deepDiff_associDiaSrc", cls.visitId.keys(), "DataFrame")
122 def setUp(self):
123 super().setUp()
124 self.butler = butlerTests.makeTestCollection(self.repo, uniqueId=self.id())
126 def testMockIsr(self):
127 # Testing MockIsrTask is tricky because the real ISR has an unstable
128 # interface with dozens of potential inputs, too many to pass through
129 # runTestQuantum. I don't see a good way to test the inputs;
130 # fortunately, this is unlikely to matter for the overall goal of
131 # testing ap_verify's interaction with the AP pipeline.
132 task = MockIsrTask()
133 pipelineTests.assertValidInitOutput(task)
134 result = task.run(afwImage.ExposureF())
135 pipelineTests.assertValidOutput(task, result)
136 # Skip runTestQuantum
138 def testMockCharacterizeImageTask(self):
139 task = MockCharacterizeImageTask()
140 pipelineTests.assertValidInitOutput(task)
141 result = task.run(afwImage.ExposureF())
142 pipelineTests.assertValidOutput(task, result)
144 self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId)
145 quantum = pipelineTests.makeQuantum(
146 task, self.butler, self.visitId,
147 {"exposure": self.exposureId,
148 "characterized": self.visitId,
149 "sourceCat": self.visitId,
150 "backgroundModel": self.visitId,
151 })
152 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
154 def testMockCalibrateTask(self):
155 task = MockCalibrateTask()
156 pipelineTests.assertValidInitOutput(task)
157 # Even the real CalibrateTask won't pass assertValidOutput, because for
158 # some reason the outputs are injected in runQuantum rather than run.
160 self.butler.put(afwImage.ExposureF(), "icExp", self.visitId)
161 self.butler.put(afwMath.BackgroundList(), "icExpBackground", self.visitId)
162 self.butler.put(afwTable.SourceCatalog(), "icSrc", self.visitId)
163 self.butler.put(afwTable.SimpleCatalog(), "gaia_dr2_20200414", self.htmId)
164 self.butler.put(afwTable.SimpleCatalog(), "ps1_pv3_3pi_20170110", self.htmId)
165 quantum = pipelineTests.makeQuantum(
166 task, self.butler, self.visitId,
167 {"exposure": self.visitId,
168 "background": self.visitId,
169 "icSourceCat": self.visitId,
170 "astromRefCat": [self.htmId],
171 "photoRefCat": [self.htmId],
172 "outputExposure": self.visitId,
173 "outputCat": self.visitId,
174 "outputBackground": self.visitId,
175 "matches": self.visitId,
176 "matchesDenormalized": self.visitId,
177 })
178 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
180 def testMockImageDifferenceTask(self):
181 task = MockImageDifferenceTask()
182 pipelineTests.assertValidInitOutput(task)
183 result = task.run(afwImage.ExposureF(), templateExposure=afwImage.ExposureF())
184 pipelineTests.assertValidOutput(task, result)
186 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
187 skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig())
188 self.butler.put(skymap, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, self.skymapId)
189 self.butler.put(afwImage.ExposureF(), "deepCoadd", self.patchId)
190 self.butler.put(afwImage.ExposureF(), "dcrCoadd", self.subfilterId)
191 quantum = pipelineTests.makeQuantum(
192 task, self.butler, self.skymapVisitId,
193 {"exposure": self.visitId,
194 "skyMap": self.skymapId,
195 "coaddExposures": [self.patchId],
196 "dcrCoadds": [self.subfilterId],
197 "subtractedExposure": self.visitId,
198 "scoreExposure": self.visitId,
199 "template": self.visitId,
200 "matchedExposure": self.visitId,
201 "diaSources": self.visitId,
202 })
203 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
205 def testMockTransformDiaSourceCatalogTask(self):
206 task = MockTransformDiaSourceCatalogTask(initInputs=afwTable.SourceCatalog())
207 pipelineTests.assertValidInitOutput(task)
208 result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42)
209 pipelineTests.assertValidOutput(task, result)
211 self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
212 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
213 quantum = pipelineTests.makeQuantum(
214 task, self.butler, self.visitId,
215 {"diaSourceCat": self.visitId,
216 "diffIm": self.visitId,
217 "diaSourceTable": self.visitId,
218 })
219 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
221 def testMockDiaPipelineTask(self):
222 task = MockDiaPipelineTask()
223 pipelineTests.assertValidInitOutput(task)
224 result = task.run(pandas.DataFrame(), pandas.DataFrame(), afwImage.ExposureF(),
225 afwImage.ExposureF(), afwImage.ExposureF(), 42, 'k')
226 pipelineTests.assertValidOutput(task, result)
228 self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId)
229 self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId)
230 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
231 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
232 self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId)
233 quantum = pipelineTests.makeQuantum(
234 task, self.butler, self.visitId,
235 {"diaSourceTable": self.visitId,
236 "solarSystemObjectTable": self.visitId,
237 "diffIm": self.visitId,
238 "exposure": self.visitId,
239 "template": self.visitId,
240 "apdbMarker": self.visitId,
241 "associatedDiaSources": self.visitId,
242 })
243 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
246class MemoryTester(lsst.utils.tests.MemoryTestCase):
247 pass
250def setup_module(module):
251 lsst.utils.tests.init()
254if __name__ == "__main__": 254 ↛ 255line 254 didn't jump to line 255, because the condition on line 254 was never true
255 lsst.utils.tests.init()
256 unittest.main()