Coverage for tests/test_testPipeline.py: 22%
139 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-28 09:59 +0000
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-28 09:59 +0000
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import shutil
25import tempfile
26import unittest
28import pandas
30import lsst.utils.tests
31import lsst.geom
32import lsst.afw.image as afwImage
33import lsst.afw.math as afwMath
34import lsst.afw.table as afwTable
35import lsst.skymap
36import lsst.daf.butler.tests as butlerTests
37import lsst.pipe.base.testUtils as pipelineTests
38from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \
39 MockCalibrateTask, MockImageDifferenceTask, MockTransformDiaSourceCatalogTask, \
40 MockDiaPipelineTask
43class MockTaskTestSuite(unittest.TestCase):
44 """Test that mock tasks have the correct inputs and outputs for the task
45 they are replacing.
47 These tests assume that the mock tasks use real config and connection
48 classes, and therefore out-of-date mocks won't match their connections.
49 """
51 @classmethod
52 def setUpClass(cls):
53 super().setUpClass()
55 repoDir = tempfile.mkdtemp()
56 cls.addClassCleanup(shutil.rmtree, repoDir, ignore_errors=True)
57 cls.repo = butlerTests.makeTestRepo(repoDir)
59 INSTRUMENT = "notACam"
60 VISIT = 42
61 CCD = 101
62 HTM = 42
63 SKYMAP = "TreasureMap"
64 TRACT = 28
65 PATCH = 4
66 BAND = 'k'
67 PHYSICAL = 'k2022'
68 SUB_FILTER = 9
69 # Mock instrument by hand, because some tasks care about parameters
70 instrumentRecord = cls.repo.registry.dimensions["instrument"].RecordClass(
71 name=INSTRUMENT, visit_max=256, exposure_max=256, detector_max=128)
72 cls.repo.registry.syncDimensionData("instrument", instrumentRecord)
73 butlerTests.addDataIdValue(cls.repo, "physical_filter", PHYSICAL, band=BAND)
74 butlerTests.addDataIdValue(cls.repo, "subfilter", SUB_FILTER)
75 butlerTests.addDataIdValue(cls.repo, "exposure", VISIT)
76 butlerTests.addDataIdValue(cls.repo, "visit", VISIT)
77 butlerTests.addDataIdValue(cls.repo, "detector", CCD)
78 butlerTests.addDataIdValue(cls.repo, "skymap", SKYMAP)
79 butlerTests.addDataIdValue(cls.repo, "tract", TRACT)
80 butlerTests.addDataIdValue(cls.repo, "patch", PATCH)
82 cls.exposureId = cls.repo.registry.expandDataId(
83 {"instrument": INSTRUMENT, "exposure": VISIT, "detector": CCD})
84 cls.visitId = cls.repo.registry.expandDataId(
85 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD})
86 cls.visitOnlyId = cls.repo.registry.expandDataId(
87 {"instrument": INSTRUMENT, "visit": VISIT})
88 cls.skymapId = cls.repo.registry.expandDataId({"skymap": SKYMAP})
89 cls.skymapVisitId = cls.repo.registry.expandDataId(
90 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD, "skymap": SKYMAP})
91 cls.patchId = cls.repo.registry.expandDataId(
92 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND})
93 cls.subfilterId = cls.repo.registry.expandDataId(
94 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND, "subfilter": SUB_FILTER})
95 cls.htmId = cls.repo.registry.expandDataId({"htm7": HTM})
97 butlerTests.addDatasetType(cls.repo, "postISRCCD", cls.exposureId.keys(), "Exposure")
98 butlerTests.addDatasetType(cls.repo, "icExp", cls.visitId.keys(), "ExposureF")
99 butlerTests.addDatasetType(cls.repo, "icSrc", cls.visitId.keys(), "SourceCatalog")
100 butlerTests.addDatasetType(cls.repo, "icExpBackground", cls.visitId.keys(), "Background")
101 butlerTests.addDatasetType(cls.repo, "gaia_dr2_20200414", cls.htmId.keys(), "SimpleCatalog")
102 butlerTests.addDatasetType(cls.repo, "ps1_pv3_3pi_20170110", cls.htmId.keys(), "SimpleCatalog")
103 butlerTests.addDatasetType(cls.repo, "calexp", cls.visitId.keys(), "ExposureF")
104 butlerTests.addDatasetType(cls.repo, "src", cls.visitId.keys(), "SourceCatalog")
105 butlerTests.addDatasetType(cls.repo, "calexpBackground", cls.visitId.keys(), "Background")
106 butlerTests.addDatasetType(cls.repo, "srcMatch", cls.visitId.keys(), "Catalog")
107 butlerTests.addDatasetType(cls.repo, "srcMatchFull", cls.visitId.keys(), "Catalog")
108 butlerTests.addDatasetType(cls.repo, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
109 cls.skymapId.keys(), "SkyMap")
110 butlerTests.addDatasetType(cls.repo, "deepCoadd", cls.patchId.keys(), "ExposureF")
111 butlerTests.addDatasetType(cls.repo, "dcrCoadd", cls.subfilterId.keys(), "ExposureF")
112 butlerTests.addDatasetType(cls.repo, "deepDiff_differenceExp", cls.visitId.keys(), "ExposureF")
113 butlerTests.addDatasetType(cls.repo, "deepDiff_scoreExp", cls.visitId.keys(), "ExposureF")
114 butlerTests.addDatasetType(cls.repo, "deepDiff_templateExp", cls.visitId.keys(), "ExposureF")
115 butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.keys(), "ExposureF")
116 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.keys(), "SourceCatalog")
117 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.keys(), "DataFrame")
118 butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.keys(), "DataFrame")
119 butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.keys(), "Config")
120 butlerTests.addDatasetType(cls.repo, "deepDiff_associDiaSrc", cls.visitId.keys(), "DataFrame")
122 def setUp(self):
123 super().setUp()
124 self.butler = butlerTests.makeTestCollection(self.repo, uniqueId=self.id())
126 def testMockIsr(self):
127 # Testing MockIsrTask is tricky because the real ISR has an unstable
128 # interface with dozens of potential inputs, too many to pass through
129 # runTestQuantum. I don't see a good way to test the inputs;
130 # fortunately, this is unlikely to matter for the overall goal of
131 # testing ap_verify's interaction with the AP pipeline.
132 task = MockIsrTask()
133 pipelineTests.assertValidInitOutput(task)
134 result = task.run(afwImage.ExposureF())
135 pipelineTests.assertValidOutput(task, result)
136 # Skip runTestQuantum
138 def testMockCharacterizeImageTask(self):
139 task = MockCharacterizeImageTask()
140 pipelineTests.assertValidInitOutput(task)
141 result = task.run(afwImage.ExposureF())
142 pipelineTests.assertValidOutput(task, result)
144 self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId)
145 quantum = pipelineTests.makeQuantum(
146 task, self.butler, self.visitId,
147 {"exposure": self.exposureId,
148 "characterized": self.visitId,
149 "sourceCat": self.visitId,
150 "backgroundModel": self.visitId,
151 })
152 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
154 def testMockCalibrateTask(self):
155 task = MockCalibrateTask()
156 pipelineTests.assertValidInitOutput(task)
157 # Even the real CalibrateTask won't pass assertValidOutput, because for
158 # some reason the outputs are injected in runQuantum rather than run.
160 self.butler.put(afwImage.ExposureF(), "icExp", self.visitId)
161 self.butler.put(afwMath.BackgroundList(), "icExpBackground", self.visitId)
162 self.butler.put(afwTable.SourceCatalog(), "icSrc", self.visitId)
163 self.butler.put(afwTable.SimpleCatalog(), "gaia_dr2_20200414", self.htmId)
164 self.butler.put(afwTable.SimpleCatalog(), "ps1_pv3_3pi_20170110", self.htmId)
165 quantum = pipelineTests.makeQuantum(
166 task, self.butler, self.visitId,
167 {"exposure": self.visitId,
168 "background": self.visitId,
169 "icSourceCat": self.visitId,
170 "astromRefCat": [self.htmId],
171 "photoRefCat": [self.htmId],
172 "outputExposure": self.visitId,
173 "outputCat": self.visitId,
174 "outputBackground": self.visitId,
175 "matches": self.visitId,
176 "matchesDenormalized": self.visitId,
177 })
178 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
180 def testMockImageDifferenceTask(self):
181 task = MockImageDifferenceTask()
182 pipelineTests.assertValidInitOutput(task)
183 result = task.run(afwImage.ExposureF(), templateExposure=afwImage.ExposureF())
184 pipelineTests.assertValidOutput(task, result)
186 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
187 skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig())
188 self.butler.put(skymap, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, self.skymapId)
189 self.butler.put(afwImage.ExposureF(), "deepCoadd", self.patchId)
190 self.butler.put(afwImage.ExposureF(), "dcrCoadd", self.subfilterId)
191 quantum = pipelineTests.makeQuantum(
192 task, self.butler, self.skymapVisitId,
193 {"exposure": self.visitId,
194 "skyMap": self.skymapId,
195 "coaddExposures": [self.patchId],
196 "dcrCoadds": [self.subfilterId],
197 "subtractedExposure": self.visitId,
198 "scoreExposure": self.visitId,
199 "template": self.visitId,
200 "matchedExposure": self.visitId,
201 "diaSources": self.visitId,
202 })
203 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
205 def testMockTransformDiaSourceCatalogTask(self):
206 task = MockTransformDiaSourceCatalogTask(initInputs=afwTable.SourceCatalog())
207 pipelineTests.assertValidInitOutput(task)
208 result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42)
209 pipelineTests.assertValidOutput(task, result)
211 self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
212 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
213 quantum = pipelineTests.makeQuantum(
214 task, self.butler, self.visitId,
215 {"diaSourceCat": self.visitId,
216 "diffIm": self.visitId,
217 "diaSourceTable": self.visitId,
218 })
219 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
221 def testMockDiaPipelineTask(self):
222 config = MockDiaPipelineTask.ConfigClass()
223 config.apdb.db_url = "testing_only"
224 task = MockDiaPipelineTask(config=config)
225 pipelineTests.assertValidInitOutput(task)
226 result = task.run(pandas.DataFrame(), pandas.DataFrame(), afwImage.ExposureF(),
227 afwImage.ExposureF(), afwImage.ExposureF(), 42, 'k')
228 pipelineTests.assertValidOutput(task, result)
230 self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId)
231 self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId)
232 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
233 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
234 self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId)
235 quantum = pipelineTests.makeQuantum(
236 task, self.butler, self.visitId,
237 {"diaSourceTable": self.visitId,
238 "solarSystemObjectTable": self.visitId,
239 "diffIm": self.visitId,
240 "exposure": self.visitId,
241 "template": self.visitId,
242 "apdbMarker": self.visitId,
243 "associatedDiaSources": self.visitId,
244 })
245 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
248class MemoryTester(lsst.utils.tests.MemoryTestCase):
249 pass
252def setup_module(module):
253 lsst.utils.tests.init()
256if __name__ == "__main__": 256 ↛ 257line 256 didn't jump to line 257, because the condition on line 256 was never true
257 lsst.utils.tests.init()
258 unittest.main()