Coverage for tests/test_pipelines.py: 43%
61 statements
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-13 04:56 +0000
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-13 04:56 +0000
1# This file is part of drp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for tracking edits to drp_pipe pipelines.
23"""
25import os
26import tempfile
27import unittest
28from typing import Any
30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig
31from lsst.daf.butler.tests import DatastoreMock
32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester
35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")
36TEST_DIR = os.path.abspath(os.path.dirname(__file__))
38# mostly common inputs
39COMMON_INPUTS = {
40 "bfKernel",
41 "bias",
42 "camera",
43 "crosstalk",
44 "dark",
45 "flat",
46 "fringe",
47 "isrOverscanCorrected",
48 "raw",
49 "skyMap",
50}
52# HSC common inputs, in addition to COMMON_INPUTS
53HSC_INPUTS = {
54 "brightObjectMask",
55 "brighterFatterKernel",
56 "defects",
57 "gaia_dr2_20200414",
58 "linearizer",
59 "ps1_pv3_3pi_20170110",
60 "sky",
61 "transmission_atmosphere",
62 "transmission_filter",
63 "transmission_optics",
64 "transmission_sensor",
65 "yBackground",
66}
68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS
69LSSTCAM_IMSIM_INPUTS = {
70 "bfk",
71 "cal_ref_cat_2_2",
72}
74# a selection of mostly common outputs
75COMMON_OUTPUTS = {
76 "calexp",
77 "calexpBackground",
78 "ccdVisitTable",
79 "deepCoadd",
80 "deepCoadd_calexp",
81 "deepCoadd_calexp_background",
82 "deepCoadd_det",
83 "deepCoadd_directWarp",
84 "deepCoadd_forced_src",
85 "deepCoadd_inputMap",
86 "deepCoadd_meas",
87 "deepCoadd_measMatch",
88 "deepCoadd_measMatchFull",
89 "deepCoadd_mergeDet",
90 "deepCoadd_nImage",
91 "deepCoadd_obj",
92 "deepCoadd_psfMatchedWarp",
93 "deepCoadd_ref",
94 "deepCoadd_scarletModelData",
95 "finalized_src_table",
96 "finalVisitSummary",
97 "icExp",
98 "icExpBackground",
99 "icSrc",
100 "isolated_star_cat",
101 "isolated_star_sources",
102 "objectTable",
103 "objectTable_tract",
104 "postISRCCD",
105 "source",
106 "sourceTable",
107 "sourceTable_visit",
108 "src",
109 "srcMatch",
110 "visitSummary",
111 "visitTable",
112}
114# HSC common outputs, in addition to COMMON_OUTPUTS
115HSC_OUTPUTS = {
116 "calexp_camera",
117 "forced_src",
118 "preSource",
119 "preSourceTable",
120 "preSourceTable_visit",
121 "skyCorr",
122 "srcMatchFull",
123}
125# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS
126LSSTCAM_IMSIM_OUTPUTS = {
127 "diaObjectTable_tract",
128 "diaSourceTable",
129 "diaSourceTable_tract",
130 "forcedSourceOnDiaObjectTable",
131 "forcedSourceOnDiaObjectTable_tract",
132 "forcedSourceTable",
133 "forcedSourceTable_tract",
134 "forced_diff",
135 "forced_diff_diaObject",
136 "forced_src",
137 "forced_src_diaObject",
138 "goodSeeingCoadd",
139 "goodSeeingCoadd_nImage",
140 "goodSeeingDiff_assocDiaSrcTable",
141 "goodSeeingDiff_diaObjTable",
142 "goodSeeingDiff_diaSrc",
143 "goodSeeingDiff_diaSrcTable",
144 "goodSeeingDiff_differenceExp",
145 "goodSeeingDiff_differenceTempExp",
146 "goodSeeingDiff_fullDiaObjTable",
147 "goodSeeingDiff_matchedExp",
148 "goodSeeingDiff_templateExp",
149 "goodSeeingVisits",
150 "mergedForcedSource",
151 "mergedForcedSourceOnDiaObject",
152}
155class PipelineTestCase(unittest.TestCase):
156 def setUp(self):
157 self.root = makeTestTempDir(TEST_DIR)
158 self.maxDiff = None
160 def tearDown(self):
161 removeTestTempDir(self.root)
163 def makeButler(self, **kwargs: Any) -> Butler:
164 """Return new Butler instance on each call."""
165 config = ButlerConfig()
167 # make separate temporary directory for registry of this instance
168 tmpdir = tempfile.mkdtemp(dir=self.root)
169 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
170 config["root"] = self.root
172 # have to make a registry first
173 registryConfig = RegistryConfig(config.get("registry"))
174 Registry.createFromConfig(registryConfig)
176 butler = Butler(config, **kwargs)
177 DatastoreMock.apply(butler)
178 return butler
180 def test_decam_drp_merian(self):
181 butler = self.makeButler(writeable=True)
182 tester0 = PipelineStepTester(
183 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
184 ["#step0"],
185 [
186 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
187 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
188 ],
189 expected_inputs={
190 "camera",
191 "raw",
192 },
193 expected_outputs={"overscanRaw"},
194 )
195 tester0.run(butler, self)
196 tester = PipelineStepTester(
197 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
198 [
199 "#step1",
200 "#step2a",
201 "#step2b",
202 "#step2d",
203 "#step2e",
204 "#step3",
205 ],
206 [
207 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
208 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
209 ],
210 expected_inputs=COMMON_INPUTS
211 | {
212 "defects",
213 "gaia_dr2_20200414",
214 "linearizer",
215 "overscanRaw",
216 "ps1_pv3_3pi_20170110",
217 },
218 expected_outputs=COMMON_OUTPUTS
219 | {
220 "goodSeeingCoadd",
221 "goodSeeingCoadd_nImage",
222 "goodSeeingVisits",
223 "preSource",
224 "preSourceTable",
225 "preSourceTable_visit",
226 "srcMatchFull",
227 },
228 )
229 tester.run(butler, self)
231 def test_hsc_drp_ci_hsc(self):
232 butler = self.makeButler(writeable=True)
233 tester = PipelineStepTester(
234 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),
235 [""],
236 [
237 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
238 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
239 ],
240 expected_inputs=COMMON_INPUTS
241 | HSC_INPUTS
242 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},
243 expected_outputs=COMMON_OUTPUTS
244 | HSC_OUTPUTS
245 | {
246 "diaObjectTable_tract",
247 "diaSourceTable",
248 "diaSourceTable_tract",
249 "forcedSourceOnDiaObjectTable",
250 "forcedSourceOnDiaObjectTable_tract",
251 "forcedSourceTable",
252 "forcedSourceTable_tract",
253 "forced_diff",
254 "forced_diff_diaObject",
255 "forced_src_diaObject",
256 "goodSeeingCoadd",
257 "goodSeeingCoadd_nImage",
258 "goodSeeingDiff_assocDiaSrcTable",
259 "goodSeeingDiff_diaObjTable",
260 "goodSeeingDiff_diaSrc",
261 "goodSeeingDiff_diaSrcTable",
262 "goodSeeingDiff_differenceExp",
263 "goodSeeingDiff_differenceTempExp",
264 "goodSeeingDiff_fullDiaObjTable",
265 "goodSeeingDiff_matchedExp",
266 "goodSeeingDiff_templateExp",
267 "goodSeeingVisits",
268 "mergedForcedSource",
269 "mergedForcedSourceOnDiaObject",
270 "objectTable_tract_astrometryRefCat_match",
271 "sourceTable_visit_astrometryRefCat_match",
272 },
273 )
274 tester.run(butler, self)
276 def test_hsc_drp_prod(self):
277 butler = self.makeButler(writeable=True)
278 tester = PipelineStepTester(
279 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),
280 [
281 "#step1",
282 "#step2a",
283 "#step2b",
284 "#step2c",
285 "#step2d",
286 "#step2e",
287 "#step3",
288 "#step4",
289 "#step7",
290 ],
291 [
292 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
293 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
294 ],
295 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
296 expected_outputs=COMMON_OUTPUTS
297 | HSC_OUTPUTS
298 | {
299 "fgcmAtmosphereParameters6",
300 "fgcmFitParameters0",
301 "fgcmFitParameters1",
302 "fgcmFitParameters2",
303 "fgcmFitParameters3",
304 "fgcmFitParameters4",
305 "fgcmFitParameters5",
306 "fgcmFitParameters6",
307 "fgcmFlaggedStars0",
308 "fgcmFlaggedStars1",
309 "fgcmFlaggedStars2",
310 "fgcmFlaggedStars3",
311 "fgcmFlaggedStars4",
312 "fgcmFlaggedStars5",
313 "fgcmFlaggedStars6",
314 "fgcmReferenceStars",
315 "fgcmStandardStars6",
316 "fgcmStarIds",
317 "fgcmStarIndices",
318 "fgcmStarObservations",
319 "fgcmZeropoints6",
320 "transmission_atmosphere_fgcm",
321 },
322 )
323 tester.run(butler, self)
325 def test_hsc_drp_rc2(self):
326 butler = self.makeButler(writeable=True)
327 tester = PipelineStepTester(
328 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),
329 [
330 "#step1",
331 "#step2a",
332 "#step2b",
333 "#step2cde",
334 "#step3",
335 "#step4",
336 "#step5",
337 "#step6",
338 "#step7",
339 ],
340 [
341 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
342 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
343 ],
344 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
345 expected_outputs=COMMON_OUTPUTS
346 | HSC_OUTPUTS
347 | {
348 "diaObjectTable_tract",
349 "diaSourceTable",
350 "diaSourceTable_tract",
351 "fgcmAtmosphereParameters4",
352 "fgcmFitParameters0",
353 "fgcmFitParameters1",
354 "fgcmFitParameters2",
355 "fgcmFitParameters3",
356 "fgcmFitParameters4",
357 "fgcmFlaggedStars0",
358 "fgcmFlaggedStars1",
359 "fgcmFlaggedStars2",
360 "fgcmFlaggedStars3",
361 "fgcmFlaggedStars4",
362 "fgcmReferenceStars",
363 "fgcmStandardStars4",
364 "fgcmStarIds",
365 "fgcmStarIndices",
366 "fgcmStarObservations",
367 "fgcmZeropoints4",
368 "forcedSourceOnDiaObjectTable",
369 "forcedSourceOnDiaObjectTable_tract",
370 "forcedSourceTable",
371 "forcedSourceTable_tract",
372 "forced_diff",
373 "forced_diff_diaObject",
374 "forced_src_diaObject",
375 "goodSeeingCoadd",
376 "goodSeeingCoadd_nImage",
377 "goodSeeingDiff_assocDiaSrcTable",
378 "goodSeeingDiff_diaObjTable",
379 "goodSeeingDiff_diaSrc",
380 "goodSeeingDiff_diaSrcTable",
381 "goodSeeingDiff_differenceExp",
382 "goodSeeingDiff_differenceTempExp",
383 "goodSeeingDiff_fullDiaObjTable",
384 "goodSeeingDiff_matchedExp",
385 "goodSeeingDiff_templateExp",
386 "goodSeeingVisits",
387 "mergedForcedSource",
388 "mergedForcedSourceOnDiaObject",
389 "transmission_atmosphere_fgcm",
390 },
391 )
392 tester.run(butler, self)
394 def test_lsstcam_imsim_drp_ci_imsim(self):
395 butler = self.makeButler(writeable=True)
396 tester = PipelineStepTester(
397 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),
398 [f"#step{N}" for N in range(1, 8)],
399 [
400 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
401 ],
402 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
403 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
404 )
405 tester.run(butler, self)
407 def test_lsstcam_imsim_drp_test_med_1(self):
408 butler = self.makeButler(writeable=True)
409 tester = PipelineStepTester(
410 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),
411 [f"#step{N}" for N in range(1, 8)],
412 [
413 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
414 ],
415 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
416 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
417 )
418 tester.run(butler, self)
421if __name__ == "__main__": 421 ↛ 422line 421 didn't jump to line 422, because the condition on line 421 was never true
422 unittest.main()