Coverage for tests/test_pipelines.py: 43%
61 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-18 03:18 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-18 03:18 -0800
1# This file is part of drp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for tracking edits to drp_pipe pipelines.
23"""
25import os
26import tempfile
27import unittest
28from typing import Any
30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig
31from lsst.daf.butler.tests import DatastoreMock
32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester
35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")
36TEST_DIR = os.path.abspath(os.path.dirname(__file__))
38# mostly common inputs
39COMMON_INPUTS = {
40 "bfKernel",
41 "bias",
42 "camera",
43 "crosstalk",
44 "dark",
45 "flat",
46 "fringe",
47 "isrOverscanCorrected",
48 "raw",
49 "skyMap",
50}
52# HSC common inputs, in addition to COMMON_INPUTS
53HSC_INPUTS = {
54 "brightObjectMask",
55 "brighterFatterKernel",
56 "defects",
57 "gaia_dr2_20200414",
58 "linearizer",
59 "ps1_pv3_3pi_20170110",
60 "sky",
61 "transmission_atmosphere",
62 "transmission_filter",
63 "transmission_optics",
64 "transmission_sensor",
65 "yBackground",
66}
68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS
69LSSTCAM_IMSIM_INPUTS = {
70 "bfk",
71 "cal_ref_cat_2_2",
72}
74# a selection of mostly common outputs
75COMMON_OUTPUTS = {
76 "calexp",
77 "calexpBackground",
78 "ccdVisitTable",
79 "deepCoadd",
80 "deepCoadd_calexp",
81 "deepCoadd_calexp_background",
82 "deepCoadd_det",
83 "deepCoadd_directWarp",
84 "deepCoadd_forced_src",
85 "deepCoadd_inputMap",
86 "deepCoadd_meas",
87 "deepCoadd_measMatch",
88 "deepCoadd_measMatchFull",
89 "deepCoadd_mergeDet",
90 "deepCoadd_nImage",
91 "deepCoadd_obj",
92 "deepCoadd_psfMatchedWarp",
93 "deepCoadd_ref",
94 "deepCoadd_scarletModelData",
95 "finalized_src_table",
96 "finalVisitSummary",
97 "icExp",
98 "icExpBackground",
99 "icSrc",
100 "isolated_star_cat",
101 "isolated_star_sources",
102 "objectTable",
103 "objectTable_tract",
104 "postISRCCD",
105 "source",
106 "sourceTable",
107 "sourceTable_visit",
108 "src",
109 "srcMatch",
110 "visitSummary",
111 "visitTable",
112}
114# HSC common outputs, in addition to COMMON_OUTPUTS
115HSC_OUTPUTS = {
116 "calexp_skyCorr_visit_mosaic",
117 "calexpBackground_skyCorr_visit_mosaic",
118 "forced_src",
119 "preSource",
120 "preSourceTable",
121 "preSourceTable_visit",
122 "skyCorr",
123 "srcMatchFull",
124}
126# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS
127LSSTCAM_IMSIM_OUTPUTS = {
128 "diaObjectTable_tract",
129 "diaSourceTable",
130 "diaSourceTable_tract",
131 "forcedSourceOnDiaObjectTable",
132 "forcedSourceOnDiaObjectTable_tract",
133 "forcedSourceTable",
134 "forcedSourceTable_tract",
135 "forced_diff",
136 "forced_diff_diaObject",
137 "forced_src",
138 "forced_src_diaObject",
139 "goodSeeingCoadd",
140 "goodSeeingCoadd_nImage",
141 "goodSeeingDiff_assocDiaSrcTable",
142 "goodSeeingDiff_diaObjTable",
143 "goodSeeingDiff_diaSrc",
144 "goodSeeingDiff_diaSrcTable",
145 "goodSeeingDiff_differenceExp",
146 "goodSeeingDiff_differenceTempExp",
147 "goodSeeingDiff_fullDiaObjTable",
148 "goodSeeingDiff_matchedExp",
149 "goodSeeingDiff_templateExp",
150 "goodSeeingVisits",
151 "mergedForcedSource",
152 "mergedForcedSourceOnDiaObject",
153}
156class PipelineTestCase(unittest.TestCase):
157 def setUp(self):
158 self.root = makeTestTempDir(TEST_DIR)
159 self.maxDiff = None
161 def tearDown(self):
162 removeTestTempDir(self.root)
164 def makeButler(self, **kwargs: Any) -> Butler:
165 """Return new Butler instance on each call."""
166 config = ButlerConfig()
168 # make separate temporary directory for registry of this instance
169 tmpdir = tempfile.mkdtemp(dir=self.root)
170 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
171 config["root"] = self.root
173 # have to make a registry first
174 registryConfig = RegistryConfig(config.get("registry"))
175 Registry.createFromConfig(registryConfig)
177 butler = Butler(config, **kwargs)
178 DatastoreMock.apply(butler)
179 return butler
181 def test_decam_drp_merian(self):
182 butler = self.makeButler(writeable=True)
183 tester0 = PipelineStepTester(
184 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
185 ["#step0"],
186 [
187 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
188 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
189 ],
190 expected_inputs={
191 "camera",
192 "raw",
193 },
194 expected_outputs={"overscanRaw"},
195 )
196 tester0.run(butler, self)
197 tester = PipelineStepTester(
198 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
199 [
200 "#step1",
201 "#step2a",
202 "#step2b",
203 "#step2d",
204 "#step2e",
205 "#step3",
206 ],
207 [
208 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
209 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
210 ],
211 expected_inputs=COMMON_INPUTS
212 | {
213 "defects",
214 "gaia_dr2_20200414",
215 "linearizer",
216 "overscanRaw",
217 "ps1_pv3_3pi_20170110",
218 },
219 expected_outputs=COMMON_OUTPUTS
220 | {
221 "goodSeeingCoadd",
222 "goodSeeingCoadd_nImage",
223 "goodSeeingVisits",
224 "preSource",
225 "preSourceTable",
226 "preSourceTable_visit",
227 "srcMatchFull",
228 },
229 )
230 tester.run(butler, self)
232 def test_hsc_drp_ci_hsc(self):
233 butler = self.makeButler(writeable=True)
234 tester = PipelineStepTester(
235 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),
236 [""],
237 [
238 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
239 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
240 ],
241 expected_inputs=COMMON_INPUTS
242 | HSC_INPUTS
243 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},
244 expected_outputs=COMMON_OUTPUTS
245 | HSC_OUTPUTS
246 | {
247 "diaObjectTable_tract",
248 "diaSourceTable",
249 "diaSourceTable_tract",
250 "forcedSourceOnDiaObjectTable",
251 "forcedSourceOnDiaObjectTable_tract",
252 "forcedSourceTable",
253 "forcedSourceTable_tract",
254 "forced_diff",
255 "forced_diff_diaObject",
256 "forced_src_diaObject",
257 "goodSeeingCoadd",
258 "goodSeeingCoadd_nImage",
259 "goodSeeingDiff_assocDiaSrcTable",
260 "goodSeeingDiff_diaObjTable",
261 "goodSeeingDiff_diaSrc",
262 "goodSeeingDiff_diaSrcTable",
263 "goodSeeingDiff_differenceExp",
264 "goodSeeingDiff_differenceTempExp",
265 "goodSeeingDiff_fullDiaObjTable",
266 "goodSeeingDiff_matchedExp",
267 "goodSeeingDiff_templateExp",
268 "goodSeeingVisits",
269 "mergedForcedSource",
270 "mergedForcedSourceOnDiaObject",
271 "objectTable_tract_astrometryRefCat_match",
272 "sourceTable_visit_astrometryRefCat_match",
273 },
274 )
275 tester.run(butler, self)
277 def test_hsc_drp_prod(self):
278 butler = self.makeButler(writeable=True)
279 tester = PipelineStepTester(
280 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),
281 [
282 "#step1",
283 "#step2a",
284 "#step2b",
285 "#step2c",
286 "#step2d",
287 "#step2e",
288 "#step3",
289 "#step4",
290 "#step7",
291 ],
292 [
293 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
294 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
295 ],
296 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
297 expected_outputs=COMMON_OUTPUTS
298 | HSC_OUTPUTS
299 | {
300 "fgcmAtmosphereParameters6",
301 "fgcmFitParameters0",
302 "fgcmFitParameters1",
303 "fgcmFitParameters2",
304 "fgcmFitParameters3",
305 "fgcmFitParameters4",
306 "fgcmFitParameters5",
307 "fgcmFitParameters6",
308 "fgcmFlaggedStars0",
309 "fgcmFlaggedStars1",
310 "fgcmFlaggedStars2",
311 "fgcmFlaggedStars3",
312 "fgcmFlaggedStars4",
313 "fgcmFlaggedStars5",
314 "fgcmFlaggedStars6",
315 "fgcmReferenceStars",
316 "fgcmStandardStars6",
317 "fgcmStarIds",
318 "fgcmStarIndices",
319 "fgcmStarObservations",
320 "fgcmZeropoints6",
321 "transmission_atmosphere_fgcm",
322 },
323 )
324 tester.run(butler, self)
326 def test_hsc_drp_rc2(self):
327 butler = self.makeButler(writeable=True)
328 tester = PipelineStepTester(
329 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),
330 [
331 "#step1",
332 "#step2a",
333 "#step2b",
334 "#step2cde",
335 "#step3",
336 "#step4",
337 "#step5",
338 "#step6",
339 "#step7",
340 ],
341 [
342 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
343 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
344 ],
345 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
346 expected_outputs=COMMON_OUTPUTS
347 | HSC_OUTPUTS
348 | {
349 "diaObjectTable_tract",
350 "diaSourceTable",
351 "diaSourceTable_tract",
352 "fgcmAtmosphereParameters4",
353 "fgcmFitParameters0",
354 "fgcmFitParameters1",
355 "fgcmFitParameters2",
356 "fgcmFitParameters3",
357 "fgcmFitParameters4",
358 "fgcmFlaggedStars0",
359 "fgcmFlaggedStars1",
360 "fgcmFlaggedStars2",
361 "fgcmFlaggedStars3",
362 "fgcmFlaggedStars4",
363 "fgcmReferenceStars",
364 "fgcmStandardStars4",
365 "fgcmStarIds",
366 "fgcmStarIndices",
367 "fgcmStarObservations",
368 "fgcmZeropoints4",
369 "forcedSourceOnDiaObjectTable",
370 "forcedSourceOnDiaObjectTable_tract",
371 "forcedSourceTable",
372 "forcedSourceTable_tract",
373 "forced_diff",
374 "forced_diff_diaObject",
375 "forced_src_diaObject",
376 "goodSeeingCoadd",
377 "goodSeeingCoadd_nImage",
378 "goodSeeingDiff_assocDiaSrcTable",
379 "goodSeeingDiff_diaObjTable",
380 "goodSeeingDiff_diaSrc",
381 "goodSeeingDiff_diaSrcTable",
382 "goodSeeingDiff_differenceExp",
383 "goodSeeingDiff_differenceTempExp",
384 "goodSeeingDiff_fullDiaObjTable",
385 "goodSeeingDiff_matchedExp",
386 "goodSeeingDiff_templateExp",
387 "goodSeeingVisits",
388 "mergedForcedSource",
389 "mergedForcedSourceOnDiaObject",
390 "transmission_atmosphere_fgcm",
391 },
392 )
393 tester.run(butler, self)
395 def test_lsstcam_imsim_drp_ci_imsim(self):
396 butler = self.makeButler(writeable=True)
397 tester = PipelineStepTester(
398 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),
399 [f"#step{N}" for N in range(1, 8)],
400 [
401 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
402 ],
403 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
404 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
405 )
406 tester.run(butler, self)
408 def test_lsstcam_imsim_drp_test_med_1(self):
409 butler = self.makeButler(writeable=True)
410 tester = PipelineStepTester(
411 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),
412 [f"#step{N}" for N in range(1, 8)],
413 [
414 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
415 ],
416 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
417 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
418 )
419 tester.run(butler, self)
422if __name__ == "__main__": 422 ↛ 423line 422 didn't jump to line 423, because the condition on line 422 was never true
423 unittest.main()