Coverage for tests/test_pipelines.py: 42%
65 statements
« prev ^ index » next coverage.py v7.2.6, created at 2023-05-24 02:46 -0700
« prev ^ index » next coverage.py v7.2.6, created at 2023-05-24 02:46 -0700
1# This file is part of drp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for tracking edits to drp_pipe pipelines.
23"""
25import os
26import tempfile
27import unittest
28from typing import Any
30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig
31from lsst.daf.butler.tests import DatastoreMock
32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester
35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")
36TEST_DIR = os.path.abspath(os.path.dirname(__file__))
38# mostly common inputs
39COMMON_INPUTS = {
40 "bfKernel",
41 "bias",
42 "camera",
43 "crosstalk",
44 "dark",
45 "flat",
46 "fringe",
47 "isrOverscanCorrected",
48 "raw",
49 "skyMap",
50}
52# HSC common inputs, in addition to COMMON_INPUTS
53HSC_INPUTS = {
54 "brightObjectMask",
55 "brighterFatterKernel",
56 "defects",
57 "gaia_dr2_20200414",
58 "linearizer",
59 "ps1_pv3_3pi_20170110",
60 "sky",
61 "transmission_atmosphere",
62 "transmission_filter",
63 "transmission_optics",
64 "transmission_sensor",
65 "yBackground",
66}
68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS
69LSSTCAM_IMSIM_INPUTS = {
70 "bfk",
71 "cal_ref_cat_2_2",
72 "truth_summary",
73}
75# a selection of mostly common outputs
76COMMON_OUTPUTS = {
77 "calexp",
78 "calexpBackground",
79 "ccdVisitTable",
80 "deepCoadd",
81 "deepCoadd_calexp",
82 "deepCoadd_calexp_background",
83 "deepCoadd_det",
84 "deepCoadd_directWarp",
85 "deepCoadd_forced_src",
86 "deepCoadd_inputMap",
87 "deepCoadd_meas",
88 "deepCoadd_measMatch",
89 "deepCoadd_measMatchFull",
90 "deepCoadd_mergeDet",
91 "deepCoadd_nImage",
92 "deepCoadd_obj",
93 "deepCoadd_psfMatchedWarp",
94 "deepCoadd_ref",
95 "deepCoadd_scarletModelData",
96 "finalized_src_table",
97 "finalVisitSummary",
98 "icExp",
99 "icExpBackground",
100 "icSrc",
101 "isolated_star_cat",
102 "isolated_star_sources",
103 "objectTable",
104 "objectTable_tract",
105 "postISRCCD",
106 "source",
107 "sourceTable",
108 "sourceTable_visit",
109 "src",
110 "srcMatch",
111 "visitSummary",
112 "visitTable",
113}
115# HSC common outputs, in addition to COMMON_OUTPUTS
116HSC_OUTPUTS = {
117 "calexp_skyCorr_visit_mosaic",
118 "calexpBackground_skyCorr_visit_mosaic",
119 "forced_src",
120 "preSource",
121 "preSourceTable",
122 "preSourceTable_visit",
123 "skyCorr",
124 "srcMatchFull",
125}
127# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS
128LSSTCAM_IMSIM_OUTPUTS = {
129 "diaObjectTable_tract",
130 "diaSourceTable",
131 "diaSourceTable_tract",
132 "forcedSourceOnDiaObjectTable",
133 "forcedSourceOnDiaObjectTable_tract",
134 "forcedSourceTable",
135 "forcedSourceTable_tract",
136 "forced_diff",
137 "forced_diff_diaObject",
138 "forced_src",
139 "forced_src_diaObject",
140 "goodSeeingCoadd",
141 "goodSeeingCoadd_nImage",
142 "goodSeeingDiff_assocDiaSrcTable",
143 "goodSeeingDiff_diaObjTable",
144 "goodSeeingDiff_diaSrc",
145 "goodSeeingDiff_diaSrcTable",
146 "goodSeeingDiff_differenceExp",
147 "goodSeeingDiff_differenceTempExp",
148 "goodSeeingDiff_fullDiaObjTable",
149 "goodSeeingDiff_matchedExp",
150 "goodSeeingDiff_templateExp",
151 "goodSeeingVisits",
152 "mergedForcedSource",
153 "mergedForcedSourceOnDiaObject",
154}
157class PipelineTestCase(unittest.TestCase):
158 def setUp(self):
159 self.root = makeTestTempDir(TEST_DIR)
160 self.maxDiff = None
162 def tearDown(self):
163 removeTestTempDir(self.root)
165 def makeButler(self, **kwargs: Any) -> Butler:
166 """Return new Butler instance on each call."""
167 config = ButlerConfig()
169 # make separate temporary directory for registry of this instance
170 tmpdir = tempfile.mkdtemp(dir=self.root)
171 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
172 config["root"] = self.root
174 # have to make a registry first
175 registryConfig = RegistryConfig(config.get("registry"))
176 Registry.createFromConfig(registryConfig)
178 butler = Butler(config, **kwargs)
179 DatastoreMock.apply(butler)
180 return butler
182 def test_decam_drp_merian(self):
183 butler = self.makeButler(writeable=True)
184 tester0 = PipelineStepTester(
185 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
186 ["#step0"],
187 [
188 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
189 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
190 ],
191 expected_inputs={
192 "camera",
193 "raw",
194 },
195 expected_outputs={"overscanRaw"},
196 )
197 tester0.run(butler, self)
198 tester = PipelineStepTester(
199 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),
200 [
201 "#step1",
202 "#step2a",
203 "#step2b",
204 "#step2d",
205 "#step2e",
206 "#step3",
207 ],
208 [
209 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
210 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
211 ],
212 expected_inputs=COMMON_INPUTS
213 | {
214 "defects",
215 "gaia_dr2_20200414",
216 "linearizer",
217 "overscanRaw",
218 "ps1_pv3_3pi_20170110",
219 },
220 expected_outputs=COMMON_OUTPUTS
221 | {
222 "goodSeeingCoadd",
223 "goodSeeingCoadd_nImage",
224 "goodSeeingVisits",
225 "preSource",
226 "preSourceTable",
227 "preSourceTable_visit",
228 "srcMatchFull",
229 },
230 )
231 tester.run(butler, self)
233 def test_hsc_drp_ci_hsc(self):
234 butler = self.makeButler(writeable=True)
235 tester = PipelineStepTester(
236 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),
237 [""],
238 [
239 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
240 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
241 ],
242 expected_inputs=COMMON_INPUTS
243 | HSC_INPUTS
244 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},
245 expected_outputs=COMMON_OUTPUTS
246 | HSC_OUTPUTS
247 | {
248 "diaObjectTable_tract",
249 "diaSourceTable",
250 "diaSourceTable_tract",
251 "forcedSourceOnDiaObjectTable",
252 "forcedSourceOnDiaObjectTable_tract",
253 "forcedSourceTable",
254 "forcedSourceTable_tract",
255 "forced_diff",
256 "forced_diff_diaObject",
257 "forced_src_diaObject",
258 "goodSeeingCoadd",
259 "goodSeeingCoadd_nImage",
260 "goodSeeingDiff_assocDiaSrcTable",
261 "goodSeeingDiff_diaObjTable",
262 "goodSeeingDiff_diaSrc",
263 "goodSeeingDiff_diaSrcTable",
264 "goodSeeingDiff_differenceExp",
265 "goodSeeingDiff_differenceTempExp",
266 "goodSeeingDiff_fullDiaObjTable",
267 "goodSeeingDiff_matchedExp",
268 "goodSeeingDiff_templateExp",
269 "goodSeeingVisits",
270 "mergedForcedSource",
271 "mergedForcedSourceOnDiaObject",
272 "objectTable_tract_astrometryRefCat_match",
273 "sourceTable_visit_astrometryRefCat_match",
274 },
275 )
276 tester.run(butler, self)
278 def test_hsc_drp_prod(self):
279 butler = self.makeButler(writeable=True)
280 tester = PipelineStepTester(
281 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),
282 [
283 "#step1",
284 "#step2a",
285 "#step2b",
286 "#step2c",
287 "#step2d",
288 "#step2e",
289 "#step3",
290 "#step4",
291 "#step7",
292 ],
293 [
294 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
295 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
296 ],
297 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
298 expected_outputs=COMMON_OUTPUTS
299 | HSC_OUTPUTS
300 | {
301 "fgcmAtmosphereParameters6",
302 "fgcmFitParameters0",
303 "fgcmFitParameters1",
304 "fgcmFitParameters2",
305 "fgcmFitParameters3",
306 "fgcmFitParameters4",
307 "fgcmFitParameters5",
308 "fgcmFitParameters6",
309 "fgcmFlaggedStars0",
310 "fgcmFlaggedStars1",
311 "fgcmFlaggedStars2",
312 "fgcmFlaggedStars3",
313 "fgcmFlaggedStars4",
314 "fgcmFlaggedStars5",
315 "fgcmFlaggedStars6",
316 "fgcm_reference_stars",
317 "fgcmStandardStars6",
318 "fgcm_star_ids",
319 "fgcm_star_observations",
320 "fgcmZeropoints6",
321 "transmission_atmosphere_fgcm",
322 },
323 )
324 tester.run(butler, self)
326 def test_hsc_drp_rc2(self):
327 butler = self.makeButler(writeable=True)
328 tester = PipelineStepTester(
329 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),
330 [
331 "#step1",
332 "#step2a",
333 "#step2b",
334 "#step2cde",
335 "#step3",
336 "#step4",
337 "#step5",
338 "#step6",
339 "#step7",
340 ],
341 [
342 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
343 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
344 ],
345 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
346 expected_outputs=COMMON_OUTPUTS
347 | HSC_OUTPUTS
348 | {
349 "diaObjectTable_tract",
350 "diaSourceTable",
351 "diaSourceTable_tract",
352 "fgcmAtmosphereParameters4",
353 "fgcmFitParameters0",
354 "fgcmFitParameters1",
355 "fgcmFitParameters2",
356 "fgcmFitParameters3",
357 "fgcmFitParameters4",
358 "fgcmFlaggedStars0",
359 "fgcmFlaggedStars1",
360 "fgcmFlaggedStars2",
361 "fgcmFlaggedStars3",
362 "fgcmFlaggedStars4",
363 "fgcm_reference_stars",
364 "fgcmStandardStars4",
365 "fgcm_star_ids",
366 "fgcm_star_observations",
367 "fgcmZeropoints4",
368 "forcedSourceOnDiaObjectTable",
369 "forcedSourceOnDiaObjectTable_tract",
370 "forcedSourceTable",
371 "forcedSourceTable_tract",
372 "forced_diff",
373 "forced_diff_diaObject",
374 "forced_src_diaObject",
375 "goodSeeingCoadd",
376 "goodSeeingCoadd_nImage",
377 "goodSeeingDiff_assocDiaSrcTable",
378 "goodSeeingDiff_diaObjTable",
379 "goodSeeingDiff_diaSrc",
380 "goodSeeingDiff_diaSrcTable",
381 "goodSeeingDiff_differenceExp",
382 "goodSeeingDiff_differenceTempExp",
383 "goodSeeingDiff_fullDiaObjTable",
384 "goodSeeingDiff_matchedExp",
385 "goodSeeingDiff_templateExp",
386 "goodSeeingVisits",
387 "mergedForcedSource",
388 "mergedForcedSourceOnDiaObject",
389 "transmission_atmosphere_fgcm",
390 },
391 )
392 tester.run(butler, self)
394 def test_hsc_drp_rc2_subset(self):
395 butler = self.makeButler(writeable=True)
396 tester = PipelineStepTester(
397 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),
398 [
399 "#nightlyStep1",
400 "#nightlyStep2a",
401 "#nightlyStep2b",
402 "#nightlyStep2c",
403 "#nightlyStep2d",
404 "#nightlyStep3",
405 "#nightlyStep4",
406 "#nightlyStep5",
407 ],
408 [
409 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False),
410 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False),
411 ],
412 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},
413 expected_outputs=COMMON_OUTPUTS
414 | HSC_OUTPUTS
415 | {
416 "fgcmAtmosphereParameters4",
417 "fgcmFitParameters0",
418 "fgcmFitParameters1",
419 "fgcmFitParameters2",
420 "fgcmFitParameters3",
421 "fgcmFitParameters4",
422 "fgcmFlaggedStars0",
423 "fgcmFlaggedStars1",
424 "fgcmFlaggedStars2",
425 "fgcmFlaggedStars3",
426 "fgcmFlaggedStars4",
427 "fgcm_reference_stars",
428 "fgcmStandardStars4",
429 "fgcm_star_ids",
430 "fgcm_star_observations",
431 "fgcmZeropoints4",
432 "transmission_atmosphere_fgcm",
433 },
434 )
435 tester.run(butler, self)
437 def test_lsstcam_imsim_drp_ci_imsim(self):
438 butler = self.makeButler(writeable=True)
439 tester = PipelineStepTester(
440 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),
441 [f"#step{N}" for N in range(1, 9)],
442 [
443 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
444 ],
445 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
446 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
447 )
448 tester.run(butler, self)
450 def test_lsstcam_imsim_drp_test_med_1(self):
451 butler = self.makeButler(writeable=True)
452 tester = PipelineStepTester(
453 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),
454 [f"#step{N}" for N in range(1, 9)],
455 [
456 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False),
457 ],
458 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,
459 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,
460 )
461 tester.run(butler, self)
464if __name__ == "__main__": 464 ↛ 465line 464 didn't jump to line 465, because the condition on line 464 was never true
465 unittest.main()