Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95 "gain_correction",

96}

98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

99LSSTCAM_IMSIM_INPUTS = {

100 "bfk",

101 "cal_ref_cat_2_2",

102 "truth_summary",

103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

104}

105

106# LSSTCam common inputs, in addition to COMMON_INPUTS

107LSSTCAM_INPUTS = {

108 "bfk",

109 "electroBfDistortionMatrix",

110 "linearizer",

111 "defects",

112 "cti",

113 "the_monster_20250219",

114 "gaia_dr3_20230707",

115 "sky",

116 "astrometry_camera",

117 "gain_correction",

118}

119

120# LSSTComCam common inputs, in addition to COMMON_INPUTS

121LSSTCOMCAM_INPUTS = {

122 "bfk",

123 "ptc",

124 "linearizer",

125 "defects",

126 "cti",

127 "the_monster_20250219",

128 "illuminationCorrection",

129 "gain_correction",

130}

131

132# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

133LSSTCOMCAMSIM_INPUTS = {

134 "uw_stars_20240524",

135 "ptc",

136 "bfk",

137 "gain_correction",

138}

139

140# a selection of mostly common outputs

141COMMON_OUTPUTS = {

142 "ccdVisitTable",

143 "compare_warp_artifact_mask",

144 "deepCoaddVisits",

145 "deepCoadd",

146 "deepCoadd_calexp",

147 "deepCoadd_calexp_background",

148 "deepCoadd_det",

149 "deepCoadd_directWarp",

150 "deepCoadd_directWarp_maskedFraction",

151 "deepCoadd_forced_src",

152 "deepCoadd_inputMap",

153 "deepCoadd_meas",

154 "deepCoadd_mergeDet",

155 "deepCoadd_nImage",

156 "deepCoadd_obj",

157 "deepCoadd_psfMatchedWarp",

158 "deepCoadd_ref",

159 "deepCoadd_scarletModelData",

160 "finalized_src_table",

161 "finalVisitSummary",

162 "initial_astrometry_match_detector",

163 "initial_photometry_match_detector",

164 "initial_psf_stars_footprints_detector",

165 "initial_psf_stars_detector",

166 "calexp",

167 "calexpBackground",

168 "src",

169 "objectTable",

170 "objectTable_tract",

171 "postISRCCD",

172 "sourceTable_visit",

173 "visitSummary",

174 "visitTable",

175 "isolated_star_presource_associations",

176 "isolated_star_presources",

177}

178

179# Translated COMMON_OUTPUTS

180COMMON_V2_OUTPUTS = {

181 'visit_detector_table',

182 'compare_warp_artifact_mask',

183 'deep_coadd_visit_selection',

184 'deep_coadd_predetection',

185 'deep_coadd',

186 'deep_coadd_background',

187 'object_detection',

188 'direct_warp',

189 'direct_warp_masked_fraction',

190 'object_forced_measurement',

191 'deep_coadd_input_map',

192 'object_unforced_measurement',

193 'object_detection_merged',

194 'deep_coadd_n_image',

195 'object_unstandardized',

196 'psf_matched_warp',

197 'object_ref_measurement',

198 'object_scarlet_models',

199 'refit_psf_star',

200 'visit_summary',

201 'initial_astrometry_match_detector',

202 'initial_photometry_match_detector',

203 'single_visit_psf_star_footprints',

204 'single_visit_psf_star',

205 'preliminary_visit_image',

206 'preliminary_visit_image_background',

207 'single_visit_star_footprints',

208 'object_patch',

209 'object_all',

210 'post_isr_image',

211 'recalibrated_star',

212 'preliminary_visit_summary',

213 'visit_table',

214 'isolated_star_association',

215 'isolated_star'

216}

217

218# HSC common outputs, in addition to COMMON_OUTPUTS

219HSC_OUTPUTS = {

220 "calexp_skyCorr_visit_mosaic",

221 "calexpBackground_skyCorr_visit_mosaic",

222 "preSourceTable",

223 "preSourceTable_visit",

224 "skyCorr",

225}

226

227# LATISS common outputs, in addition to COMMON_OUTPUTS

228LATISS_OUTPUTS = {

229 "diaObjectTable_tract",

230 "diaSourceTable",

231 "diaSourceTable_tract",

232 "fgcm_Cycle0_FitParameters",

233 "fgcm_Cycle1_FitParameters",

234 "fgcm_Cycle2_FitParameters",

235 "fgcm_Cycle3_FitParameters",

236 "fgcm_Cycle4_FitParameters",

237 "fgcm_Cycle0_FlaggedStars",

238 "fgcm_Cycle1_FlaggedStars",

239 "fgcm_Cycle2_FlaggedStars",

240 "fgcm_Cycle3_FlaggedStars",

241 "fgcm_Cycle4_FlaggedStars",

242 "fgcm_reference_stars",

243 "fgcm_star_ids",

244 "fgcm_star_observations",

245 "fgcm_standard_star",

246 "forcedSourceOnDiaObjectTable",

247 "forcedSourceOnDiaObjectTable_tract",

248 "forcedSourceTable",

249 "forcedSourceTable_tract",

250 "goodSeeingCoadd",

251 "goodSeeingCoadd_nImage",

252 "goodSeeingDiff_assocDiaSrcTable",

253 "goodSeeingDiff_diaObjTable",

254 "goodSeeingDiff_diaSrc",

255 "goodSeeingDiff_diaSrcTable",

256 "goodSeeingDiff_differenceExp",

257 "goodSeeingDiff_differenceTempExp",

258 "goodSeeingDiff_fullDiaObjTable",

259 "goodSeeingDiff_matchedExp",

260 "goodSeeingDiff_templateExp",

261 "goodSeeingVisits",

262 "mergedForcedSource",

263 "mergedForcedSourceOnDiaObject",

264 "transmission_atmosphere_fgcm",

265}

266

267# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

268LSSTCAM_IMSIM_OUTPUTS = {

269 "diaObjectTable_tract",

270 "diaSourceTable",

271 "diaSourceTable_tract",

272 "forcedSourceOnDiaObjectTable",

273 "forcedSourceOnDiaObjectTable_tract",

274 "forcedSourceTable",

275 "forcedSourceTable_tract",

276 "goodSeeingCoadd",

277 "goodSeeingCoadd_nImage",

278 "goodSeeingDiff_assocDiaSrcTable",

279 "goodSeeingDiff_diaObjTable",

280 "goodSeeingDiff_diaSrc",

281 "goodSeeingDiff_diaSrcTable",

282 "goodSeeingDiff_differenceExp",

283 "goodSeeingDiff_differenceTempExp",

284 "goodSeeingDiff_fullDiaObjTable",

285 "goodSeeingDiff_matchedExp",

286 "goodSeeingDiff_templateExp",

287 "goodSeeingVisits",

288 "mergedForcedSource",

289 "mergedForcedSourceOnDiaObject",

290}

291

292# v2 outputs common to all "quickLook" pipelines, which only include

293QUICKLOOK_V2_OUTPUTS = {

294 "preliminary_visit_image", "preliminary_visit_summary"

295}

296

297# Outputs common to all "quickLook" pipelines, which only include

298QUICKLOOK_OUTPUTS = {

299 "calexp", "visitSummary"

300}

301

302QUICKLOOK_BIAS_OUTPUTS = {

303 "post_isr_image",

304}

305

306QUICKLOOK_DARK_OUTPUTS = {

307 "post_isr_image",

308}

309

310QUICKLOOK_FLAT_OUTPUTS = {

311 "post_isr_image",

312}

313

314# All refcats used by any pipelines, for inclusion as the initial_dataset_types

315# argument to PipelineStepTester; the intent is just to allow the 'skypix'

316# dimension placeholder used in these connections to be resolved.

317REFCATS = [

318 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

319 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

320 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

321 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

322 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

323 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

324 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

325]

326

327

328class PipelineTestCase(unittest.TestCase):

329 def setUp(self):

330 self.root = makeTestTempDir(TEST_DIR)

331 self.maxDiff = None

332

333 def tearDown(self):

334 removeTestTempDir(self.root)

335

336 def makeButler(self, **kwargs: Any) -> Butler:

337 """Return new Butler instance on each call."""

338 config = Config()

339

340 # make separate temporary directory for registry of this instance

341 tmpdir = tempfile.mkdtemp(dir=self.root)

342 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

343 config = Butler.makeRepo(self.root, config)

344 butler = Butler.from_config(config, **kwargs)

345 self.enterContext(butler)

346 DatastoreMock.apply(butler)

347 return butler

348

349 def test_decam_isrForCrosstalkSources(self):

350 butler = self.makeButler(writeable=True)

351 tester = PipelineStepTester(

352 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

353 ["#step0"],

354 initial_dataset_types=REFCATS,

355 expected_inputs={

356 "camera",

357 "raw",

358 },

359 expected_outputs={"overscanRaw"},

360 )

361 tester.run(butler, self)

362

363 def test_decam_drp_merian(self):

364 butler = self.makeButler(writeable=True)

365 tester = PipelineStepTester(

366 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

367 [

368 "#step1",

369 "#step2a",

370 "#step2b",

371 "#step2d",

372 "#step2e",

373 "#step3",

374 ],

375 initial_dataset_types=REFCATS,

376 expected_inputs=COMMON_INPUTS

377 | {

378 "defects",

379 "gaia_dr2_20200414",

380 "gaia_dr3_20230707",

381 "linearizer",

382 "overscanRaw",

383 "ps1_pv3_3pi_20170110",

384 "sky",

385 },

386 expected_outputs=COMMON_OUTPUTS

387 | {

388 "goodSeeingCoadd",

389 "goodSeeingCoadd_nImage",

390 "goodSeeingVisits",

391 "preSourceTable",

392 "preSourceTable_visit",

393 },

394 )

395 tester.run(butler, self)

396

397 def test_hsc_drp_ci_hsc(self):

398 butler = self.makeButler(writeable=True)

399 tester = PipelineStepTester(

400 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

401 [""],

402 initial_dataset_types=REFCATS,

403 expected_inputs=COMMON_INPUTS

404 | HSC_INPUTS

405 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

406 expected_outputs=COMMON_OUTPUTS

407 | HSC_OUTPUTS

408 | {

409 "diaObjectTable_tract",

410 "diaSourceTable",

411 "diaSourceTable_tract",

412 "forcedSourceOnDiaObjectTable",

413 "forcedSourceOnDiaObjectTable_tract",

414 "forcedSourceTable",

415 "forcedSourceTable_tract",

416 "goodSeeingCoadd",

417 "goodSeeingCoadd_nImage",

418 "goodSeeingDiff_assocDiaSrcTable",

419 "goodSeeingDiff_diaObjTable",

420 "goodSeeingDiff_diaSrc",

421 "goodSeeingDiff_diaSrcTable",

422 "goodSeeingDiff_differenceTempExp",

423 "goodSeeingDiff_fullDiaObjTable",

424 "goodSeeingDiff_matchedExp",

425 "goodSeeingDiff_templateExp",

426 "goodSeeingVisits",

427 "mergedForcedSource",

428 "mergedForcedSourceOnDiaObject",

429 },

430 )

431 tester.run(butler, self)

432

433 def test_hsc_drp_prod(self):

434 butler = self.makeButler(writeable=True)

435 tester = PipelineStepTester(

436 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

437 [

438 "#step1",

439 "#step2a",

440 "#step2b",

441 "#step2c",

442 "#step2d",

443 "#step2e",

444 "#step3",

445 "#step4",

446 "#step7",

447 ],

448 initial_dataset_types=REFCATS,

449 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

450 expected_outputs=COMMON_OUTPUTS

451 | HSC_OUTPUTS

452 | {

453 "fgcm_Cycle6_AtmosphereParameters",

454 "fgcm_Cycle0_FitParameters",

455 "fgcm_Cycle1_FitParameters",

456 "fgcm_Cycle2_FitParameters",

457 "fgcm_Cycle3_FitParameters",

458 "fgcm_Cycle4_FitParameters",

459 "fgcm_Cycle5_FitParameters",

460 "fgcm_Cycle6_FitParameters",

461 "fgcm_Cycle0_FlaggedStars",

462 "fgcm_Cycle1_FlaggedStars",

463 "fgcm_Cycle2_FlaggedStars",

464 "fgcm_Cycle3_FlaggedStars",

465 "fgcm_Cycle4_FlaggedStars",

466 "fgcm_Cycle5_FlaggedStars",

467 "fgcm_Cycle6_FlaggedStars",

468 "fgcm_reference_stars",

469 "fgcm_Cycle6_StandardStars",

470 "fgcm_star_ids",

471 "fgcm_star_observations",

472 "fgcm_Cycle6_Zeropoints",

473 "transmission_atmosphere_fgcm",

474 "fgcm_standard_star",

475 },

476 )

477 tester.run(butler, self)

478

479 def test_hsc_drp_rc2(self):

480 butler = self.makeButler(writeable=True)

481 tester = PipelineStepTester(

482 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

483 [

484 "#step1",

485 "#step2a",

486 "#step2b",

487 "#step2cde",

488 "#step3",

489 "#step4",

490 "#step5",

491 "#step6",

492 "#step7",

493 ],

494 initial_dataset_types=REFCATS,

495 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

496 expected_outputs=COMMON_OUTPUTS

497 | HSC_OUTPUTS

498 | {

499 "diaObjectTable_tract",

500 "diaSourceTable",

501 "diaSourceTable_tract",

502 "fgcm_Cycle4_AtmosphereParameters",

503 "fgcm_Cycle0_FitParameters",

504 "fgcm_Cycle1_FitParameters",

505 "fgcm_Cycle2_FitParameters",

506 "fgcm_Cycle3_FitParameters",

507 "fgcm_Cycle4_FitParameters",

508 "fgcm_Cycle0_FlaggedStars",

509 "fgcm_Cycle1_FlaggedStars",

510 "fgcm_Cycle2_FlaggedStars",

511 "fgcm_Cycle3_FlaggedStars",

512 "fgcm_Cycle4_FlaggedStars",

513 "fgcm_reference_stars",

514 "fgcm_Cycle4_StandardStars",

515 "fgcm_star_ids",

516 "fgcm_star_observations",

517 "fgcm_Cycle4_Zeropoints",

518 "fgcm_standard_star",

519 "forcedSourceOnDiaObjectTable",

520 "forcedSourceOnDiaObjectTable_tract",

521 "forcedSourceTable",

522 "forcedSourceTable_tract",

523 "goodSeeingCoadd",

524 "goodSeeingCoadd_nImage",

525 "goodSeeingDiff_assocDiaSrcTable",

526 "goodSeeingDiff_diaObjTable",

527 "goodSeeingDiff_diaSrc",

528 "goodSeeingDiff_diaSrcTable",

529 "goodSeeingDiff_differenceTempExp",

530 "goodSeeingDiff_fullDiaObjTable",

531 "goodSeeingDiff_matchedExp",

532 "goodSeeingDiff_templateExp",

533 "goodSeeingVisits",

534 "mergedForcedSource",

535 "mergedForcedSourceOnDiaObject",

536 "transmission_atmosphere_fgcm",

537 },

538 )

539 tester.run(butler, self)

540

541 def test_hsc_drp_rc2_subset(self):

542 butler = self.makeButler(writeable=True)

543 tester = PipelineStepTester(

544 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

545 [

546 "#nightlyStep1",

547 "#nightlyStep2a",

548 "#nightlyStep2b",

549 "#nightlyStep2c",

550 "#nightlyStep2d",

551 "#nightlyStep3",

552 "#nightlyStep4",

553 ],

554 initial_dataset_types=REFCATS,

555 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

556 expected_outputs=COMMON_OUTPUTS

557 | HSC_OUTPUTS

558 | {

559 "fgcm_Cycle4_AtmosphereParameters",

560 "fgcm_Cycle0_FitParameters",

561 "fgcm_Cycle1_FitParameters",

562 "fgcm_Cycle2_FitParameters",

563 "fgcm_Cycle3_FitParameters",

564 "fgcm_Cycle4_FitParameters",

565 "fgcm_Cycle0_FlaggedStars",

566 "fgcm_Cycle1_FlaggedStars",

567 "fgcm_Cycle2_FlaggedStars",

568 "fgcm_Cycle3_FlaggedStars",

569 "fgcm_Cycle4_FlaggedStars",

570 "fgcm_reference_stars",

571 "fgcm_Cycle4_StandardStars",

572 "fgcm_star_ids",

573 "fgcm_star_observations",

574 "fgcm_Cycle4_Zeropoints",

575 "transmission_atmosphere_fgcm",

576 "fgcm_standard_star",

577 },

578 )

579 tester.run(butler, self)

580

581 def test_latiss_drp(self):

582 butler = self.makeButler(writeable=True)

583 tester = PipelineStepTester(

584 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

585 [

586 "#step1",

587 "#step2a",

588 "#step2bcde",

589 "#step3a",

590 "#step3b",

591 "#step3c",

592 "#step4",

593 "#step5",

594 "#step6",

595 ],

596 initial_dataset_types=REFCATS,

597 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

598 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

599 )

600 tester.run(butler, self)

601

602 def test_lsstcam_quickLook(self):

603 butler = self.makeButler(writeable=True)

604 tester = PipelineStepTester(

605 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

606 [

607 "#step1a-single-visit-detectors",

608 "#step1b-single-visit-visits",

609 "#step1d-single-visit-global",

610 ],

611 initial_dataset_types=REFCATS,

612 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

613 expected_outputs=QUICKLOOK_V2_OUTPUTS,

614 )

615 tester.run(butler, self)

616

617 def test_lsstcam_quickLook_bias(self):

618 butler = self.makeButler(writeable=True)

619 tester = PipelineStepTester(

620 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

621 [

622 "#verifyBiasIsr",

623 ],

624 initial_dataset_types=[],

625 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

626 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

627 )

628 tester.run(butler, self)

629

630 def test_lsstcam_quickLook_dark(self):

631 butler = self.makeButler(writeable=True)

632 tester = PipelineStepTester(

633 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

634 [

635 "#verifyDarkIsr",

636 ],

637 initial_dataset_types=[],

638 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

639 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

640 )

641 tester.run(butler, self)

642

643 def test_lsstcam_quickLook_flat(self):

644 butler = self.makeButler(writeable=True)

645 tester = PipelineStepTester(

646 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

647 [

648 "#verifyFlatIsr",

649 ],

650 initial_dataset_types=[],

651 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

652 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

653 )

654 tester.run(butler, self)

655

656 def test_lsstcam_nightly_validation(self):

657 butler = self.makeButler(writeable=True)

658 tester = PipelineStepTester(

659 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

660 [

661 "#step1a-single-visit-detectors",

662 "#step1b-single-visit-visits",

663 "#step1c-single-visit-tracts",

664 "#step1d-single-visit-global",

665 "#stage3-coadd"

666

667 ],

668 initial_dataset_types=REFCATS,

669 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

670 # Check for some basic stage 1 outputs

671 expected_outputs={"preliminary_visit_image",

672 "preliminary_visit_summary",

673 "preliminary_visit_table",

674 "preliminary_visit_detector_table",

675 "single_visit_star_association_metrics",

676 "deep_coadd"}

677 )

678 tester.run(butler, self)

679

680 def test_lsstcam_drp(self):

681 butler = self.makeButler(writeable=True)

682 tester = PipelineStepTester(

683 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

684 [

685 "#stage1-single-visit",

686 "#stage2-recalibrate",

687 "#stage3-coadd",

688 "#stage4-measure-variability",

689 ],

690 initial_dataset_types=REFCATS,

691 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

692 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

693 )

694 tester.run(butler, self)

695

696 def test_lsstcam_drp_compat(self):

697 butler = self.makeButler(writeable=True)

698 tester = PipelineStepTester(

699 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

700 [

701 "#stage1-single-visit",

702 "#stage2-recalibrate",

703 "#stage3-coadd",

704 "#stage4-measure-variability",

705 ],

706 initial_dataset_types=REFCATS,

707 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

708 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

709 )

710 tester.run(butler, self)

711

712 def test_lsstcam_drp_fl(self):

713 butler = self.makeButler(writeable=True)

714 tester = PipelineStepTester(

715 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

716 [

717 "#stage1-single-visit",

718 "#stage2-recalibrate",

719 "#stage3-coadd",

720 "#stage4-measure-variability",

721 ],

722 initial_dataset_types=REFCATS,

723 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

724 # Check for some basic outputs

725 expected_outputs={"preliminary_visit_image",

726 "preliminary_visit_summary",

727 "deep_coadd"}

728 )

729 tester.run(butler, self)

730

731 def test_lsstcam_nv_fl(self):

732 butler = self.makeButler(writeable=True)

733 tester = PipelineStepTester(

734 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

735 [

736 "#stage1-single-visit",

737 "#stage2-recalibrate",

738 "#stage3-coadd",

739 ],

740 initial_dataset_types=REFCATS,

741 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

742 # Check for some basic outputs

743 expected_outputs={"preliminary_visit_image",

744 "preliminary_visit_summary",

745 "deep_coadd_predetection"}

746 )

747 tester.run(butler, self)

748

749 def test_lsstcam_imsim_drp_ci_imsim(self):

750 butler = self.makeButler(writeable=True)

751 tester = PipelineStepTester(

752 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

753 [""],

754 initial_dataset_types=REFCATS,

755 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

756 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

757 )

758 tester.run(butler, self)

759

760 def test_lsstcam_imsim_drp_test_med_1(self):

761 butler = self.makeButler(writeable=True)

762 tester = PipelineStepTester(

763 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

764 [f"#step{N}" for N in range(1, 8)],

765 initial_dataset_types=REFCATS,

766 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

767 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

768 )

769 tester.run(butler, self)

770

771 def test_lsstcam_imsim_drp(self):

772 butler = self.makeButler(writeable=True)

773 tester = PipelineStepTester(

774 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

775 [f"#step{N}" for N in range(1, 8)],

776 initial_dataset_types=REFCATS,

777 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

778 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

779 )

780 tester.run(butler, self)

781

782 def test_lsstcam_imsim_nightly_validation(self):

783 butler = self.makeButler(writeable=True)

784 tester = PipelineStepTester(

785 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

786 [

787 "#step1",

788 "#step2",

789 "#step3",

790 "#step7",

791 ],

792 initial_dataset_types=REFCATS,

793 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

794 expected_outputs=COMMON_OUTPUTS,

795 )

796 tester.run(butler, self)

797

798 def test_lsstcomcamsim_drp(self):

799 butler = self.makeButler(writeable=True)

800 tester = PipelineStepTester(

801 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

802 [

803 "#step1",

804 "#step2a",

805 "#step2b",

806 "#step2c",

807 "#step2d",

808 "#step2e",

809 "#step3",

810 "#step4",

811 "#step5",

812 "#step6",

813 "#step7",

814 ],

815 initial_dataset_types=REFCATS,

816 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

817 expected_outputs=COMMON_OUTPUTS,

818 )

819 tester.run(butler, self)

820

821 def test_lsstcomcamsim_nightly_validation(self):

822 butler = self.makeButler(writeable=True)

823 tester = PipelineStepTester(

824 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

825 [

826 "#step1",

827 "#step2a",

828 "#nightlyRollup",

829 "#step2b",

830 "#step2d",

831 "#step2e",

832 "#step3",

833 "#step7",

834 ],

835 initial_dataset_types=REFCATS,

836 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

837 expected_outputs=COMMON_OUTPUTS,

838 )

839 tester.run(butler, self)

840

841 def test_lsstcomcamsim_quickLook(self):

842 butler = self.makeButler(writeable=True)

843 tester = PipelineStepTester(

844 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

845 [

846 "#step1",

847 "#step2a",

848 "#nightlyRollup",

849 ],

850 initial_dataset_types=REFCATS,

851 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

852 expected_outputs=QUICKLOOK_OUTPUTS,

853 )

854 tester.run(butler, self)

855

856 def test_lsstcomcam_drp(self):

857 butler = self.makeButler(writeable=True)

858 tester = PipelineStepTester(

859 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

860 [

861 "#step1",

862 "#step2a",

863 "#step2b",

864 "#step2c",

865 "#step2d",

866 "#step2e",

867 "#step3a",

868 "#step3b",

869 "#step4",

870 "#step5",

871 "#step6",

872 "#step7",

873 ],

874 initial_dataset_types=REFCATS,

875 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

876 expected_outputs=COMMON_OUTPUTS,

877 )

878 tester.run(butler, self)

879

880 def test_lsstcomcam_nightly_validation(self):

881 butler = self.makeButler(writeable=True)

882 tester = PipelineStepTester(

883 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

884 [

885 "#step1",

886 "#step2a",

887 "#nightlyRollup",

888 "#step2b",

889 "#step2d",

890 "#step2e",

891 "#step3a",

892 "#step3b",

893 "#step7",

894 ],

895 initial_dataset_types=REFCATS,

896 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

897 expected_outputs=COMMON_OUTPUTS,

898 )

899 tester.run(butler, self)

900

901 def test_lsstcomcam_quickLook(self):

902 butler = self.makeButler(writeable=True)

903 tester = PipelineStepTester(

904 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

905 [

906 "#step1",

907 "#step2a",

908 "#nightlyRollup",

909 ],

910 initial_dataset_types=REFCATS,

911 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

912 expected_outputs=QUICKLOOK_OUTPUTS,

913 )

914 tester.run(butler, self)

915

916

917if __name__ == "__main__": 917 ↛ 918line 917 didn't jump to line 918 because the condition on line 917 was never true

918 unittest.main()

Coverage for tests / test_pipelines.py: 40%

149 statements