Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95 "gain_correction",

96}

98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

99LSSTCAM_IMSIM_INPUTS = {

100 "bfk",

101 "cal_ref_cat_2_2",

102 "truth_summary",

103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

104}

105

106# LSSTCam common inputs, in addition to COMMON_INPUTS

107LSSTCAM_INPUTS = {

108 "bfk",

109 "electroBfDistortionMatrix",

110 "linearizer",

111 "defects",

112 "cti",

113 "the_monster_20250219",

114 "gaia_dr3_20230707",

115 "sky",

116 "astrometry_camera",

117 "gain_correction",

118}

119

120# LSSTComCam common inputs, in addition to COMMON_INPUTS

121LSSTCOMCAM_INPUTS = {

122 "bfk",

123 "ptc",

124 "linearizer",

125 "defects",

126 "cti",

127 "the_monster_20250219",

128 "illuminationCorrection",

129 "gain_correction",

130}

131

132# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

133LSSTCOMCAMSIM_INPUTS = {

134 "uw_stars_20240524",

135 "ptc",

136 "bfk",

137 "gain_correction",

138}

139

140# a selection of mostly common outputs

141COMMON_OUTPUTS = {

142 "ccdVisitTable",

143 "compare_warp_artifact_mask",

144 "deepCoaddVisits",

145 "deepCoadd",

146 "deepCoadd_calexp",

147 "deepCoadd_calexp_background",

148 "deepCoadd_det",

149 "deepCoadd_directWarp",

150 "deepCoadd_directWarp_maskedFraction",

151 "deepCoadd_forced_src",

152 "deepCoadd_inputMap",

153 "deepCoadd_meas",

154 "deepCoadd_mergeDet",

155 "deepCoadd_nImage",

156 "deepCoadd_obj",

157 "deepCoadd_psfMatchedWarp",

158 "deepCoadd_ref",

159 "deepCoadd_scarletModelData",

160 "finalized_src_table",

161 "finalVisitSummary",

162 "initial_astrometry_match_detector",

163 "initial_photometry_match_detector",

164 "initial_psf_stars_footprints_detector",

165 "initial_psf_stars_detector",

166 "calexp",

167 "calexpBackground",

168 "src",

169 "objectTable",

170 "objectTable_tract",

171 "postISRCCD",

172 "sourceTable_visit",

173 "visitSummary",

174 "visitTable",

175 "isolated_star_presource_associations",

176 "isolated_star_presources",

177}

178

179# Translated COMMON_OUTPUTS

180COMMON_V2_OUTPUTS = {

181 'visit_detector_table',

182 'compare_warp_artifact_mask',

183 'deep_coadd_visit_selection',

184 'deep_coadd_predetection',

185 'deep_coadd',

186 'deep_coadd_background',

187 'object_detection',

188 'direct_warp',

189 'direct_warp_masked_fraction',

190 'object_forced_measurement',

191 'deep_coadd_input_map',

192 'object_unforced_measurement',

193 'object_detection_merged',

194 'deep_coadd_n_image',

195 'object_unstandardized',

196 'psf_matched_warp',

197 'object_ref_measurement',

198 'object_scarlet_models',

199 'refit_psf_star',

200 'visit_summary',

201 'initial_astrometry_match_detector',

202 'initial_photometry_match_detector',

203 'single_visit_psf_star_footprints',

204 'single_visit_psf_star',

205 'preliminary_visit_image',

206 'preliminary_visit_image_background',

207 'single_visit_star_footprints',

208 'object_patch',

209 'object_all',

210 'post_isr_image',

211 'recalibrated_star',

212 'preliminary_visit_summary',

213 'visit_table',

214 'isolated_star_association',

215 'isolated_star'

216}

217

218# HSC common outputs, in addition to COMMON_OUTPUTS

219HSC_OUTPUTS = {

220 "calexp_skyCorr_visit_mosaic",

221 "calexpBackground_skyCorr_visit_mosaic",

222 "preSourceTable",

223 "preSourceTable_visit",

224 "skyCorr",

225}

226

227# LATISS common outputs, in addition to COMMON_OUTPUTS

228LATISS_OUTPUTS = {

229 "diaObjectTable_tract",

230 "diaSourceTable",

231 "diaSourceTable_tract",

232 "fgcm_Cycle0_FitParameters",

233 "fgcm_Cycle1_FitParameters",

234 "fgcm_Cycle2_FitParameters",

235 "fgcm_Cycle3_FitParameters",

236 "fgcm_Cycle4_FitParameters",

237 "fgcm_Cycle0_FlaggedStars",

238 "fgcm_Cycle1_FlaggedStars",

239 "fgcm_Cycle2_FlaggedStars",

240 "fgcm_Cycle3_FlaggedStars",

241 "fgcm_Cycle4_FlaggedStars",

242 "fgcm_reference_stars",

243 "fgcm_star_ids",

244 "fgcm_star_observations",

245 "fgcm_standard_star",

246 "forcedSourceOnDiaObjectTable",

247 "forcedSourceOnDiaObjectTable_tract",

248 "forcedSourceTable",

249 "forcedSourceTable_tract",

250 "goodSeeingCoadd",

251 "goodSeeingCoadd_nImage",

252 "goodSeeingDiff_assocDiaSrcTable",

253 "goodSeeingDiff_diaObjTable",

254 "goodSeeingDiff_diaSrc",

255 "goodSeeingDiff_diaSrcTable",

256 "goodSeeingDiff_differenceExp",

257 "goodSeeingDiff_differenceTempExp",

258 "goodSeeingDiff_fullDiaObjTable",

259 "goodSeeingDiff_matchedExp",

260 "goodSeeingDiff_templateExp",

261 "goodSeeingVisits",

262 "mergedForcedSource",

263 "mergedForcedSourceOnDiaObject",

264 "transmission_atmosphere_fgcm",

265}

266

267# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

268LSSTCAM_IMSIM_OUTPUTS = {

269 "diaObjectTable_tract",

270 "diaSourceTable",

271 "diaSourceTable_tract",

272 "forcedSourceOnDiaObjectTable",

273 "forcedSourceOnDiaObjectTable_tract",

274 "forcedSourceTable",

275 "forcedSourceTable_tract",

276 "goodSeeingCoadd",

277 "goodSeeingCoadd_nImage",

278 "goodSeeingDiff_assocDiaSrcTable",

279 "goodSeeingDiff_diaObjTable",

280 "goodSeeingDiff_diaSrc",

281 "goodSeeingDiff_diaSrcTable",

282 "goodSeeingDiff_differenceExp",

283 "goodSeeingDiff_differenceTempExp",

284 "goodSeeingDiff_fullDiaObjTable",

285 "goodSeeingDiff_matchedExp",

286 "goodSeeingDiff_templateExp",

287 "goodSeeingVisits",

288 "mergedForcedSource",

289 "mergedForcedSourceOnDiaObject",

290}

291

292# v2 outputs common to all "quickLook" pipelines, which only include

293QUICKLOOK_V2_OUTPUTS = {

294 "preliminary_visit_image", "preliminary_visit_summary"

295}

296

297# Outputs common to all "quickLook" pipelines, which only include

298QUICKLOOK_OUTPUTS = {

299 "calexp", "visitSummary"

300}

301

302QUICKLOOK_BIAS_OUTPUTS = {

303 "post_isr_image",

304}

305

306QUICKLOOK_DARK_OUTPUTS = {

307 "post_isr_image",

308}

309

310QUICKLOOK_FLAT_OUTPUTS = {

311 "post_isr_image",

312}

313

314# All refcats used by any pipelines, for inclusion as the initial_dataset_types

315# argument to PipelineStepTester; the intent is just to allow the 'skypix'

316# dimension placeholder used in these connections to be resolved.

317REFCATS = [

318 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

319 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

320 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

321 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

322 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

323 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

324 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

325]

326

327

328class PipelineTestCase(unittest.TestCase):

329 def setUp(self):

330 self.root = makeTestTempDir(TEST_DIR)

331 self.maxDiff = None

332

333 def tearDown(self):

334 removeTestTempDir(self.root)

335

336 def makeButler(self, **kwargs: Any) -> Butler:

337 """Return new Butler instance on each call."""

338 config = Config()

339

340 # make separate temporary directory for registry of this instance

341 tmpdir = tempfile.mkdtemp(dir=self.root)

342 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

343 config = Butler.makeRepo(self.root, config)

344 butler = Butler.from_config(config, **kwargs)

345 DatastoreMock.apply(butler)

346 return butler

347

348 def test_decam_isrForCrosstalkSources(self):

349 butler = self.makeButler(writeable=True)

350 tester = PipelineStepTester(

351 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

352 ["#step0"],

353 initial_dataset_types=REFCATS,

354 expected_inputs={

355 "camera",

356 "raw",

357 },

358 expected_outputs={"overscanRaw"},

359 )

360 tester.run(butler, self)

361

362 def test_decam_drp_merian(self):

363 butler = self.makeButler(writeable=True)

364 tester = PipelineStepTester(

365 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

366 [

367 "#step1",

368 "#step2a",

369 "#step2b",

370 "#step2d",

371 "#step2e",

372 "#step3",

373 ],

374 initial_dataset_types=REFCATS,

375 expected_inputs=COMMON_INPUTS

376 | {

377 "defects",

378 "gaia_dr2_20200414",

379 "gaia_dr3_20230707",

380 "linearizer",

381 "overscanRaw",

382 "ps1_pv3_3pi_20170110",

383 },

384 expected_outputs=COMMON_OUTPUTS

385 | {

386 "goodSeeingCoadd",

387 "goodSeeingCoadd_nImage",

388 "goodSeeingVisits",

389 "preSourceTable",

390 "preSourceTable_visit",

391 },

392 )

393 tester.run(butler, self)

394

395 def test_hsc_drp_ci_hsc(self):

396 butler = self.makeButler(writeable=True)

397 tester = PipelineStepTester(

398 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

399 [""],

400 initial_dataset_types=REFCATS,

401 expected_inputs=COMMON_INPUTS

402 | HSC_INPUTS

403 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

404 expected_outputs=COMMON_OUTPUTS

405 | HSC_OUTPUTS

406 | {

407 "diaObjectTable_tract",

408 "diaSourceTable",

409 "diaSourceTable_tract",

410 "forcedSourceOnDiaObjectTable",

411 "forcedSourceOnDiaObjectTable_tract",

412 "forcedSourceTable",

413 "forcedSourceTable_tract",

414 "goodSeeingCoadd",

415 "goodSeeingCoadd_nImage",

416 "goodSeeingDiff_assocDiaSrcTable",

417 "goodSeeingDiff_diaObjTable",

418 "goodSeeingDiff_diaSrc",

419 "goodSeeingDiff_diaSrcTable",

420 "goodSeeingDiff_differenceTempExp",

421 "goodSeeingDiff_fullDiaObjTable",

422 "goodSeeingDiff_matchedExp",

423 "goodSeeingDiff_templateExp",

424 "goodSeeingVisits",

425 "mergedForcedSource",

426 "mergedForcedSourceOnDiaObject",

427 },

428 )

429 tester.run(butler, self)

430

431 def test_hsc_drp_prod(self):

432 butler = self.makeButler(writeable=True)

433 tester = PipelineStepTester(

434 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

435 [

436 "#step1",

437 "#step2a",

438 "#step2b",

439 "#step2c",

440 "#step2d",

441 "#step2e",

442 "#step3",

443 "#step4",

444 "#step7",

445 ],

446 initial_dataset_types=REFCATS,

447 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

448 expected_outputs=COMMON_OUTPUTS

449 | HSC_OUTPUTS

450 | {

451 "fgcm_Cycle6_AtmosphereParameters",

452 "fgcm_Cycle0_FitParameters",

453 "fgcm_Cycle1_FitParameters",

454 "fgcm_Cycle2_FitParameters",

455 "fgcm_Cycle3_FitParameters",

456 "fgcm_Cycle4_FitParameters",

457 "fgcm_Cycle5_FitParameters",

458 "fgcm_Cycle6_FitParameters",

459 "fgcm_Cycle0_FlaggedStars",

460 "fgcm_Cycle1_FlaggedStars",

461 "fgcm_Cycle2_FlaggedStars",

462 "fgcm_Cycle3_FlaggedStars",

463 "fgcm_Cycle4_FlaggedStars",

464 "fgcm_Cycle5_FlaggedStars",

465 "fgcm_Cycle6_FlaggedStars",

466 "fgcm_reference_stars",

467 "fgcm_Cycle6_StandardStars",

468 "fgcm_star_ids",

469 "fgcm_star_observations",

470 "fgcm_Cycle6_Zeropoints",

471 "transmission_atmosphere_fgcm",

472 "fgcm_standard_star",

473 },

474 )

475 tester.run(butler, self)

476

477 def test_hsc_drp_rc2(self):

478 butler = self.makeButler(writeable=True)

479 tester = PipelineStepTester(

480 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

481 [

482 "#step1",

483 "#step2a",

484 "#step2b",

485 "#step2cde",

486 "#step3",

487 "#step4",

488 "#step5",

489 "#step6",

490 "#step7",

491 ],

492 initial_dataset_types=REFCATS,

493 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

494 expected_outputs=COMMON_OUTPUTS

495 | HSC_OUTPUTS

496 | {

497 "diaObjectTable_tract",

498 "diaSourceTable",

499 "diaSourceTable_tract",

500 "fgcm_Cycle4_AtmosphereParameters",

501 "fgcm_Cycle0_FitParameters",

502 "fgcm_Cycle1_FitParameters",

503 "fgcm_Cycle2_FitParameters",

504 "fgcm_Cycle3_FitParameters",

505 "fgcm_Cycle4_FitParameters",

506 "fgcm_Cycle0_FlaggedStars",

507 "fgcm_Cycle1_FlaggedStars",

508 "fgcm_Cycle2_FlaggedStars",

509 "fgcm_Cycle3_FlaggedStars",

510 "fgcm_Cycle4_FlaggedStars",

511 "fgcm_reference_stars",

512 "fgcm_Cycle4_StandardStars",

513 "fgcm_star_ids",

514 "fgcm_star_observations",

515 "fgcm_Cycle4_Zeropoints",

516 "fgcm_standard_star",

517 "forcedSourceOnDiaObjectTable",

518 "forcedSourceOnDiaObjectTable_tract",

519 "forcedSourceTable",

520 "forcedSourceTable_tract",

521 "goodSeeingCoadd",

522 "goodSeeingCoadd_nImage",

523 "goodSeeingDiff_assocDiaSrcTable",

524 "goodSeeingDiff_diaObjTable",

525 "goodSeeingDiff_diaSrc",

526 "goodSeeingDiff_diaSrcTable",

527 "goodSeeingDiff_differenceTempExp",

528 "goodSeeingDiff_fullDiaObjTable",

529 "goodSeeingDiff_matchedExp",

530 "goodSeeingDiff_templateExp",

531 "goodSeeingVisits",

532 "mergedForcedSource",

533 "mergedForcedSourceOnDiaObject",

534 "transmission_atmosphere_fgcm",

535 },

536 )

537 tester.run(butler, self)

538

539 def test_hsc_drp_rc2_subset(self):

540 butler = self.makeButler(writeable=True)

541 tester = PipelineStepTester(

542 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

543 [

544 "#nightlyStep1",

545 "#nightlyStep2a",

546 "#nightlyStep2b",

547 "#nightlyStep2c",

548 "#nightlyStep2d",

549 "#nightlyStep3",

550 "#nightlyStep4",

551 ],

552 initial_dataset_types=REFCATS,

553 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

554 expected_outputs=COMMON_OUTPUTS

555 | HSC_OUTPUTS

556 | {

557 "fgcm_Cycle4_AtmosphereParameters",

558 "fgcm_Cycle0_FitParameters",

559 "fgcm_Cycle1_FitParameters",

560 "fgcm_Cycle2_FitParameters",

561 "fgcm_Cycle3_FitParameters",

562 "fgcm_Cycle4_FitParameters",

563 "fgcm_Cycle0_FlaggedStars",

564 "fgcm_Cycle1_FlaggedStars",

565 "fgcm_Cycle2_FlaggedStars",

566 "fgcm_Cycle3_FlaggedStars",

567 "fgcm_Cycle4_FlaggedStars",

568 "fgcm_reference_stars",

569 "fgcm_Cycle4_StandardStars",

570 "fgcm_star_ids",

571 "fgcm_star_observations",

572 "fgcm_Cycle4_Zeropoints",

573 "transmission_atmosphere_fgcm",

574 "fgcm_standard_star",

575 },

576 )

577 tester.run(butler, self)

578

579 def test_latiss_drp(self):

580 butler = self.makeButler(writeable=True)

581 tester = PipelineStepTester(

582 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

583 [

584 "#step1",

585 "#step2a",

586 "#step2bcde",

587 "#step3a",

588 "#step3b",

589 "#step3c",

590 "#step4",

591 "#step5",

592 "#step6",

593 ],

594 initial_dataset_types=REFCATS,

595 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

596 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

597 )

598 tester.run(butler, self)

599

600 def test_lsstcam_quickLook(self):

601 butler = self.makeButler(writeable=True)

602 tester = PipelineStepTester(

603 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

604 [

605 "#step1a-single-visit-detectors",

606 "#step1b-single-visit-visits",

607 "#step1d-single-visit-global",

608 ],

609 initial_dataset_types=REFCATS,

610 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

611 expected_outputs=QUICKLOOK_V2_OUTPUTS,

612 )

613 tester.run(butler, self)

614

615 def test_lsstcam_quickLook_bias(self):

616 butler = self.makeButler(writeable=True)

617 tester = PipelineStepTester(

618 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

619 [

620 "#verifyBiasIsr",

621 ],

622 initial_dataset_types=[],

623 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

624 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

625 )

626 tester.run(butler, self)

627

628 def test_lsstcam_quickLook_dark(self):

629 butler = self.makeButler(writeable=True)

630 tester = PipelineStepTester(

631 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

632 [

633 "#verifyDarkIsr",

634 ],

635 initial_dataset_types=[],

636 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

637 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

638 )

639 tester.run(butler, self)

640

641 def test_lsstcam_quickLook_flat(self):

642 butler = self.makeButler(writeable=True)

643 tester = PipelineStepTester(

644 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

645 [

646 "#verifyFlatIsr",

647 ],

648 initial_dataset_types=[],

649 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

650 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

651 )

652 tester.run(butler, self)

653

654 def test_lsstcam_nightly_validation(self):

655 butler = self.makeButler(writeable=True)

656 tester = PipelineStepTester(

657 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

658 [

659 "#step1a-single-visit-detectors",

660 "#step1b-single-visit-visits",

661 "#step1c-single-visit-tracts",

662 "#step1d-single-visit-global",

663 "#stage3-coadd"

664

665 ],

666 initial_dataset_types=REFCATS,

667 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

668 # Check for some basic stage 1 outputs

669 expected_outputs={"preliminary_visit_image",

670 "preliminary_visit_summary",

671 "preliminary_visit_table",

672 "preliminary_visit_detector_table",

673 "single_visit_star_association_metrics",

674 "deep_coadd"}

675 )

676 tester.run(butler, self)

677

678 def test_lsstcam_drp(self):

679 butler = self.makeButler(writeable=True)

680 tester = PipelineStepTester(

681 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

682 [

683 "#stage1-single-visit",

684 "#stage2-recalibrate",

685 "#stage3-coadd",

686 "#stage4-measure-variability",

687 ],

688 initial_dataset_types=REFCATS,

689 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

690 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

691 )

692 tester.run(butler, self)

693

694 def test_lsstcam_drp_compat(self):

695 butler = self.makeButler(writeable=True)

696 tester = PipelineStepTester(

697 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

698 [

699 "#stage1-single-visit",

700 "#stage2-recalibrate",

701 "#stage3-coadd",

702 "#stage4-measure-variability",

703 ],

704 initial_dataset_types=REFCATS,

705 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

706 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

707 )

708 tester.run(butler, self)

709

710 def test_lsstcam_drp_fl(self):

711 butler = self.makeButler(writeable=True)

712 tester = PipelineStepTester(

713 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

714 [

715 "#stage1-single-visit",

716 "#stage2-recalibrate",

717 "#stage3-coadd",

718 "#stage4-measure-variability",

719 ],

720 initial_dataset_types=REFCATS,

721 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

722 # Check for some basic outputs

723 expected_outputs={"preliminary_visit_image",

724 "preliminary_visit_summary",

725 "deep_coadd"}

726 )

727 tester.run(butler, self)

728

729 def test_lsstcam_nv_fl(self):

730 butler = self.makeButler(writeable=True)

731 tester = PipelineStepTester(

732 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

733 [

734 "#stage1-single-visit",

735 "#stage2-recalibrate",

736 "#stage3-coadd",

737 ],

738 initial_dataset_types=REFCATS,

739 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

740 # Check for some basic outputs

741 expected_outputs={"preliminary_visit_image",

742 "preliminary_visit_summary",

743 "deep_coadd_predetection"}

744 )

745 tester.run(butler, self)

746

747 def test_lsstcam_imsim_drp_ci_imsim(self):

748 butler = self.makeButler(writeable=True)

749 tester = PipelineStepTester(

750 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

751 [""],

752 initial_dataset_types=REFCATS,

753 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

754 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

755 )

756 tester.run(butler, self)

757

758 def test_lsstcam_imsim_drp_test_med_1(self):

759 butler = self.makeButler(writeable=True)

760 tester = PipelineStepTester(

761 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

762 [f"#step{N}" for N in range(1, 8)],

763 initial_dataset_types=REFCATS,

764 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

765 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

766 )

767 tester.run(butler, self)

768

769 def test_lsstcam_imsim_drp(self):

770 butler = self.makeButler(writeable=True)

771 tester = PipelineStepTester(

772 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

773 [f"#step{N}" for N in range(1, 8)],

774 initial_dataset_types=REFCATS,

775 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

776 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

777 )

778 tester.run(butler, self)

779

780 def test_lsstcam_imsim_nightly_validation(self):

781 butler = self.makeButler(writeable=True)

782 tester = PipelineStepTester(

783 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

784 [

785 "#step1",

786 "#step2",

787 "#step3",

788 "#step7",

789 ],

790 initial_dataset_types=REFCATS,

791 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

792 expected_outputs=COMMON_OUTPUTS,

793 )

794 tester.run(butler, self)

795

796 def test_lsstcomcamsim_drp(self):

797 butler = self.makeButler(writeable=True)

798 tester = PipelineStepTester(

799 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

800 [

801 "#step1",

802 "#step2a",

803 "#step2b",

804 "#step2c",

805 "#step2d",

806 "#step2e",

807 "#step3",

808 "#step4",

809 "#step5",

810 "#step6",

811 "#step7",

812 ],

813 initial_dataset_types=REFCATS,

814 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

815 expected_outputs=COMMON_OUTPUTS,

816 )

817 tester.run(butler, self)

818

819 def test_lsstcomcamsim_nightly_validation(self):

820 butler = self.makeButler(writeable=True)

821 tester = PipelineStepTester(

822 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

823 [

824 "#step1",

825 "#step2a",

826 "#nightlyRollup",

827 "#step2b",

828 "#step2d",

829 "#step2e",

830 "#step3",

831 "#step7",

832 ],

833 initial_dataset_types=REFCATS,

834 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

835 expected_outputs=COMMON_OUTPUTS,

836 )

837 tester.run(butler, self)

838

839 def test_lsstcomcamsim_quickLook(self):

840 butler = self.makeButler(writeable=True)

841 tester = PipelineStepTester(

842 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

843 [

844 "#step1",

845 "#step2a",

846 "#nightlyRollup",

847 ],

848 initial_dataset_types=REFCATS,

849 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

850 expected_outputs=QUICKLOOK_OUTPUTS,

851 )

852 tester.run(butler, self)

853

854 def test_lsstcomcam_drp(self):

855 butler = self.makeButler(writeable=True)

856 tester = PipelineStepTester(

857 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

858 [

859 "#step1",

860 "#step2a",

861 "#step2b",

862 "#step2c",

863 "#step2d",

864 "#step2e",

865 "#step3a",

866 "#step3b",

867 "#step4",

868 "#step5",

869 "#step6",

870 "#step7",

871 ],

872 initial_dataset_types=REFCATS,

873 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

874 expected_outputs=COMMON_OUTPUTS,

875 )

876 tester.run(butler, self)

877

878 def test_lsstcomcam_nightly_validation(self):

879 butler = self.makeButler(writeable=True)

880 tester = PipelineStepTester(

881 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

882 [

883 "#step1",

884 "#step2a",

885 "#nightlyRollup",

886 "#step2b",

887 "#step2d",

888 "#step2e",

889 "#step3a",

890 "#step3b",

891 "#step7",

892 ],

893 initial_dataset_types=REFCATS,

894 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

895 expected_outputs=COMMON_OUTPUTS,

896 )

897 tester.run(butler, self)

898

899 def test_lsstcomcam_quickLook(self):

900 butler = self.makeButler(writeable=True)

901 tester = PipelineStepTester(

902 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

903 [

904 "#step1",

905 "#step2a",

906 "#nightlyRollup",

907 ],

908 initial_dataset_types=REFCATS,

909 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

910 expected_outputs=QUICKLOOK_OUTPUTS,

911 )

912 tester.run(butler, self)

913

914

915if __name__ == "__main__": 915 ↛ 916line 915 didn't jump to line 916 because the condition on line 915 was never true

916 unittest.main()

Coverage for tests / test_pipelines.py: 40%

148 statements