Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95 "gain_correction",

96}

98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

99LSSTCAM_IMSIM_INPUTS = {

100 "bfk",

101 "cal_ref_cat_2_2",

102 "truth_summary",

103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

104}

105

106# LSSTCam common inputs, in addition to COMMON_INPUTS

107LSSTCAM_INPUTS = {

108 "bfk",

109 "electroBfDistortionMatrix",

110 "linearizer",

111 "defects",

112 "cti",

113 "the_monster_20250219",

114 "gaia_dr3_20230707",

115 "sky",

116 "astrometry_camera",

117 "gain_correction",

118}

119

120# LSSTComCam common inputs, in addition to COMMON_INPUTS

121LSSTCOMCAM_INPUTS = {

122 "bfk",

123 "ptc",

124 "linearizer",

125 "defects",

126 "cti",

127 "the_monster_20250219",

128 "illuminationCorrection",

129 "gain_correction",

130}

131

132# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

133LSSTCOMCAMSIM_INPUTS = {

134 "uw_stars_20240524",

135 "ptc",

136 "bfk",

137 "gain_correction",

138}

139

140# a selection of mostly common outputs

141COMMON_OUTPUTS = {

142 "ccdVisitTable",

143 "compare_warp_artifact_mask",

144 "deepCoaddVisits",

145 "deepCoadd",

146 "deepCoadd_calexp",

147 "deepCoadd_calexp_background",

148 "deepCoadd_det",

149 "deepCoadd_directWarp",

150 "deepCoadd_directWarp_maskedFraction",

151 "deepCoadd_forced_src",

152 "deepCoadd_inputMap",

153 "deepCoadd_meas",

154 "deepCoadd_mergeDet",

155 "deepCoadd_nImage",

156 "deepCoadd_obj",

157 "deepCoadd_psfMatchedWarp",

158 "deepCoadd_ref",

159 "deepCoadd_scarletModelData",

160 "finalized_src_table",

161 "finalVisitSummary",

162 "initial_astrometry_match_detector",

163 "initial_photometry_match_detector",

164 "initial_psf_stars_footprints_detector",

165 "initial_psf_stars_detector",

166 "calexp",

167 "calexpBackground",

168 "src",

169 "objectTable",

170 "objectTable_tract",

171 "postISRCCD",

172 "sourceTable_visit",

173 "visitSummary",

174 "visitTable",

175 "isolated_star_presource_associations",

176 "isolated_star_presources",

177}

178

179# Translated COMMON_OUTPUTS

180COMMON_V2_OUTPUTS = {

181 'visit_detector_table',

182 'compare_warp_artifact_mask',

183 'deep_coadd_visit_selection',

184 'deep_coadd_predetection',

185 'deep_coadd',

186 'deep_coadd_background',

187 'object_detection',

188 'direct_warp',

189 'direct_warp_masked_fraction',

190 'object_forced_measurement',

191 'deep_coadd_input_map',

192 'object_unforced_measurement',

193 'object_detection_merged',

194 'deep_coadd_n_image',

195 'object_unstandardized',

196 'psf_matched_warp',

197 'object_ref_measurement',

198 'object_scarlet_models',

199 'refit_psf_star',

200 'visit_summary',

201 'initial_astrometry_match_detector',

202 'initial_photometry_match_detector',

203 'single_visit_psf_star_footprints',

204 'single_visit_psf_star',

205 'preliminary_visit_image',

206 'preliminary_visit_image_background',

207 'single_visit_star_footprints',

208 'object_patch',

209 'object_all',

210 'post_isr_image',

211 'recalibrated_star',

212 'preliminary_visit_summary',

213 'visit_table',

214 'isolated_star_association',

215 'isolated_star'

216}

217

218# HSC common outputs, in addition to COMMON_OUTPUTS

219HSC_OUTPUTS = {

220 "calexp_skyCorr_visit_mosaic",

221 "calexpBackground_skyCorr_visit_mosaic",

222 "preSourceTable",

223 "preSourceTable_visit",

224 "skyCorr",

225}

226

227# LATISS common outputs, in addition to COMMON_OUTPUTS

228LATISS_OUTPUTS = {

229 "diaObjectTable_tract",

230 "diaSourceTable",

231 "diaSourceTable_tract",

232 "fgcm_Cycle0_FitParameters",

233 "fgcm_Cycle1_FitParameters",

234 "fgcm_Cycle2_FitParameters",

235 "fgcm_Cycle3_FitParameters",

236 "fgcm_Cycle4_FitParameters",

237 "fgcm_Cycle0_FlaggedStars",

238 "fgcm_Cycle1_FlaggedStars",

239 "fgcm_Cycle2_FlaggedStars",

240 "fgcm_Cycle3_FlaggedStars",

241 "fgcm_Cycle4_FlaggedStars",

242 "fgcm_reference_stars",

243 "fgcm_star_ids",

244 "fgcm_star_observations",

245 "fgcm_standard_star",

246 "forcedSourceOnDiaObjectTable",

247 "forcedSourceOnDiaObjectTable_tract",

248 "forcedSourceTable",

249 "forcedSourceTable_tract",

250 "goodSeeingCoadd",

251 "goodSeeingCoadd_nImage",

252 "goodSeeingDiff_assocDiaSrcTable",

253 "goodSeeingDiff_diaObjTable",

254 "goodSeeingDiff_diaSrc",

255 "goodSeeingDiff_diaSrcTable",

256 "goodSeeingDiff_differenceExp",

257 "goodSeeingDiff_differenceTempExp",

258 "goodSeeingDiff_fullDiaObjTable",

259 "goodSeeingDiff_matchedExp",

260 "goodSeeingDiff_templateExp",

261 "goodSeeingVisits",

262 "mergedForcedSource",

263 "mergedForcedSourceOnDiaObject",

264 "transmission_atmosphere_fgcm",

265}

266

267# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

268LSSTCAM_IMSIM_OUTPUTS = {

269 "diaObjectTable_tract",

270 "diaSourceTable",

271 "diaSourceTable_tract",

272 "forcedSourceOnDiaObjectTable",

273 "forcedSourceOnDiaObjectTable_tract",

274 "forcedSourceTable",

275 "forcedSourceTable_tract",

276 "goodSeeingCoadd",

277 "goodSeeingCoadd_nImage",

278 "goodSeeingDiff_assocDiaSrcTable",

279 "goodSeeingDiff_diaObjTable",

280 "goodSeeingDiff_diaSrc",

281 "goodSeeingDiff_diaSrcTable",

282 "goodSeeingDiff_differenceExp",

283 "goodSeeingDiff_differenceTempExp",

284 "goodSeeingDiff_fullDiaObjTable",

285 "goodSeeingDiff_matchedExp",

286 "goodSeeingDiff_templateExp",

287 "goodSeeingVisits",

288 "mergedForcedSource",

289 "mergedForcedSourceOnDiaObject",

290}

291

292# v2 outputs common to all "quickLook" pipelines, which only include

293QUICKLOOK_V2_OUTPUTS = {

294 "preliminary_visit_image", "preliminary_visit_summary"

295}

296

297# Outputs common to all "quickLook" pipelines, which only include

298QUICKLOOK_OUTPUTS = {

299 "calexp", "visitSummary"

300}

301

302QUICKLOOK_BIAS_OUTPUTS = {

303 "post_isr_image",

304}

305

306QUICKLOOK_DARK_OUTPUTS = {

307 "post_isr_image",

308}

309

310QUICKLOOK_FLAT_OUTPUTS = {

311 "post_isr_image",

312}

313

314# All refcats used by any pipelines, for inclusion as the initial_dataset_types

315# argument to PipelineStepTester; the intent is just to allow the 'skypix'

316# dimension placeholder used in these connections to be resolved.

317REFCATS = [

318 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

319 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

320 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

321 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

322 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

323 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

324 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

325]

326

327

328class PipelineTestCase(unittest.TestCase):

329 def setUp(self):

330 self.root = makeTestTempDir(TEST_DIR)

331 self.maxDiff = None

332

333 def tearDown(self):

334 removeTestTempDir(self.root)

335

336 def makeButler(self, **kwargs: Any) -> Butler:

337 """Return new Butler instance on each call."""

338 config = Config()

339

340 # make separate temporary directory for registry of this instance

341 tmpdir = tempfile.mkdtemp(dir=self.root)

342 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

343 config = Butler.makeRepo(self.root, config)

344 butler = Butler.from_config(config, **kwargs)

345 self.enterContext(butler)

346 DatastoreMock.apply(butler)

347 return butler

348

349 def test_decam_isrForCrosstalkSources(self):

350 butler = self.makeButler(writeable=True)

351 tester = PipelineStepTester(

352 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

353 ["#step0"],

354 initial_dataset_types=REFCATS,

355 expected_inputs={

356 "camera",

357 "raw",

358 },

359 expected_outputs={"overscanRaw"},

360 )

361 tester.run(butler, self)

362

363 def test_decam_drp_merian(self):

364 butler = self.makeButler(writeable=True)

365 tester = PipelineStepTester(

366 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

367 [

368 "#step1",

369 "#step2a",

370 "#step2b",

371 "#step2d",

372 "#step2e",

373 "#step3",

374 ],

375 initial_dataset_types=REFCATS,

376 expected_inputs=COMMON_INPUTS

377 | {

378 "defects",

379 "gaia_dr2_20200414",

380 "gaia_dr3_20230707",

381 "linearizer",

382 "overscanRaw",

383 "ps1_pv3_3pi_20170110",

384 },

385 expected_outputs=COMMON_OUTPUTS

386 | {

387 "goodSeeingCoadd",

388 "goodSeeingCoadd_nImage",

389 "goodSeeingVisits",

390 "preSourceTable",

391 "preSourceTable_visit",

392 },

393 )

394 tester.run(butler, self)

395

396 def test_hsc_drp_ci_hsc(self):

397 butler = self.makeButler(writeable=True)

398 tester = PipelineStepTester(

399 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

400 [""],

401 initial_dataset_types=REFCATS,

402 expected_inputs=COMMON_INPUTS

403 | HSC_INPUTS

404 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

405 expected_outputs=COMMON_OUTPUTS

406 | HSC_OUTPUTS

407 | {

408 "diaObjectTable_tract",

409 "diaSourceTable",

410 "diaSourceTable_tract",

411 "forcedSourceOnDiaObjectTable",

412 "forcedSourceOnDiaObjectTable_tract",

413 "forcedSourceTable",

414 "forcedSourceTable_tract",

415 "goodSeeingCoadd",

416 "goodSeeingCoadd_nImage",

417 "goodSeeingDiff_assocDiaSrcTable",

418 "goodSeeingDiff_diaObjTable",

419 "goodSeeingDiff_diaSrc",

420 "goodSeeingDiff_diaSrcTable",

421 "goodSeeingDiff_differenceTempExp",

422 "goodSeeingDiff_fullDiaObjTable",

423 "goodSeeingDiff_matchedExp",

424 "goodSeeingDiff_templateExp",

425 "goodSeeingVisits",

426 "mergedForcedSource",

427 "mergedForcedSourceOnDiaObject",

428 },

429 )

430 tester.run(butler, self)

431

432 def test_hsc_drp_prod(self):

433 butler = self.makeButler(writeable=True)

434 tester = PipelineStepTester(

435 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

436 [

437 "#step1",

438 "#step2a",

439 "#step2b",

440 "#step2c",

441 "#step2d",

442 "#step2e",

443 "#step3",

444 "#step4",

445 "#step7",

446 ],

447 initial_dataset_types=REFCATS,

448 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

449 expected_outputs=COMMON_OUTPUTS

450 | HSC_OUTPUTS

451 | {

452 "fgcm_Cycle6_AtmosphereParameters",

453 "fgcm_Cycle0_FitParameters",

454 "fgcm_Cycle1_FitParameters",

455 "fgcm_Cycle2_FitParameters",

456 "fgcm_Cycle3_FitParameters",

457 "fgcm_Cycle4_FitParameters",

458 "fgcm_Cycle5_FitParameters",

459 "fgcm_Cycle6_FitParameters",

460 "fgcm_Cycle0_FlaggedStars",

461 "fgcm_Cycle1_FlaggedStars",

462 "fgcm_Cycle2_FlaggedStars",

463 "fgcm_Cycle3_FlaggedStars",

464 "fgcm_Cycle4_FlaggedStars",

465 "fgcm_Cycle5_FlaggedStars",

466 "fgcm_Cycle6_FlaggedStars",

467 "fgcm_reference_stars",

468 "fgcm_Cycle6_StandardStars",

469 "fgcm_star_ids",

470 "fgcm_star_observations",

471 "fgcm_Cycle6_Zeropoints",

472 "transmission_atmosphere_fgcm",

473 "fgcm_standard_star",

474 },

475 )

476 tester.run(butler, self)

477

478 def test_hsc_drp_rc2(self):

479 butler = self.makeButler(writeable=True)

480 tester = PipelineStepTester(

481 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

482 [

483 "#step1",

484 "#step2a",

485 "#step2b",

486 "#step2cde",

487 "#step3",

488 "#step4",

489 "#step5",

490 "#step6",

491 "#step7",

492 ],

493 initial_dataset_types=REFCATS,

494 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

495 expected_outputs=COMMON_OUTPUTS

496 | HSC_OUTPUTS

497 | {

498 "diaObjectTable_tract",

499 "diaSourceTable",

500 "diaSourceTable_tract",

501 "fgcm_Cycle4_AtmosphereParameters",

502 "fgcm_Cycle0_FitParameters",

503 "fgcm_Cycle1_FitParameters",

504 "fgcm_Cycle2_FitParameters",

505 "fgcm_Cycle3_FitParameters",

506 "fgcm_Cycle4_FitParameters",

507 "fgcm_Cycle0_FlaggedStars",

508 "fgcm_Cycle1_FlaggedStars",

509 "fgcm_Cycle2_FlaggedStars",

510 "fgcm_Cycle3_FlaggedStars",

511 "fgcm_Cycle4_FlaggedStars",

512 "fgcm_reference_stars",

513 "fgcm_Cycle4_StandardStars",

514 "fgcm_star_ids",

515 "fgcm_star_observations",

516 "fgcm_Cycle4_Zeropoints",

517 "fgcm_standard_star",

518 "forcedSourceOnDiaObjectTable",

519 "forcedSourceOnDiaObjectTable_tract",

520 "forcedSourceTable",

521 "forcedSourceTable_tract",

522 "goodSeeingCoadd",

523 "goodSeeingCoadd_nImage",

524 "goodSeeingDiff_assocDiaSrcTable",

525 "goodSeeingDiff_diaObjTable",

526 "goodSeeingDiff_diaSrc",

527 "goodSeeingDiff_diaSrcTable",

528 "goodSeeingDiff_differenceTempExp",

529 "goodSeeingDiff_fullDiaObjTable",

530 "goodSeeingDiff_matchedExp",

531 "goodSeeingDiff_templateExp",

532 "goodSeeingVisits",

533 "mergedForcedSource",

534 "mergedForcedSourceOnDiaObject",

535 "transmission_atmosphere_fgcm",

536 },

537 )

538 tester.run(butler, self)

539

540 def test_hsc_drp_rc2_subset(self):

541 butler = self.makeButler(writeable=True)

542 tester = PipelineStepTester(

543 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

544 [

545 "#nightlyStep1",

546 "#nightlyStep2a",

547 "#nightlyStep2b",

548 "#nightlyStep2c",

549 "#nightlyStep2d",

550 "#nightlyStep3",

551 "#nightlyStep4",

552 ],

553 initial_dataset_types=REFCATS,

554 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

555 expected_outputs=COMMON_OUTPUTS

556 | HSC_OUTPUTS

557 | {

558 "fgcm_Cycle4_AtmosphereParameters",

559 "fgcm_Cycle0_FitParameters",

560 "fgcm_Cycle1_FitParameters",

561 "fgcm_Cycle2_FitParameters",

562 "fgcm_Cycle3_FitParameters",

563 "fgcm_Cycle4_FitParameters",

564 "fgcm_Cycle0_FlaggedStars",

565 "fgcm_Cycle1_FlaggedStars",

566 "fgcm_Cycle2_FlaggedStars",

567 "fgcm_Cycle3_FlaggedStars",

568 "fgcm_Cycle4_FlaggedStars",

569 "fgcm_reference_stars",

570 "fgcm_Cycle4_StandardStars",

571 "fgcm_star_ids",

572 "fgcm_star_observations",

573 "fgcm_Cycle4_Zeropoints",

574 "transmission_atmosphere_fgcm",

575 "fgcm_standard_star",

576 },

577 )

578 tester.run(butler, self)

579

580 def test_latiss_drp(self):

581 butler = self.makeButler(writeable=True)

582 tester = PipelineStepTester(

583 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

584 [

585 "#step1",

586 "#step2a",

587 "#step2bcde",

588 "#step3a",

589 "#step3b",

590 "#step3c",

591 "#step4",

592 "#step5",

593 "#step6",

594 ],

595 initial_dataset_types=REFCATS,

596 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

597 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

598 )

599 tester.run(butler, self)

600

601 def test_lsstcam_quickLook(self):

602 butler = self.makeButler(writeable=True)

603 tester = PipelineStepTester(

604 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

605 [

606 "#step1a-single-visit-detectors",

607 "#step1b-single-visit-visits",

608 "#step1d-single-visit-global",

609 ],

610 initial_dataset_types=REFCATS,

611 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

612 expected_outputs=QUICKLOOK_V2_OUTPUTS,

613 )

614 tester.run(butler, self)

615

616 def test_lsstcam_quickLook_bias(self):

617 butler = self.makeButler(writeable=True)

618 tester = PipelineStepTester(

619 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

620 [

621 "#verifyBiasIsr",

622 ],

623 initial_dataset_types=[],

624 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

625 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

626 )

627 tester.run(butler, self)

628

629 def test_lsstcam_quickLook_dark(self):

630 butler = self.makeButler(writeable=True)

631 tester = PipelineStepTester(

632 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

633 [

634 "#verifyDarkIsr",

635 ],

636 initial_dataset_types=[],

637 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

638 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

639 )

640 tester.run(butler, self)

641

642 def test_lsstcam_quickLook_flat(self):

643 butler = self.makeButler(writeable=True)

644 tester = PipelineStepTester(

645 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

646 [

647 "#verifyFlatIsr",

648 ],

649 initial_dataset_types=[],

650 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

651 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

652 )

653 tester.run(butler, self)

654

655 def test_lsstcam_nightly_validation(self):

656 butler = self.makeButler(writeable=True)

657 tester = PipelineStepTester(

658 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

659 [

660 "#step1a-single-visit-detectors",

661 "#step1b-single-visit-visits",

662 "#step1c-single-visit-tracts",

663 "#step1d-single-visit-global",

664 "#stage3-coadd"

665

666 ],

667 initial_dataset_types=REFCATS,

668 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

669 # Check for some basic stage 1 outputs

670 expected_outputs={"preliminary_visit_image",

671 "preliminary_visit_summary",

672 "preliminary_visit_table",

673 "preliminary_visit_detector_table",

674 "single_visit_star_association_metrics",

675 "deep_coadd"}

676 )

677 tester.run(butler, self)

678

679 def test_lsstcam_drp(self):

680 butler = self.makeButler(writeable=True)

681 tester = PipelineStepTester(

682 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

683 [

684 "#stage1-single-visit",

685 "#stage2-recalibrate",

686 "#stage3-coadd",

687 "#stage4-measure-variability",

688 ],

689 initial_dataset_types=REFCATS,

690 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

691 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

692 )

693 tester.run(butler, self)

694

695 def test_lsstcam_drp_compat(self):

696 butler = self.makeButler(writeable=True)

697 tester = PipelineStepTester(

698 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

699 [

700 "#stage1-single-visit",

701 "#stage2-recalibrate",

702 "#stage3-coadd",

703 "#stage4-measure-variability",

704 ],

705 initial_dataset_types=REFCATS,

706 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

707 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

708 )

709 tester.run(butler, self)

710

711 def test_lsstcam_drp_fl(self):

712 butler = self.makeButler(writeable=True)

713 tester = PipelineStepTester(

714 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

715 [

716 "#stage1-single-visit",

717 "#stage2-recalibrate",

718 "#stage3-coadd",

719 "#stage4-measure-variability",

720 ],

721 initial_dataset_types=REFCATS,

722 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

723 # Check for some basic outputs

724 expected_outputs={"preliminary_visit_image",

725 "preliminary_visit_summary",

726 "deep_coadd"}

727 )

728 tester.run(butler, self)

729

730 def test_lsstcam_nv_fl(self):

731 butler = self.makeButler(writeable=True)

732 tester = PipelineStepTester(

733 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

734 [

735 "#stage1-single-visit",

736 "#stage2-recalibrate",

737 "#stage3-coadd",

738 ],

739 initial_dataset_types=REFCATS,

740 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

741 # Check for some basic outputs

742 expected_outputs={"preliminary_visit_image",

743 "preliminary_visit_summary",

744 "deep_coadd_predetection"}

745 )

746 tester.run(butler, self)

747

748 def test_lsstcam_imsim_drp_ci_imsim(self):

749 butler = self.makeButler(writeable=True)

750 tester = PipelineStepTester(

751 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

752 [""],

753 initial_dataset_types=REFCATS,

754 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

755 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

756 )

757 tester.run(butler, self)

758

759 def test_lsstcam_imsim_drp_test_med_1(self):

760 butler = self.makeButler(writeable=True)

761 tester = PipelineStepTester(

762 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

763 [f"#step{N}" for N in range(1, 8)],

764 initial_dataset_types=REFCATS,

765 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

766 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

767 )

768 tester.run(butler, self)

769

770 def test_lsstcam_imsim_drp(self):

771 butler = self.makeButler(writeable=True)

772 tester = PipelineStepTester(

773 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

774 [f"#step{N}" for N in range(1, 8)],

775 initial_dataset_types=REFCATS,

776 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

777 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

778 )

779 tester.run(butler, self)

780

781 def test_lsstcam_imsim_nightly_validation(self):

782 butler = self.makeButler(writeable=True)

783 tester = PipelineStepTester(

784 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

785 [

786 "#step1",

787 "#step2",

788 "#step3",

789 "#step7",

790 ],

791 initial_dataset_types=REFCATS,

792 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

793 expected_outputs=COMMON_OUTPUTS,

794 )

795 tester.run(butler, self)

796

797 def test_lsstcomcamsim_drp(self):

798 butler = self.makeButler(writeable=True)

799 tester = PipelineStepTester(

800 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

801 [

802 "#step1",

803 "#step2a",

804 "#step2b",

805 "#step2c",

806 "#step2d",

807 "#step2e",

808 "#step3",

809 "#step4",

810 "#step5",

811 "#step6",

812 "#step7",

813 ],

814 initial_dataset_types=REFCATS,

815 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

816 expected_outputs=COMMON_OUTPUTS,

817 )

818 tester.run(butler, self)

819

820 def test_lsstcomcamsim_nightly_validation(self):

821 butler = self.makeButler(writeable=True)

822 tester = PipelineStepTester(

823 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

824 [

825 "#step1",

826 "#step2a",

827 "#nightlyRollup",

828 "#step2b",

829 "#step2d",

830 "#step2e",

831 "#step3",

832 "#step7",

833 ],

834 initial_dataset_types=REFCATS,

835 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

836 expected_outputs=COMMON_OUTPUTS,

837 )

838 tester.run(butler, self)

839

840 def test_lsstcomcamsim_quickLook(self):

841 butler = self.makeButler(writeable=True)

842 tester = PipelineStepTester(

843 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

844 [

845 "#step1",

846 "#step2a",

847 "#nightlyRollup",

848 ],

849 initial_dataset_types=REFCATS,

850 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

851 expected_outputs=QUICKLOOK_OUTPUTS,

852 )

853 tester.run(butler, self)

854

855 def test_lsstcomcam_drp(self):

856 butler = self.makeButler(writeable=True)

857 tester = PipelineStepTester(

858 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

859 [

860 "#step1",

861 "#step2a",

862 "#step2b",

863 "#step2c",

864 "#step2d",

865 "#step2e",

866 "#step3a",

867 "#step3b",

868 "#step4",

869 "#step5",

870 "#step6",

871 "#step7",

872 ],

873 initial_dataset_types=REFCATS,

874 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

875 expected_outputs=COMMON_OUTPUTS,

876 )

877 tester.run(butler, self)

878

879 def test_lsstcomcam_nightly_validation(self):

880 butler = self.makeButler(writeable=True)

881 tester = PipelineStepTester(

882 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

883 [

884 "#step1",

885 "#step2a",

886 "#nightlyRollup",

887 "#step2b",

888 "#step2d",

889 "#step2e",

890 "#step3a",

891 "#step3b",

892 "#step7",

893 ],

894 initial_dataset_types=REFCATS,

895 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

896 expected_outputs=COMMON_OUTPUTS,

897 )

898 tester.run(butler, self)

899

900 def test_lsstcomcam_quickLook(self):

901 butler = self.makeButler(writeable=True)

902 tester = PipelineStepTester(

903 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

904 [

905 "#step1",

906 "#step2a",

907 "#nightlyRollup",

908 ],

909 initial_dataset_types=REFCATS,

910 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

911 expected_outputs=QUICKLOOK_OUTPUTS,

912 )

913 tester.run(butler, self)

914

915

916if __name__ == "__main__": 916 ↛ 917line 916 didn't jump to line 917 because the condition on line 916 was never true

917 unittest.main()

Coverage for tests / test_pipelines.py: 40%

149 statements