Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95 "gain_correction",

96}

98# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

99LSSTCAM_IMSIM_INPUTS = {

100 "bfk",

101 "cal_ref_cat_2_2",

102 "truth_summary",

103 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

104}

105

106# LSSTCam common inputs, in addition to COMMON_INPUTS

107LSSTCAM_INPUTS = {

108 "bfk",

109 "linearizer",

110 "defects",

111 "cti",

112 "the_monster_20250219",

113 "gaia_dr3_20230707",

114 "sky",

115 "astrometry_camera",

116 "gain_correction",

117}

118

119# LSSTComCam common inputs, in addition to COMMON_INPUTS

120LSSTCOMCAM_INPUTS = {

121 "bfk",

122 "ptc",

123 "linearizer",

124 "defects",

125 "cti",

126 "the_monster_20250219",

127 "illuminationCorrection",

128 "gain_correction",

129}

130

131# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

132LSSTCOMCAMSIM_INPUTS = {

133 "uw_stars_20240524",

134 "ptc",

135 "bfk",

136 "gain_correction",

137}

138

139# a selection of mostly common outputs

140COMMON_OUTPUTS = {

141 "ccdVisitTable",

142 "compare_warp_artifact_mask",

143 "deepCoaddVisits",

144 "deepCoadd",

145 "deepCoadd_calexp",

146 "deepCoadd_calexp_background",

147 "deepCoadd_det",

148 "deepCoadd_directWarp",

149 "deepCoadd_directWarp_maskedFraction",

150 "deepCoadd_forced_src",

151 "deepCoadd_inputMap",

152 "deepCoadd_meas",

153 "deepCoadd_mergeDet",

154 "deepCoadd_nImage",

155 "deepCoadd_obj",

156 "deepCoadd_psfMatchedWarp",

157 "deepCoadd_ref",

158 "deepCoadd_scarletModelData",

159 "finalized_src_table",

160 "finalVisitSummary",

161 "initial_astrometry_match_detector",

162 "initial_photometry_match_detector",

163 "initial_psf_stars_footprints_detector",

164 "initial_psf_stars_detector",

165 "calexp",

166 "calexpBackground",

167 "src",

168 "objectTable",

169 "objectTable_tract",

170 "postISRCCD",

171 "sourceTable_visit",

172 "visitSummary",

173 "visitTable",

174 "isolated_star_presource_associations",

175 "isolated_star_presources",

176}

177

178# Translated COMMON_OUTPUTS

179COMMON_V2_OUTPUTS = {

180 'visit_detector_table',

181 'compare_warp_artifact_mask',

182 'deep_coadd_visit_selection',

183 'deep_coadd_predetection',

184 'deep_coadd',

185 'deep_coadd_background',

186 'object_detection',

187 'direct_warp',

188 'direct_warp_masked_fraction',

189 'object_forced_measurement',

190 'deep_coadd_input_map',

191 'object_unforced_measurement',

192 'object_detection_merged',

193 'deep_coadd_n_image',

194 'object_unstandardized',

195 'psf_matched_warp',

196 'object_ref_measurement',

197 'object_scarlet_models',

198 'refit_psf_star',

199 'visit_summary',

200 'initial_astrometry_match_detector',

201 'initial_photometry_match_detector',

202 'single_visit_psf_star_footprints',

203 'single_visit_psf_star',

204 'preliminary_visit_image',

205 'preliminary_visit_image_background',

206 'single_visit_star_footprints',

207 'object_patch',

208 'object_all',

209 'post_isr_image',

210 'recalibrated_star',

211 'preliminary_visit_summary',

212 'visit_table',

213 'isolated_star_association',

214 'isolated_star'

215}

216

217# HSC common outputs, in addition to COMMON_OUTPUTS

218HSC_OUTPUTS = {

219 "calexp_skyCorr_visit_mosaic",

220 "calexpBackground_skyCorr_visit_mosaic",

221 "preSourceTable",

222 "preSourceTable_visit",

223 "skyCorr",

224}

225

226# LATISS common outputs, in addition to COMMON_OUTPUTS

227LATISS_OUTPUTS = {

228 "diaObjectTable_tract",

229 "diaSourceTable",

230 "diaSourceTable_tract",

231 "fgcm_Cycle0_FitParameters",

232 "fgcm_Cycle1_FitParameters",

233 "fgcm_Cycle2_FitParameters",

234 "fgcm_Cycle3_FitParameters",

235 "fgcm_Cycle4_FitParameters",

236 "fgcm_Cycle0_FlaggedStars",

237 "fgcm_Cycle1_FlaggedStars",

238 "fgcm_Cycle2_FlaggedStars",

239 "fgcm_Cycle3_FlaggedStars",

240 "fgcm_Cycle4_FlaggedStars",

241 "fgcm_reference_stars",

242 "fgcm_star_ids",

243 "fgcm_star_observations",

244 "fgcm_standard_star",

245 "forcedSourceOnDiaObjectTable",

246 "forcedSourceOnDiaObjectTable_tract",

247 "forcedSourceTable",

248 "forcedSourceTable_tract",

249 "goodSeeingCoadd",

250 "goodSeeingCoadd_nImage",

251 "goodSeeingDiff_assocDiaSrcTable",

252 "goodSeeingDiff_diaObjTable",

253 "goodSeeingDiff_diaSrc",

254 "goodSeeingDiff_diaSrcTable",

255 "goodSeeingDiff_differenceExp",

256 "goodSeeingDiff_differenceTempExp",

257 "goodSeeingDiff_fullDiaObjTable",

258 "goodSeeingDiff_matchedExp",

259 "goodSeeingDiff_templateExp",

260 "goodSeeingVisits",

261 "mergedForcedSource",

262 "mergedForcedSourceOnDiaObject",

263 "transmission_atmosphere_fgcm",

264}

265

266# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

267LSSTCAM_IMSIM_OUTPUTS = {

268 "diaObjectTable_tract",

269 "diaSourceTable",

270 "diaSourceTable_tract",

271 "forcedSourceOnDiaObjectTable",

272 "forcedSourceOnDiaObjectTable_tract",

273 "forcedSourceTable",

274 "forcedSourceTable_tract",

275 "goodSeeingCoadd",

276 "goodSeeingCoadd_nImage",

277 "goodSeeingDiff_assocDiaSrcTable",

278 "goodSeeingDiff_diaObjTable",

279 "goodSeeingDiff_diaSrc",

280 "goodSeeingDiff_diaSrcTable",

281 "goodSeeingDiff_differenceExp",

282 "goodSeeingDiff_differenceTempExp",

283 "goodSeeingDiff_fullDiaObjTable",

284 "goodSeeingDiff_matchedExp",

285 "goodSeeingDiff_templateExp",

286 "goodSeeingVisits",

287 "mergedForcedSource",

288 "mergedForcedSourceOnDiaObject",

289}

290

291# v2 outputs common to all "quickLook" pipelines, which only include

292QUICKLOOK_V2_OUTPUTS = {

293 "preliminary_visit_image", "preliminary_visit_summary"

294}

295

296# Outputs common to all "quickLook" pipelines, which only include

297QUICKLOOK_OUTPUTS = {

298 "calexp", "visitSummary"

299}

300

301QUICKLOOK_BIAS_OUTPUTS = {

302 "post_isr_image",

303}

304

305QUICKLOOK_DARK_OUTPUTS = {

306 "post_isr_image",

307}

308

309QUICKLOOK_FLAT_OUTPUTS = {

310 "post_isr_image",

311}

312

313# All refcats used by any pipelines, for inclusion as the initial_dataset_types

314# argument to PipelineStepTester; the intent is just to allow the 'skypix'

315# dimension placeholder used in these connections to be resolved.

316REFCATS = [

317 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

318 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

319 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

320 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

321 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

322 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

323 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

324]

325

326

327class PipelineTestCase(unittest.TestCase):

328 def setUp(self):

329 self.root = makeTestTempDir(TEST_DIR)

330 self.maxDiff = None

331

332 def tearDown(self):

333 removeTestTempDir(self.root)

334

335 def makeButler(self, **kwargs: Any) -> Butler:

336 """Return new Butler instance on each call."""

337 config = Config()

338

339 # make separate temporary directory for registry of this instance

340 tmpdir = tempfile.mkdtemp(dir=self.root)

341 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

342 config = Butler.makeRepo(self.root, config)

343 butler = Butler.from_config(config, **kwargs)

344 DatastoreMock.apply(butler)

345 return butler

346

347 def test_decam_isrForCrosstalkSources(self):

348 butler = self.makeButler(writeable=True)

349 tester = PipelineStepTester(

350 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

351 ["#step0"],

352 initial_dataset_types=REFCATS,

353 expected_inputs={

354 "camera",

355 "raw",

356 },

357 expected_outputs={"overscanRaw"},

358 )

359 tester.run(butler, self)

360

361 def test_decam_drp_merian(self):

362 butler = self.makeButler(writeable=True)

363 tester = PipelineStepTester(

364 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

365 [

366 "#step1",

367 "#step2a",

368 "#step2b",

369 "#step2d",

370 "#step2e",

371 "#step3",

372 ],

373 initial_dataset_types=REFCATS,

374 expected_inputs=COMMON_INPUTS

375 | {

376 "defects",

377 "gaia_dr2_20200414",

378 "gaia_dr3_20230707",

379 "linearizer",

380 "overscanRaw",

381 "ps1_pv3_3pi_20170110",

382 },

383 expected_outputs=COMMON_OUTPUTS

384 | {

385 "goodSeeingCoadd",

386 "goodSeeingCoadd_nImage",

387 "goodSeeingVisits",

388 "preSourceTable",

389 "preSourceTable_visit",

390 },

391 )

392 tester.run(butler, self)

393

394 def test_hsc_drp_ci_hsc(self):

395 butler = self.makeButler(writeable=True)

396 tester = PipelineStepTester(

397 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

398 [""],

399 initial_dataset_types=REFCATS,

400 expected_inputs=COMMON_INPUTS

401 | HSC_INPUTS

402 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

403 expected_outputs=COMMON_OUTPUTS

404 | HSC_OUTPUTS

405 | {

406 "diaObjectTable_tract",

407 "diaSourceTable",

408 "diaSourceTable_tract",

409 "forcedSourceOnDiaObjectTable",

410 "forcedSourceOnDiaObjectTable_tract",

411 "forcedSourceTable",

412 "forcedSourceTable_tract",

413 "goodSeeingCoadd",

414 "goodSeeingCoadd_nImage",

415 "goodSeeingDiff_assocDiaSrcTable",

416 "goodSeeingDiff_diaObjTable",

417 "goodSeeingDiff_diaSrc",

418 "goodSeeingDiff_diaSrcTable",

419 "goodSeeingDiff_differenceTempExp",

420 "goodSeeingDiff_fullDiaObjTable",

421 "goodSeeingDiff_matchedExp",

422 "goodSeeingDiff_templateExp",

423 "goodSeeingVisits",

424 "mergedForcedSource",

425 "mergedForcedSourceOnDiaObject",

426 },

427 )

428 tester.run(butler, self)

429

430 def test_hsc_drp_prod(self):

431 butler = self.makeButler(writeable=True)

432 tester = PipelineStepTester(

433 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

434 [

435 "#step1",

436 "#step2a",

437 "#step2b",

438 "#step2c",

439 "#step2d",

440 "#step2e",

441 "#step3",

442 "#step4",

443 "#step7",

444 ],

445 initial_dataset_types=REFCATS,

446 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

447 expected_outputs=COMMON_OUTPUTS

448 | HSC_OUTPUTS

449 | {

450 "fgcm_Cycle6_AtmosphereParameters",

451 "fgcm_Cycle0_FitParameters",

452 "fgcm_Cycle1_FitParameters",

453 "fgcm_Cycle2_FitParameters",

454 "fgcm_Cycle3_FitParameters",

455 "fgcm_Cycle4_FitParameters",

456 "fgcm_Cycle5_FitParameters",

457 "fgcm_Cycle6_FitParameters",

458 "fgcm_Cycle0_FlaggedStars",

459 "fgcm_Cycle1_FlaggedStars",

460 "fgcm_Cycle2_FlaggedStars",

461 "fgcm_Cycle3_FlaggedStars",

462 "fgcm_Cycle4_FlaggedStars",

463 "fgcm_Cycle5_FlaggedStars",

464 "fgcm_Cycle6_FlaggedStars",

465 "fgcm_reference_stars",

466 "fgcm_Cycle6_StandardStars",

467 "fgcm_star_ids",

468 "fgcm_star_observations",

469 "fgcm_Cycle6_Zeropoints",

470 "transmission_atmosphere_fgcm",

471 "fgcm_standard_star",

472 },

473 )

474 tester.run(butler, self)

475

476 def test_hsc_drp_rc2(self):

477 butler = self.makeButler(writeable=True)

478 tester = PipelineStepTester(

479 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

480 [

481 "#step1",

482 "#step2a",

483 "#step2b",

484 "#step2cde",

485 "#step3",

486 "#step4",

487 "#step5",

488 "#step6",

489 "#step7",

490 ],

491 initial_dataset_types=REFCATS,

492 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

493 expected_outputs=COMMON_OUTPUTS

494 | HSC_OUTPUTS

495 | {

496 "diaObjectTable_tract",

497 "diaSourceTable",

498 "diaSourceTable_tract",

499 "fgcm_Cycle4_AtmosphereParameters",

500 "fgcm_Cycle0_FitParameters",

501 "fgcm_Cycle1_FitParameters",

502 "fgcm_Cycle2_FitParameters",

503 "fgcm_Cycle3_FitParameters",

504 "fgcm_Cycle4_FitParameters",

505 "fgcm_Cycle0_FlaggedStars",

506 "fgcm_Cycle1_FlaggedStars",

507 "fgcm_Cycle2_FlaggedStars",

508 "fgcm_Cycle3_FlaggedStars",

509 "fgcm_Cycle4_FlaggedStars",

510 "fgcm_reference_stars",

511 "fgcm_Cycle4_StandardStars",

512 "fgcm_star_ids",

513 "fgcm_star_observations",

514 "fgcm_Cycle4_Zeropoints",

515 "fgcm_standard_star",

516 "forcedSourceOnDiaObjectTable",

517 "forcedSourceOnDiaObjectTable_tract",

518 "forcedSourceTable",

519 "forcedSourceTable_tract",

520 "goodSeeingCoadd",

521 "goodSeeingCoadd_nImage",

522 "goodSeeingDiff_assocDiaSrcTable",

523 "goodSeeingDiff_diaObjTable",

524 "goodSeeingDiff_diaSrc",

525 "goodSeeingDiff_diaSrcTable",

526 "goodSeeingDiff_differenceTempExp",

527 "goodSeeingDiff_fullDiaObjTable",

528 "goodSeeingDiff_matchedExp",

529 "goodSeeingDiff_templateExp",

530 "goodSeeingVisits",

531 "mergedForcedSource",

532 "mergedForcedSourceOnDiaObject",

533 "transmission_atmosphere_fgcm",

534 },

535 )

536 tester.run(butler, self)

537

538 def test_hsc_drp_rc2_subset(self):

539 butler = self.makeButler(writeable=True)

540 tester = PipelineStepTester(

541 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

542 [

543 "#nightlyStep1",

544 "#nightlyStep2a",

545 "#nightlyStep2b",

546 "#nightlyStep2c",

547 "#nightlyStep2d",

548 "#nightlyStep3",

549 "#nightlyStep4",

550 ],

551 initial_dataset_types=REFCATS,

552 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

553 expected_outputs=COMMON_OUTPUTS

554 | HSC_OUTPUTS

555 | {

556 "fgcm_Cycle4_AtmosphereParameters",

557 "fgcm_Cycle0_FitParameters",

558 "fgcm_Cycle1_FitParameters",

559 "fgcm_Cycle2_FitParameters",

560 "fgcm_Cycle3_FitParameters",

561 "fgcm_Cycle4_FitParameters",

562 "fgcm_Cycle0_FlaggedStars",

563 "fgcm_Cycle1_FlaggedStars",

564 "fgcm_Cycle2_FlaggedStars",

565 "fgcm_Cycle3_FlaggedStars",

566 "fgcm_Cycle4_FlaggedStars",

567 "fgcm_reference_stars",

568 "fgcm_Cycle4_StandardStars",

569 "fgcm_star_ids",

570 "fgcm_star_observations",

571 "fgcm_Cycle4_Zeropoints",

572 "transmission_atmosphere_fgcm",

573 "fgcm_standard_star",

574 },

575 )

576 tester.run(butler, self)

577

578 def test_latiss_drp(self):

579 butler = self.makeButler(writeable=True)

580 tester = PipelineStepTester(

581 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

582 [

583 "#step1",

584 "#step2a",

585 "#step2bcde",

586 "#step3a",

587 "#step3b",

588 "#step3c",

589 "#step4",

590 "#step5",

591 "#step6",

592 ],

593 initial_dataset_types=REFCATS,

594 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

595 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

596 )

597 tester.run(butler, self)

598

599 def test_lsstcam_quickLook(self):

600 butler = self.makeButler(writeable=True)

601 tester = PipelineStepTester(

602 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

603 [

604 "#step1a-single-visit-detectors",

605 "#step1b-single-visit-visits",

606 "#step1d-single-visit-global",

607 ],

608 initial_dataset_types=REFCATS,

609 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

610 expected_outputs=QUICKLOOK_V2_OUTPUTS,

611 )

612 tester.run(butler, self)

613

614 def test_lsstcam_quickLook_bias(self):

615 butler = self.makeButler(writeable=True)

616 tester = PipelineStepTester(

617 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

618 [

619 "#verifyBiasIsr",

620 ],

621 initial_dataset_types=[],

622 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

623 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

624 )

625 tester.run(butler, self)

626

627 def test_lsstcam_quickLook_dark(self):

628 butler = self.makeButler(writeable=True)

629 tester = PipelineStepTester(

630 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

631 [

632 "#verifyDarkIsr",

633 ],

634 initial_dataset_types=[],

635 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

636 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

637 )

638 tester.run(butler, self)

639

640 def test_lsstcam_quickLook_flat(self):

641 butler = self.makeButler(writeable=True)

642 tester = PipelineStepTester(

643 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

644 [

645 "#verifyFlatIsr",

646 ],

647 initial_dataset_types=[],

648 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

649 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

650 )

651 tester.run(butler, self)

652

653 def test_lsstcam_nightly_validation(self):

654 butler = self.makeButler(writeable=True)

655 tester = PipelineStepTester(

656 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

657 [

658 "#step1a-single-visit-detectors",

659 "#step1b-single-visit-visits",

660 "#step1c-single-visit-tracts",

661 "#step1d-single-visit-global",

662 "#stage3-coadd"

663

664 ],

665 initial_dataset_types=REFCATS,

666 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

667 # Check for some basic stage 1 outputs

668 expected_outputs={"preliminary_visit_image",

669 "preliminary_visit_summary",

670 "preliminary_visit_table",

671 "preliminary_visit_detector_table",

672 "single_visit_star_association_metrics",

673 "deep_coadd"}

674 )

675 tester.run(butler, self)

676

677 def test_lsstcam_drp(self):

678 butler = self.makeButler(writeable=True)

679 tester = PipelineStepTester(

680 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

681 [

682 "#stage1-single-visit",

683 "#stage2-recalibrate",

684 "#stage3-coadd",

685 "#stage4-measure-variability",

686 ],

687 initial_dataset_types=REFCATS,

688 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

689 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

690 )

691 tester.run(butler, self)

692

693 def test_lsstcam_drp_compat(self):

694 butler = self.makeButler(writeable=True)

695 tester = PipelineStepTester(

696 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

697 [

698 "#stage1-single-visit",

699 "#stage2-recalibrate",

700 "#stage3-coadd",

701 "#stage4-measure-variability",

702 ],

703 initial_dataset_types=REFCATS,

704 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

705 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

706 )

707 tester.run(butler, self)

708

709 def test_lsstcam_drp_fl(self):

710 butler = self.makeButler(writeable=True)

711 tester = PipelineStepTester(

712 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

713 [

714 "#stage1-single-visit",

715 "#stage2-recalibrate",

716 "#stage3-coadd",

717 "#stage4-measure-variability",

718 ],

719 initial_dataset_types=REFCATS,

720 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

721 # Check for some basic outputs

722 expected_outputs={"preliminary_visit_image",

723 "preliminary_visit_summary",

724 "deep_coadd"}

725 )

726 tester.run(butler, self)

727

728 def test_lsstcam_nv_fl(self):

729 butler = self.makeButler(writeable=True)

730 tester = PipelineStepTester(

731 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

732 [

733 "#stage1-single-visit",

734 "#stage2-recalibrate",

735 "#stage3-coadd",

736 ],

737 initial_dataset_types=REFCATS,

738 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

739 # Check for some basic outputs

740 expected_outputs={"preliminary_visit_image",

741 "preliminary_visit_summary",

742 "deep_coadd_predetection"}

743 )

744 tester.run(butler, self)

745

746 def test_lsstcam_imsim_drp_ci_imsim(self):

747 butler = self.makeButler(writeable=True)

748 tester = PipelineStepTester(

749 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

750 [""],

751 initial_dataset_types=REFCATS,

752 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

753 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

754 )

755 tester.run(butler, self)

756

757 def test_lsstcam_imsim_drp_test_med_1(self):

758 butler = self.makeButler(writeable=True)

759 tester = PipelineStepTester(

760 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

761 [f"#step{N}" for N in range(1, 8)],

762 initial_dataset_types=REFCATS,

763 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

764 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

765 )

766 tester.run(butler, self)

767

768 def test_lsstcam_imsim_drp(self):

769 butler = self.makeButler(writeable=True)

770 tester = PipelineStepTester(

771 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

772 [f"#step{N}" for N in range(1, 8)],

773 initial_dataset_types=REFCATS,

774 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

775 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

776 )

777 tester.run(butler, self)

778

779 def test_lsstcam_imsim_nightly_validation(self):

780 butler = self.makeButler(writeable=True)

781 tester = PipelineStepTester(

782 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

783 [

784 "#step1",

785 "#step2",

786 "#step3",

787 "#step7",

788 ],

789 initial_dataset_types=REFCATS,

790 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

791 expected_outputs=COMMON_OUTPUTS,

792 )

793 tester.run(butler, self)

794

795 def test_lsstcomcamsim_drp(self):

796 butler = self.makeButler(writeable=True)

797 tester = PipelineStepTester(

798 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

799 [

800 "#step1",

801 "#step2a",

802 "#step2b",

803 "#step2c",

804 "#step2d",

805 "#step2e",

806 "#step3",

807 "#step4",

808 "#step5",

809 "#step6",

810 "#step7",

811 ],

812 initial_dataset_types=REFCATS,

813 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

814 expected_outputs=COMMON_OUTPUTS,

815 )

816 tester.run(butler, self)

817

818 def test_lsstcomcamsim_nightly_validation(self):

819 butler = self.makeButler(writeable=True)

820 tester = PipelineStepTester(

821 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

822 [

823 "#step1",

824 "#step2a",

825 "#nightlyRollup",

826 "#step2b",

827 "#step2d",

828 "#step2e",

829 "#step3",

830 "#step7",

831 ],

832 initial_dataset_types=REFCATS,

833 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

834 expected_outputs=COMMON_OUTPUTS,

835 )

836 tester.run(butler, self)

837

838 def test_lsstcomcamsim_quickLook(self):

839 butler = self.makeButler(writeable=True)

840 tester = PipelineStepTester(

841 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

842 [

843 "#step1",

844 "#step2a",

845 "#nightlyRollup",

846 ],

847 initial_dataset_types=REFCATS,

848 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

849 expected_outputs=QUICKLOOK_OUTPUTS,

850 )

851 tester.run(butler, self)

852

853 def test_lsstcomcam_drp(self):

854 butler = self.makeButler(writeable=True)

855 tester = PipelineStepTester(

856 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

857 [

858 "#step1",

859 "#step2a",

860 "#step2b",

861 "#step2c",

862 "#step2d",

863 "#step2e",

864 "#step3a",

865 "#step3b",

866 "#step4",

867 "#step5",

868 "#step6",

869 "#step7",

870 ],

871 initial_dataset_types=REFCATS,

872 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

873 expected_outputs=COMMON_OUTPUTS,

874 )

875 tester.run(butler, self)

876

877 def test_lsstcomcam_nightly_validation(self):

878 butler = self.makeButler(writeable=True)

879 tester = PipelineStepTester(

880 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

881 [

882 "#step1",

883 "#step2a",

884 "#nightlyRollup",

885 "#step2b",

886 "#step2d",

887 "#step2e",

888 "#step3a",

889 "#step3b",

890 "#step7",

891 ],

892 initial_dataset_types=REFCATS,

893 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

894 expected_outputs=COMMON_OUTPUTS,

895 )

896 tester.run(butler, self)

897

898 def test_lsstcomcam_quickLook(self):

899 butler = self.makeButler(writeable=True)

900 tester = PipelineStepTester(

901 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

902 [

903 "#step1",

904 "#step2a",

905 "#nightlyRollup",

906 ],

907 initial_dataset_types=REFCATS,

908 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

909 expected_outputs=QUICKLOOK_OUTPUTS,

910 )

911 tester.run(butler, self)

912

913

914if __name__ == "__main__": 914 ↛ 915line 914 didn't jump to line 915 because the condition on line 914 was never true

915 unittest.main()