Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 "preloaded_DRP_SsObjects",

57}

59# HSC common inputs, in addition to COMMON_INPUTS

60HSC_INPUTS = {

61 "brightObjectMask",

62 "brighterFatterKernel",

63 "defects",

64 "gaia_dr2_20200414",

65 "gaia_dr3_20230707",

66 "linearizer",

67 "ps1_pv3_3pi_20170110",

68 "sky",

69 "transmission_atmosphere",

70 "transmission_filter",

71 "transmission_optics",

72 "transmission_sensor",

73 "yBackground",

74}

76# LATISS common inputs, in addition to COMMON_INPUTS

77LATISS_INPUTS = {

78 "atlas_refcat2_20220201",

79 "defects",

80 "gaia_dr3_20230707",

81 "sky",

82 "transmission_atmosphere_fgcm",

83}

85# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

86LSSTCAM_IMSIM_INPUTS = {

87 "bfk",

88 "cal_ref_cat_2_2",

89 "truth_summary",

90 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

91}

93# LSSTCam common inputs, in addition to COMMON_INPUTS

94LSSTCAM_INPUTS = {

95 "bfk",

96 "linearizer",

97 "defects",

98 "cti",

99 "the_monster_20250219",

100 "gaia_dr3_20230707",

101 "sky",

102 "astrometry_camera",

103}

104

105# LSSTComCam common inputs, in addition to COMMON_INPUTS

106LSSTCOMCAM_INPUTS = {

107 "bfk",

108 "ptc",

109 "linearizer",

110 "defects",

111 "cti",

112 "the_monster_20250219",

113 "illuminationCorrection",

114}

115

116# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

117LSSTCOMCAMSIM_INPUTS = {

118 "uw_stars_20240524",

119 "ptc",

120 "bfk",

121}

122

123# a selection of mostly common outputs

124COMMON_OUTPUTS = {

125 "ccdVisitTable",

126 "compare_warp_artifact_mask",

127 "deepCoaddVisits",

128 "deepCoadd",

129 "deepCoadd_calexp",

130 "deepCoadd_calexp_background",

131 "deepCoadd_det",

132 "deepCoadd_directWarp",

133 "deepCoadd_directWarp_maskedFraction",

134 "deepCoadd_forced_src",

135 "deepCoadd_inputMap",

136 "deepCoadd_meas",

137 "deepCoadd_mergeDet",

138 "deepCoadd_nImage",

139 "deepCoadd_obj",

140 "deepCoadd_psfMatchedWarp",

141 "deepCoadd_ref",

142 "deepCoadd_scarletModelData",

143 "finalized_src_table",

144 "finalVisitSummary",

145 "initial_astrometry_match_detector",

146 "initial_photometry_match_detector",

147 "initial_psf_stars_footprints_detector",

148 "initial_psf_stars_detector",

149 "calexp",

150 "calexpBackground",

151 "src",

152 "objectTable",

153 "objectTable_tract",

154 "postISRCCD",

155 "sourceTable_visit",

156 "visitSummary",

157 "visitTable",

158 "isolated_star_presource_associations",

159 "isolated_star_presources",

160}

161

162# Translated COMMON_OUTPUTS

163COMMON_V2_OUTPUTS = {

164 'visit_detector_table',

165 'compare_warp_artifact_mask',

166 'deep_coadd_visit_selection',

167 'deep_coadd_predetection',

168 'deep_coadd',

169 'deep_coadd_background',

170 'object_detection',

171 'direct_warp',

172 'direct_warp_masked_fraction',

173 'object_forced_measurement',

174 'deep_coadd_input_map',

175 'object_unforced_measurement',

176 'object_detection_merged',

177 'deep_coadd_n_image',

178 'object_unstandardized',

179 'psf_matched_warp',

180 'object_ref_measurement',

181 'object_scarlet_models',

182 'refit_psf_star',

183 'visit_summary',

184 'initial_astrometry_match_detector',

185 'initial_photometry_match_detector',

186 'single_visit_psf_star_footprints',

187 'single_visit_psf_star',

188 'preliminary_visit_image',

189 'preliminary_visit_image_background',

190 'single_visit_star_footprints',

191 'object_patch',

192 'object_all',

193 'post_isr_image',

194 'recalibrated_star',

195 'preliminary_visit_summary',

196 'visit_table',

197 'isolated_star_association',

198 'isolated_star'

199}

200

201# HSC common outputs, in addition to COMMON_OUTPUTS

202HSC_OUTPUTS = {

203 "calexp_skyCorr_visit_mosaic",

204 "calexpBackground_skyCorr_visit_mosaic",

205 "preSourceTable",

206 "preSourceTable_visit",

207 "skyCorr",

208}

209

210# LATISS common outputs, in addition to COMMON_OUTPUTS

211LATISS_OUTPUTS = {

212 "diaObjectTable_tract",

213 "diaSourceTable",

214 "diaSourceTable_tract",

215 "fgcm_Cycle0_FitParameters",

216 "fgcm_Cycle1_FitParameters",

217 "fgcm_Cycle2_FitParameters",

218 "fgcm_Cycle3_FitParameters",

219 "fgcm_Cycle4_FitParameters",

220 "fgcm_Cycle0_FlaggedStars",

221 "fgcm_Cycle1_FlaggedStars",

222 "fgcm_Cycle2_FlaggedStars",

223 "fgcm_Cycle3_FlaggedStars",

224 "fgcm_Cycle4_FlaggedStars",

225 "fgcm_reference_stars",

226 "fgcm_star_ids",

227 "fgcm_star_observations",

228 "fgcm_standard_star",

229 "forcedSourceOnDiaObjectTable",

230 "forcedSourceOnDiaObjectTable_tract",

231 "forcedSourceTable",

232 "forcedSourceTable_tract",

233 "goodSeeingCoadd",

234 "goodSeeingCoadd_nImage",

235 "goodSeeingDiff_assocDiaSrcTable",

236 "goodSeeingDiff_diaObjTable",

237 "goodSeeingDiff_diaSrc",

238 "goodSeeingDiff_diaSrcTable",

239 "goodSeeingDiff_differenceExp",

240 "goodSeeingDiff_differenceTempExp",

241 "goodSeeingDiff_fullDiaObjTable",

242 "goodSeeingDiff_matchedExp",

243 "goodSeeingDiff_templateExp",

244 "goodSeeingVisits",

245 "mergedForcedSource",

246 "mergedForcedSourceOnDiaObject",

247 "transmission_atmosphere_fgcm",

248}

249

250# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

251LSSTCAM_IMSIM_OUTPUTS = {

252 "diaObjectTable_tract",

253 "diaSourceTable",

254 "diaSourceTable_tract",

255 "forcedSourceOnDiaObjectTable",

256 "forcedSourceOnDiaObjectTable_tract",

257 "forcedSourceTable",

258 "forcedSourceTable_tract",

259 "goodSeeingCoadd",

260 "goodSeeingCoadd_nImage",

261 "goodSeeingDiff_assocDiaSrcTable",

262 "goodSeeingDiff_diaObjTable",

263 "goodSeeingDiff_diaSrc",

264 "goodSeeingDiff_diaSrcTable",

265 "goodSeeingDiff_differenceExp",

266 "goodSeeingDiff_differenceTempExp",

267 "goodSeeingDiff_fullDiaObjTable",

268 "goodSeeingDiff_matchedExp",

269 "goodSeeingDiff_templateExp",

270 "goodSeeingVisits",

271 "mergedForcedSource",

272 "mergedForcedSourceOnDiaObject",

273}

274

275# v2 outputs common to all "quickLook" pipelines, which only include

276QUICKLOOK_V2_OUTPUTS = {

277 "preliminary_visit_image", "preliminary_visit_summary"

278}

279

280# Outputs common to all "quickLook" pipelines, which only include

281QUICKLOOK_OUTPUTS = {

282 "calexp", "visitSummary"

283}

284

285QUICKLOOK_BIAS_OUTPUTS = {

286 "post_isr_image",

287}

288

289QUICKLOOK_DARK_OUTPUTS = {

290 "post_isr_image",

291}

292

293QUICKLOOK_FLAT_OUTPUTS = {

294 "post_isr_image",

295}

296

297# All refcats used by any pipelines, for inclusion as the initial_dataset_types

298# argument to PipelineStepTester; the intent is just to allow the 'skypix'

299# dimension placeholder used in these connections to be resolved.

300REFCATS = [

301 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

302 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

303 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

304 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

305 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

306 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

307 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

308]

309

310

311class PipelineTestCase(unittest.TestCase):

312 def setUp(self):

313 self.root = makeTestTempDir(TEST_DIR)

314 self.maxDiff = None

315

316 def tearDown(self):

317 removeTestTempDir(self.root)

318

319 def makeButler(self, **kwargs: Any) -> Butler:

320 """Return new Butler instance on each call."""

321 config = Config()

322

323 # make separate temporary directory for registry of this instance

324 tmpdir = tempfile.mkdtemp(dir=self.root)

325 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

326 config = Butler.makeRepo(self.root, config)

327 butler = Butler.from_config(config, **kwargs)

328 DatastoreMock.apply(butler)

329 return butler

330

331 def test_decam_isrForCrosstalkSources(self):

332 butler = self.makeButler(writeable=True)

333 tester = PipelineStepTester(

334 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

335 ["#step0"],

336 initial_dataset_types=REFCATS,

337 expected_inputs={

338 "camera",

339 "raw",

340 },

341 expected_outputs={"overscanRaw"},

342 )

343 tester.run(butler, self)

344

345 def test_decam_drp_merian(self):

346 butler = self.makeButler(writeable=True)

347 tester = PipelineStepTester(

348 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

349 [

350 "#step1",

351 "#step2a",

352 "#step2b",

353 "#step2d",

354 "#step2e",

355 "#step3",

356 ],

357 initial_dataset_types=REFCATS,

358 expected_inputs=COMMON_INPUTS

359 | {

360 "defects",

361 "gaia_dr2_20200414",

362 "gaia_dr3_20230707",

363 "linearizer",

364 "overscanRaw",

365 "ps1_pv3_3pi_20170110",

366 },

367 expected_outputs=COMMON_OUTPUTS

368 | {

369 "goodSeeingCoadd",

370 "goodSeeingCoadd_nImage",

371 "goodSeeingVisits",

372 "preSourceTable",

373 "preSourceTable_visit",

374 },

375 )

376 tester.run(butler, self)

377

378 def test_hsc_drp_ci_hsc(self):

379 butler = self.makeButler(writeable=True)

380 tester = PipelineStepTester(

381 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

382 [""],

383 initial_dataset_types=REFCATS,

384 expected_inputs=COMMON_INPUTS

385 | HSC_INPUTS

386 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

387 expected_outputs=COMMON_OUTPUTS

388 | HSC_OUTPUTS

389 | {

390 "diaObjectTable_tract",

391 "diaSourceTable",

392 "diaSourceTable_tract",

393 "forcedSourceOnDiaObjectTable",

394 "forcedSourceOnDiaObjectTable_tract",

395 "forcedSourceTable",

396 "forcedSourceTable_tract",

397 "goodSeeingCoadd",

398 "goodSeeingCoadd_nImage",

399 "goodSeeingDiff_assocDiaSrcTable",

400 "goodSeeingDiff_diaObjTable",

401 "goodSeeingDiff_diaSrc",

402 "goodSeeingDiff_diaSrcTable",

403 "goodSeeingDiff_differenceTempExp",

404 "goodSeeingDiff_fullDiaObjTable",

405 "goodSeeingDiff_matchedExp",

406 "goodSeeingDiff_templateExp",

407 "goodSeeingVisits",

408 "mergedForcedSource",

409 "mergedForcedSourceOnDiaObject",

410 },

411 )

412 tester.run(butler, self)

413

414 def test_hsc_drp_prod(self):

415 butler = self.makeButler(writeable=True)

416 tester = PipelineStepTester(

417 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

418 [

419 "#step1",

420 "#step2a",

421 "#step2b",

422 "#step2c",

423 "#step2d",

424 "#step2e",

425 "#step3",

426 "#step4",

427 "#step7",

428 ],

429 initial_dataset_types=REFCATS,

430 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

431 expected_outputs=COMMON_OUTPUTS

432 | HSC_OUTPUTS

433 | {

434 "fgcm_Cycle6_AtmosphereParameters",

435 "fgcm_Cycle0_FitParameters",

436 "fgcm_Cycle1_FitParameters",

437 "fgcm_Cycle2_FitParameters",

438 "fgcm_Cycle3_FitParameters",

439 "fgcm_Cycle4_FitParameters",

440 "fgcm_Cycle5_FitParameters",

441 "fgcm_Cycle6_FitParameters",

442 "fgcm_Cycle0_FlaggedStars",

443 "fgcm_Cycle1_FlaggedStars",

444 "fgcm_Cycle2_FlaggedStars",

445 "fgcm_Cycle3_FlaggedStars",

446 "fgcm_Cycle4_FlaggedStars",

447 "fgcm_Cycle5_FlaggedStars",

448 "fgcm_Cycle6_FlaggedStars",

449 "fgcm_reference_stars",

450 "fgcm_Cycle6_StandardStars",

451 "fgcm_star_ids",

452 "fgcm_star_observations",

453 "fgcm_Cycle6_Zeropoints",

454 "transmission_atmosphere_fgcm",

455 "fgcm_standard_star",

456 },

457 )

458 tester.run(butler, self)

459

460 def test_hsc_drp_rc2(self):

461 butler = self.makeButler(writeable=True)

462 tester = PipelineStepTester(

463 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

464 [

465 "#step1",

466 "#step2a",

467 "#step2b",

468 "#step2cde",

469 "#step3",

470 "#step4",

471 "#step5",

472 "#step6",

473 "#step7",

474 ],

475 initial_dataset_types=REFCATS,

476 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

477 expected_outputs=COMMON_OUTPUTS

478 | HSC_OUTPUTS

479 | {

480 "diaObjectTable_tract",

481 "diaSourceTable",

482 "diaSourceTable_tract",

483 "fgcm_Cycle4_AtmosphereParameters",

484 "fgcm_Cycle0_FitParameters",

485 "fgcm_Cycle1_FitParameters",

486 "fgcm_Cycle2_FitParameters",

487 "fgcm_Cycle3_FitParameters",

488 "fgcm_Cycle4_FitParameters",

489 "fgcm_Cycle0_FlaggedStars",

490 "fgcm_Cycle1_FlaggedStars",

491 "fgcm_Cycle2_FlaggedStars",

492 "fgcm_Cycle3_FlaggedStars",

493 "fgcm_Cycle4_FlaggedStars",

494 "fgcm_reference_stars",

495 "fgcm_Cycle4_StandardStars",

496 "fgcm_star_ids",

497 "fgcm_star_observations",

498 "fgcm_Cycle4_Zeropoints",

499 "fgcm_standard_star",

500 "forcedSourceOnDiaObjectTable",

501 "forcedSourceOnDiaObjectTable_tract",

502 "forcedSourceTable",

503 "forcedSourceTable_tract",

504 "goodSeeingCoadd",

505 "goodSeeingCoadd_nImage",

506 "goodSeeingDiff_assocDiaSrcTable",

507 "goodSeeingDiff_diaObjTable",

508 "goodSeeingDiff_diaSrc",

509 "goodSeeingDiff_diaSrcTable",

510 "goodSeeingDiff_differenceTempExp",

511 "goodSeeingDiff_fullDiaObjTable",

512 "goodSeeingDiff_matchedExp",

513 "goodSeeingDiff_templateExp",

514 "goodSeeingVisits",

515 "mergedForcedSource",

516 "mergedForcedSourceOnDiaObject",

517 "transmission_atmosphere_fgcm",

518 },

519 )

520 tester.run(butler, self)

521

522 def test_hsc_drp_rc2_subset(self):

523 butler = self.makeButler(writeable=True)

524 tester = PipelineStepTester(

525 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

526 [

527 "#nightlyStep1",

528 "#nightlyStep2a",

529 "#nightlyStep2b",

530 "#nightlyStep2c",

531 "#nightlyStep2d",

532 "#nightlyStep3",

533 "#nightlyStep4",

534 ],

535 initial_dataset_types=REFCATS,

536 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

537 expected_outputs=COMMON_OUTPUTS

538 | HSC_OUTPUTS

539 | {

540 "fgcm_Cycle4_AtmosphereParameters",

541 "fgcm_Cycle0_FitParameters",

542 "fgcm_Cycle1_FitParameters",

543 "fgcm_Cycle2_FitParameters",

544 "fgcm_Cycle3_FitParameters",

545 "fgcm_Cycle4_FitParameters",

546 "fgcm_Cycle0_FlaggedStars",

547 "fgcm_Cycle1_FlaggedStars",

548 "fgcm_Cycle2_FlaggedStars",

549 "fgcm_Cycle3_FlaggedStars",

550 "fgcm_Cycle4_FlaggedStars",

551 "fgcm_reference_stars",

552 "fgcm_Cycle4_StandardStars",

553 "fgcm_star_ids",

554 "fgcm_star_observations",

555 "fgcm_Cycle4_Zeropoints",

556 "transmission_atmosphere_fgcm",

557 "fgcm_standard_star",

558 },

559 )

560 tester.run(butler, self)

561

562 def test_latiss_drp(self):

563 butler = self.makeButler(writeable=True)

564 tester = PipelineStepTester(

565 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

566 [

567 "#step1",

568 "#step2a",

569 "#step2bcde",

570 "#step3a",

571 "#step3b",

572 "#step3c",

573 "#step4",

574 "#step5",

575 "#step6",

576 ],

577 initial_dataset_types=REFCATS,

578 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

579 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

580 )

581 tester.run(butler, self)

582

583 def test_lsstcam_quickLook(self):

584 butler = self.makeButler(writeable=True)

585 tester = PipelineStepTester(

586 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

587 [

588 "#step1a-single-visit-detectors",

589 "#step1b-single-visit-visits",

590 "#step1d-single-visit-global",

591 ],

592 initial_dataset_types=REFCATS,

593 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

594 expected_outputs=QUICKLOOK_V2_OUTPUTS,

595 )

596 tester.run(butler, self)

597

598 def test_lsstcam_quickLook_bias(self):

599 butler = self.makeButler(writeable=True)

600 tester = PipelineStepTester(

601 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

602 [

603 "#verifyBiasIsr",

604 ],

605 initial_dataset_types=[],

606 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

607 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

608 )

609 tester.run(butler, self)

610

611 def test_lsstcam_quickLook_dark(self):

612 butler = self.makeButler(writeable=True)

613 tester = PipelineStepTester(

614 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

615 [

616 "#verifyDarkIsr",

617 ],

618 initial_dataset_types=[],

619 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

620 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

621 )

622 tester.run(butler, self)

623

624 def test_lsstcam_quickLook_flat(self):

625 butler = self.makeButler(writeable=True)

626 tester = PipelineStepTester(

627 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

628 [

629 "#verifyFlatIsr",

630 ],

631 initial_dataset_types=[],

632 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

633 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

634 )

635 tester.run(butler, self)

636

637 def test_lsstcam_nightly_validation(self):

638 butler = self.makeButler(writeable=True)

639 tester = PipelineStepTester(

640 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

641 [

642 "#step1a-single-visit-detectors",

643 "#step1b-single-visit-visits",

644 "#step1c-single-visit-tracts",

645 "#step1d-single-visit-global",

646 "#stage3-coadd"

647

648 ],

649 initial_dataset_types=REFCATS,

650 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

651 # Check for some basic stage 1 outputs

652 expected_outputs={"preliminary_visit_image",

653 "preliminary_visit_summary",

654 "preliminary_visit_table",

655 "preliminary_visit_detector_table",

656 "single_visit_star_association_metrics",

657 "deep_coadd"}

658 )

659 tester.run(butler, self)

660

661 def test_lsstcam_drp(self):

662 butler = self.makeButler(writeable=True)

663 tester = PipelineStepTester(

664 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

665 [

666 "#stage1-single-visit",

667 "#stage2-recalibrate",

668 "#stage3-coadd",

669 "#stage4-measure-variability",

670 ],

671 initial_dataset_types=REFCATS,

672 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

673 expected_outputs=COMMON_V2_OUTPUTS,

674 )

675 tester.run(butler, self)

676

677 def test_lsstcam_drp_fl(self):

678 butler = self.makeButler(writeable=True)

679 tester = PipelineStepTester(

680 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

681 [

682 "#stage1-single-visit",

683 "#stage2-recalibrate",

684 "#stage3-coadd",

685 "#stage4-measure-variability",

686 ],

687 initial_dataset_types=REFCATS,

688 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

689 # Check for some basic outputs

690 expected_outputs={"preliminary_visit_image",

691 "preliminary_visit_summary",

692 "deep_coadd"}

693 )

694 tester.run(butler, self)

695

696 def test_lsstcam_nv_fl(self):

697 butler = self.makeButler(writeable=True)

698 tester = PipelineStepTester(

699 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

700 [

701 "#stage1-single-visit",

702 "#stage2-recalibrate",

703 "#stage3-coadd",

704 ],

705 initial_dataset_types=REFCATS,

706 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

707 # Check for some basic outputs

708 expected_outputs={"preliminary_visit_image",

709 "preliminary_visit_summary",

710 "deep_coadd_predetection"}

711 )

712 tester.run(butler, self)

713

714 def test_lsstcam_imsim_drp_ci_imsim(self):

715 butler = self.makeButler(writeable=True)

716 tester = PipelineStepTester(

717 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

718 [""],

719 initial_dataset_types=REFCATS,

720 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

721 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

722 )

723 tester.run(butler, self)

724

725 def test_lsstcam_imsim_drp_test_med_1(self):

726 butler = self.makeButler(writeable=True)

727 tester = PipelineStepTester(

728 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

729 [f"#step{N}" for N in range(1, 8)],

730 initial_dataset_types=REFCATS,

731 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

732 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

733 )

734 tester.run(butler, self)

735

736 def test_lsstcam_imsim_drp(self):

737 butler = self.makeButler(writeable=True)

738 tester = PipelineStepTester(

739 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

740 [f"#step{N}" for N in range(1, 8)],

741 initial_dataset_types=REFCATS,

742 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

743 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

744 )

745 tester.run(butler, self)

746

747 def test_lsstcam_imsim_nightly_validation(self):

748 butler = self.makeButler(writeable=True)

749 tester = PipelineStepTester(

750 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

751 [

752 "#step1",

753 "#step2",

754 "#step3",

755 "#step7",

756 ],

757 initial_dataset_types=REFCATS,

758 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

759 expected_outputs=COMMON_OUTPUTS,

760 )

761 tester.run(butler, self)

762

763 def test_lsstcomcamsim_drp(self):

764 butler = self.makeButler(writeable=True)

765 tester = PipelineStepTester(

766 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

767 [

768 "#step1",

769 "#step2a",

770 "#step2b",

771 "#step2c",

772 "#step2d",

773 "#step2e",

774 "#step3",

775 "#step4",

776 "#step5",

777 "#step6",

778 "#step7",

779 ],

780 initial_dataset_types=REFCATS,

781 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

782 expected_outputs=COMMON_OUTPUTS,

783 )

784 tester.run(butler, self)

785

786 def test_lsstcomcamsim_nightly_validation(self):

787 butler = self.makeButler(writeable=True)

788 tester = PipelineStepTester(

789 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

790 [

791 "#step1",

792 "#step2a",

793 "#nightlyRollup",

794 "#step2b",

795 "#step2d",

796 "#step2e",

797 "#step3",

798 "#step7",

799 ],

800 initial_dataset_types=REFCATS,

801 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

802 expected_outputs=COMMON_OUTPUTS,

803 )

804 tester.run(butler, self)

805

806 def test_lsstcomcamsim_quickLook(self):

807 butler = self.makeButler(writeable=True)

808 tester = PipelineStepTester(

809 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

810 [

811 "#step1",

812 "#step2a",

813 "#nightlyRollup",

814 ],

815 initial_dataset_types=REFCATS,

816 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

817 expected_outputs=QUICKLOOK_OUTPUTS,

818 )

819 tester.run(butler, self)

820

821 def test_lsstcomcam_drp(self):

822 butler = self.makeButler(writeable=True)

823 tester = PipelineStepTester(

824 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

825 [

826 "#step1",

827 "#step2a",

828 "#step2b",

829 "#step2c",

830 "#step2d",

831 "#step2e",

832 "#step3a",

833 "#step3b",

834 "#step4",

835 "#step5",

836 "#step6",

837 "#step7",

838 ],

839 initial_dataset_types=REFCATS,

840 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

841 expected_outputs=COMMON_OUTPUTS,

842 )

843 tester.run(butler, self)

844

845 def test_lsstcomcam_nightly_validation(self):

846 butler = self.makeButler(writeable=True)

847 tester = PipelineStepTester(

848 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

849 [

850 "#step1",

851 "#step2a",

852 "#nightlyRollup",

853 "#step2b",

854 "#step2d",

855 "#step2e",

856 "#step3a",

857 "#step3b",

858 "#step7",

859 ],

860 initial_dataset_types=REFCATS,

861 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

862 expected_outputs=COMMON_OUTPUTS,

863 )

864 tester.run(butler, self)

865

866 def test_lsstcomcam_quickLook(self):

867 butler = self.makeButler(writeable=True)

868 tester = PipelineStepTester(

869 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

870 [

871 "#step1",

872 "#step2a",

873 "#nightlyRollup",

874 ],

875 initial_dataset_types=REFCATS,

876 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

877 expected_outputs=QUICKLOOK_OUTPUTS,

878 )

879 tester.run(butler, self)

880

881

882if __name__ == "__main__": 882 ↛ 883line 882 didn't jump to line 883 because the condition on line 882 was never true

883 unittest.main()