Coverage for tests/test_pipelines.py: 40%

1# This file is part of drp_pipe.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <http://www.gnu.org/licenses/>.

22"""Unit tests for tracking edits to drp_pipe pipelines.

23"""

25import os

26import tempfile

27import unittest

28from typing import Any

30from lsst.daf.butler import Butler, Config

31from lsst.daf.butler.tests import DatastoreMock

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines")

36TEST_DIR = os.path.abspath(os.path.dirname(__file__))

38# mostly common inputs

39COMMON_INPUTS = {

40 "bfKernel",

41 "bias",

42 "camera",

43 "crosstalk",

44 "dark",

45 "flat",

46 "fringe",

47 "isrOverscanCorrected",

48 "raw",

49 "skyMap",

50 # New ISR task inputs (LATISS, LSSTComCam, LSSTCam)

51 "ptc",

52 "linearizer",

53 "bfk",

54 "pretrainedModelPackage",

55 # Optional preloaded solar system objects.

56 # Generated by generateEphemerides in base-v2 but not required in general.

57 "preloaded_ss_object_visit",

58 # Solar system state required to generateEphemerides

59 "mpcorb",

60 "de440s",

61 "sb441_n16",

62 "obscodes",

63 "linux_p1550p2650",

64 "pck00010",

65 "earth_latest_high_prec",

66 "earth_620120_250826",

67 "earth_2025_250826_2125_predict",

68 "naif0012"

69}

71# HSC common inputs, in addition to COMMON_INPUTS

72HSC_INPUTS = {

73 "brightObjectMask",

74 "brighterFatterKernel",

75 "defects",

76 "gaia_dr2_20200414",

77 "gaia_dr3_20230707",

78 "linearizer",

79 "ps1_pv3_3pi_20170110",

80 "sky",

81 "transmission_atmosphere",

82 "transmission_filter",

83 "transmission_optics",

84 "transmission_sensor",

85 "yBackground",

86}

88# LATISS common inputs, in addition to COMMON_INPUTS

89LATISS_INPUTS = {

90 "atlas_refcat2_20220201",

91 "defects",

92 "gaia_dr3_20230707",

93 "sky",

94 "transmission_atmosphere_fgcm",

95}

97# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS

98LSSTCAM_IMSIM_INPUTS = {

99 "bfk",

100 "cal_ref_cat_2_2",

101 "truth_summary",

102 "cosmodc2_1_1_4_redmapper_v0_8_1_redgals"

103}

104

105# LSSTCam common inputs, in addition to COMMON_INPUTS

106LSSTCAM_INPUTS = {

107 "bfk",

108 "linearizer",

109 "defects",

110 "cti",

111 "the_monster_20250219",

112 "gaia_dr3_20230707",

113 "sky",

114 "astrometry_camera",

115}

116

117# LSSTComCam common inputs, in addition to COMMON_INPUTS

118LSSTCOMCAM_INPUTS = {

119 "bfk",

120 "ptc",

121 "linearizer",

122 "defects",

123 "cti",

124 "the_monster_20250219",

125 "illuminationCorrection",

126}

127

128# LSSTComCamSim common inputs, in addition to COMMON_INPUTS

129LSSTCOMCAMSIM_INPUTS = {

130 "uw_stars_20240524",

131 "ptc",

132 "bfk",

133}

134

135# a selection of mostly common outputs

136COMMON_OUTPUTS = {

137 "ccdVisitTable",

138 "compare_warp_artifact_mask",

139 "deepCoaddVisits",

140 "deepCoadd",

141 "deepCoadd_calexp",

142 "deepCoadd_calexp_background",

143 "deepCoadd_det",

144 "deepCoadd_directWarp",

145 "deepCoadd_directWarp_maskedFraction",

146 "deepCoadd_forced_src",

147 "deepCoadd_inputMap",

148 "deepCoadd_meas",

149 "deepCoadd_mergeDet",

150 "deepCoadd_nImage",

151 "deepCoadd_obj",

152 "deepCoadd_psfMatchedWarp",

153 "deepCoadd_ref",

154 "deepCoadd_scarletModelData",

155 "finalized_src_table",

156 "finalVisitSummary",

157 "initial_astrometry_match_detector",

158 "initial_photometry_match_detector",

159 "initial_psf_stars_footprints_detector",

160 "initial_psf_stars_detector",

161 "calexp",

162 "calexpBackground",

163 "src",

164 "objectTable",

165 "objectTable_tract",

166 "postISRCCD",

167 "sourceTable_visit",

168 "visitSummary",

169 "visitTable",

170 "isolated_star_presource_associations",

171 "isolated_star_presources",

172}

173

174# Translated COMMON_OUTPUTS

175COMMON_V2_OUTPUTS = {

176 'visit_detector_table',

177 'compare_warp_artifact_mask',

178 'deep_coadd_visit_selection',

179 'deep_coadd_predetection',

180 'deep_coadd',

181 'deep_coadd_background',

182 'object_detection',

183 'direct_warp',

184 'direct_warp_masked_fraction',

185 'object_forced_measurement',

186 'deep_coadd_input_map',

187 'object_unforced_measurement',

188 'object_detection_merged',

189 'deep_coadd_n_image',

190 'object_unstandardized',

191 'psf_matched_warp',

192 'object_ref_measurement',

193 'object_scarlet_models',

194 'refit_psf_star',

195 'visit_summary',

196 'initial_astrometry_match_detector',

197 'initial_photometry_match_detector',

198 'single_visit_psf_star_footprints',

199 'single_visit_psf_star',

200 'preliminary_visit_image',

201 'preliminary_visit_image_background',

202 'single_visit_star_footprints',

203 'object_patch',

204 'object_all',

205 'post_isr_image',

206 'recalibrated_star',

207 'preliminary_visit_summary',

208 'visit_table',

209 'isolated_star_association',

210 'isolated_star'

211}

212

213# HSC common outputs, in addition to COMMON_OUTPUTS

214HSC_OUTPUTS = {

215 "calexp_skyCorr_visit_mosaic",

216 "calexpBackground_skyCorr_visit_mosaic",

217 "preSourceTable",

218 "preSourceTable_visit",

219 "skyCorr",

220}

221

222# LATISS common outputs, in addition to COMMON_OUTPUTS

223LATISS_OUTPUTS = {

224 "diaObjectTable_tract",

225 "diaSourceTable",

226 "diaSourceTable_tract",

227 "fgcm_Cycle0_FitParameters",

228 "fgcm_Cycle1_FitParameters",

229 "fgcm_Cycle2_FitParameters",

230 "fgcm_Cycle3_FitParameters",

231 "fgcm_Cycle4_FitParameters",

232 "fgcm_Cycle0_FlaggedStars",

233 "fgcm_Cycle1_FlaggedStars",

234 "fgcm_Cycle2_FlaggedStars",

235 "fgcm_Cycle3_FlaggedStars",

236 "fgcm_Cycle4_FlaggedStars",

237 "fgcm_reference_stars",

238 "fgcm_star_ids",

239 "fgcm_star_observations",

240 "fgcm_standard_star",

241 "forcedSourceOnDiaObjectTable",

242 "forcedSourceOnDiaObjectTable_tract",

243 "forcedSourceTable",

244 "forcedSourceTable_tract",

245 "goodSeeingCoadd",

246 "goodSeeingCoadd_nImage",

247 "goodSeeingDiff_assocDiaSrcTable",

248 "goodSeeingDiff_diaObjTable",

249 "goodSeeingDiff_diaSrc",

250 "goodSeeingDiff_diaSrcTable",

251 "goodSeeingDiff_differenceExp",

252 "goodSeeingDiff_differenceTempExp",

253 "goodSeeingDiff_fullDiaObjTable",

254 "goodSeeingDiff_matchedExp",

255 "goodSeeingDiff_templateExp",

256 "goodSeeingVisits",

257 "mergedForcedSource",

258 "mergedForcedSourceOnDiaObject",

259 "transmission_atmosphere_fgcm",

260}

261

262# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS

263LSSTCAM_IMSIM_OUTPUTS = {

264 "diaObjectTable_tract",

265 "diaSourceTable",

266 "diaSourceTable_tract",

267 "forcedSourceOnDiaObjectTable",

268 "forcedSourceOnDiaObjectTable_tract",

269 "forcedSourceTable",

270 "forcedSourceTable_tract",

271 "goodSeeingCoadd",

272 "goodSeeingCoadd_nImage",

273 "goodSeeingDiff_assocDiaSrcTable",

274 "goodSeeingDiff_diaObjTable",

275 "goodSeeingDiff_diaSrc",

276 "goodSeeingDiff_diaSrcTable",

277 "goodSeeingDiff_differenceExp",

278 "goodSeeingDiff_differenceTempExp",

279 "goodSeeingDiff_fullDiaObjTable",

280 "goodSeeingDiff_matchedExp",

281 "goodSeeingDiff_templateExp",

282 "goodSeeingVisits",

283 "mergedForcedSource",

284 "mergedForcedSourceOnDiaObject",

285}

286

287# v2 outputs common to all "quickLook" pipelines, which only include

288QUICKLOOK_V2_OUTPUTS = {

289 "preliminary_visit_image", "preliminary_visit_summary"

290}

291

292# Outputs common to all "quickLook" pipelines, which only include

293QUICKLOOK_OUTPUTS = {

294 "calexp", "visitSummary"

295}

296

297QUICKLOOK_BIAS_OUTPUTS = {

298 "post_isr_image",

299}

300

301QUICKLOOK_DARK_OUTPUTS = {

302 "post_isr_image",

303}

304

305QUICKLOOK_FLAT_OUTPUTS = {

306 "post_isr_image",

307}

308

309# All refcats used by any pipelines, for inclusion as the initial_dataset_types

310# argument to PipelineStepTester; the intent is just to allow the 'skypix'

311# dimension placeholder used in these connections to be resolved.

312REFCATS = [

313 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),

314 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),

315 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),

316 ("the_monster_20250219", {"htm7"}, "SimpleCatalog", False),

317 ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),

318 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False),

319 ("uw_stars_20240524", {"htm7"}, "SimpleCatalog", False),

320]

321

322

323class PipelineTestCase(unittest.TestCase):

324 def setUp(self):

325 self.root = makeTestTempDir(TEST_DIR)

326 self.maxDiff = None

327

328 def tearDown(self):

329 removeTestTempDir(self.root)

330

331 def makeButler(self, **kwargs: Any) -> Butler:

332 """Return new Butler instance on each call."""

333 config = Config()

334

335 # make separate temporary directory for registry of this instance

336 tmpdir = tempfile.mkdtemp(dir=self.root)

337 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"

338 config = Butler.makeRepo(self.root, config)

339 butler = Butler.from_config(config, **kwargs)

340 DatastoreMock.apply(butler)

341 return butler

342

343 def test_decam_isrForCrosstalkSources(self):

344 butler = self.makeButler(writeable=True)

345 tester = PipelineStepTester(

346 os.path.join(PIPELINES_DIR, "DECam", "isrForCrosstalkSources.yaml"),

347 ["#step0"],

348 initial_dataset_types=REFCATS,

349 expected_inputs={

350 "camera",

351 "raw",

352 },

353 expected_outputs={"overscanRaw"},

354 )

355 tester.run(butler, self)

356

357 def test_decam_drp_merian(self):

358 butler = self.makeButler(writeable=True)

359 tester = PipelineStepTester(

360 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"),

361 [

362 "#step1",

363 "#step2a",

364 "#step2b",

365 "#step2d",

366 "#step2e",

367 "#step3",

368 ],

369 initial_dataset_types=REFCATS,

370 expected_inputs=COMMON_INPUTS

371 | {

372 "defects",

373 "gaia_dr2_20200414",

374 "gaia_dr3_20230707",

375 "linearizer",

376 "overscanRaw",

377 "ps1_pv3_3pi_20170110",

378 },

379 expected_outputs=COMMON_OUTPUTS

380 | {

381 "goodSeeingCoadd",

382 "goodSeeingCoadd_nImage",

383 "goodSeeingVisits",

384 "preSourceTable",

385 "preSourceTable_visit",

386 },

387 )

388 tester.run(butler, self)

389

390 def test_hsc_drp_ci_hsc(self):

391 butler = self.makeButler(writeable=True)

392 tester = PipelineStepTester(

393 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"),

394 [""],

395 initial_dataset_types=REFCATS,

396 expected_inputs=COMMON_INPUTS

397 | HSC_INPUTS

398 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"},

399 expected_outputs=COMMON_OUTPUTS

400 | HSC_OUTPUTS

401 | {

402 "diaObjectTable_tract",

403 "diaSourceTable",

404 "diaSourceTable_tract",

405 "forcedSourceOnDiaObjectTable",

406 "forcedSourceOnDiaObjectTable_tract",

407 "forcedSourceTable",

408 "forcedSourceTable_tract",

409 "goodSeeingCoadd",

410 "goodSeeingCoadd_nImage",

411 "goodSeeingDiff_assocDiaSrcTable",

412 "goodSeeingDiff_diaObjTable",

413 "goodSeeingDiff_diaSrc",

414 "goodSeeingDiff_diaSrcTable",

415 "goodSeeingDiff_differenceTempExp",

416 "goodSeeingDiff_fullDiaObjTable",

417 "goodSeeingDiff_matchedExp",

418 "goodSeeingDiff_templateExp",

419 "goodSeeingVisits",

420 "mergedForcedSource",

421 "mergedForcedSourceOnDiaObject",

422 },

423 )

424 tester.run(butler, self)

425

426 def test_hsc_drp_prod(self):

427 butler = self.makeButler(writeable=True)

428 tester = PipelineStepTester(

429 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"),

430 [

431 "#step1",

432 "#step2a",

433 "#step2b",

434 "#step2c",

435 "#step2d",

436 "#step2e",

437 "#step3",

438 "#step4",

439 "#step7",

440 ],

441 initial_dataset_types=REFCATS,

442 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

443 expected_outputs=COMMON_OUTPUTS

444 | HSC_OUTPUTS

445 | {

446 "fgcm_Cycle6_AtmosphereParameters",

447 "fgcm_Cycle0_FitParameters",

448 "fgcm_Cycle1_FitParameters",

449 "fgcm_Cycle2_FitParameters",

450 "fgcm_Cycle3_FitParameters",

451 "fgcm_Cycle4_FitParameters",

452 "fgcm_Cycle5_FitParameters",

453 "fgcm_Cycle6_FitParameters",

454 "fgcm_Cycle0_FlaggedStars",

455 "fgcm_Cycle1_FlaggedStars",

456 "fgcm_Cycle2_FlaggedStars",

457 "fgcm_Cycle3_FlaggedStars",

458 "fgcm_Cycle4_FlaggedStars",

459 "fgcm_Cycle5_FlaggedStars",

460 "fgcm_Cycle6_FlaggedStars",

461 "fgcm_reference_stars",

462 "fgcm_Cycle6_StandardStars",

463 "fgcm_star_ids",

464 "fgcm_star_observations",

465 "fgcm_Cycle6_Zeropoints",

466 "transmission_atmosphere_fgcm",

467 "fgcm_standard_star",

468 },

469 )

470 tester.run(butler, self)

471

472 def test_hsc_drp_rc2(self):

473 butler = self.makeButler(writeable=True)

474 tester = PipelineStepTester(

475 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"),

476 [

477 "#step1",

478 "#step2a",

479 "#step2b",

480 "#step2cde",

481 "#step3",

482 "#step4",

483 "#step5",

484 "#step6",

485 "#step7",

486 ],

487 initial_dataset_types=REFCATS,

488 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

489 expected_outputs=COMMON_OUTPUTS

490 | HSC_OUTPUTS

491 | {

492 "diaObjectTable_tract",

493 "diaSourceTable",

494 "diaSourceTable_tract",

495 "fgcm_Cycle4_AtmosphereParameters",

496 "fgcm_Cycle0_FitParameters",

497 "fgcm_Cycle1_FitParameters",

498 "fgcm_Cycle2_FitParameters",

499 "fgcm_Cycle3_FitParameters",

500 "fgcm_Cycle4_FitParameters",

501 "fgcm_Cycle0_FlaggedStars",

502 "fgcm_Cycle1_FlaggedStars",

503 "fgcm_Cycle2_FlaggedStars",

504 "fgcm_Cycle3_FlaggedStars",

505 "fgcm_Cycle4_FlaggedStars",

506 "fgcm_reference_stars",

507 "fgcm_Cycle4_StandardStars",

508 "fgcm_star_ids",

509 "fgcm_star_observations",

510 "fgcm_Cycle4_Zeropoints",

511 "fgcm_standard_star",

512 "forcedSourceOnDiaObjectTable",

513 "forcedSourceOnDiaObjectTable_tract",

514 "forcedSourceTable",

515 "forcedSourceTable_tract",

516 "goodSeeingCoadd",

517 "goodSeeingCoadd_nImage",

518 "goodSeeingDiff_assocDiaSrcTable",

519 "goodSeeingDiff_diaObjTable",

520 "goodSeeingDiff_diaSrc",

521 "goodSeeingDiff_diaSrcTable",

522 "goodSeeingDiff_differenceTempExp",

523 "goodSeeingDiff_fullDiaObjTable",

524 "goodSeeingDiff_matchedExp",

525 "goodSeeingDiff_templateExp",

526 "goodSeeingVisits",

527 "mergedForcedSource",

528 "mergedForcedSourceOnDiaObject",

529 "transmission_atmosphere_fgcm",

530 },

531 )

532 tester.run(butler, self)

533

534 def test_hsc_drp_rc2_subset(self):

535 butler = self.makeButler(writeable=True)

536 tester = PipelineStepTester(

537 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"),

538 [

539 "#nightlyStep1",

540 "#nightlyStep2a",

541 "#nightlyStep2b",

542 "#nightlyStep2c",

543 "#nightlyStep2d",

544 "#nightlyStep3",

545 "#nightlyStep4",

546 ],

547 initial_dataset_types=REFCATS,

548 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"},

549 expected_outputs=COMMON_OUTPUTS

550 | HSC_OUTPUTS

551 | {

552 "fgcm_Cycle4_AtmosphereParameters",

553 "fgcm_Cycle0_FitParameters",

554 "fgcm_Cycle1_FitParameters",

555 "fgcm_Cycle2_FitParameters",

556 "fgcm_Cycle3_FitParameters",

557 "fgcm_Cycle4_FitParameters",

558 "fgcm_Cycle0_FlaggedStars",

559 "fgcm_Cycle1_FlaggedStars",

560 "fgcm_Cycle2_FlaggedStars",

561 "fgcm_Cycle3_FlaggedStars",

562 "fgcm_Cycle4_FlaggedStars",

563 "fgcm_reference_stars",

564 "fgcm_Cycle4_StandardStars",

565 "fgcm_star_ids",

566 "fgcm_star_observations",

567 "fgcm_Cycle4_Zeropoints",

568 "transmission_atmosphere_fgcm",

569 "fgcm_standard_star",

570 },

571 )

572 tester.run(butler, self)

573

574 def test_latiss_drp(self):

575 butler = self.makeButler(writeable=True)

576 tester = PipelineStepTester(

577 os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),

578 [

579 "#step1",

580 "#step2a",

581 "#step2bcde",

582 "#step3a",

583 "#step3b",

584 "#step3c",

585 "#step4",

586 "#step5",

587 "#step6",

588 ],

589 initial_dataset_types=REFCATS,

590 expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},

591 expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,

592 )

593 tester.run(butler, self)

594

595 def test_lsstcam_quickLook(self):

596 butler = self.makeButler(writeable=True)

597 tester = PipelineStepTester(

598 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLook.yaml"),

599 [

600 "#step1a-single-visit-detectors",

601 "#step1b-single-visit-visits",

602 "#step1d-single-visit-global",

603 ],

604 initial_dataset_types=REFCATS,

605 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

606 expected_outputs=QUICKLOOK_V2_OUTPUTS,

607 )

608 tester.run(butler, self)

609

610 def test_lsstcam_quickLook_bias(self):

611 butler = self.makeButler(writeable=True)

612 tester = PipelineStepTester(

613 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookBias.yaml"),

614 [

615 "#verifyBiasIsr",

616 ],

617 initial_dataset_types=[],

618 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

619 expected_outputs=QUICKLOOK_BIAS_OUTPUTS,

620 )

621 tester.run(butler, self)

622

623 def test_lsstcam_quickLook_dark(self):

624 butler = self.makeButler(writeable=True)

625 tester = PipelineStepTester(

626 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookDark.yaml"),

627 [

628 "#verifyDarkIsr",

629 ],

630 initial_dataset_types=[],

631 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

632 expected_outputs=QUICKLOOK_DARK_OUTPUTS,

633 )

634 tester.run(butler, self)

635

636 def test_lsstcam_quickLook_flat(self):

637 butler = self.makeButler(writeable=True)

638 tester = PipelineStepTester(

639 os.path.join(PIPELINES_DIR, "LSSTCam", "quickLookFlat.yaml"),

640 [

641 "#verifyFlatIsr",

642 ],

643 initial_dataset_types=[],

644 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

645 expected_outputs=QUICKLOOK_FLAT_OUTPUTS,

646 )

647 tester.run(butler, self)

648

649 def test_lsstcam_nightly_validation(self):

650 butler = self.makeButler(writeable=True)

651 tester = PipelineStepTester(

652 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation.yaml"),

653 [

654 "#step1a-single-visit-detectors",

655 "#step1b-single-visit-visits",

656 "#step1c-single-visit-tracts",

657 "#step1d-single-visit-global",

658 "#stage3-coadd"

659

660 ],

661 initial_dataset_types=REFCATS,

662 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS,

663 # Check for some basic stage 1 outputs

664 expected_outputs={"preliminary_visit_image",

665 "preliminary_visit_summary",

666 "preliminary_visit_table",

667 "preliminary_visit_detector_table",

668 "single_visit_star_association_metrics",

669 "deep_coadd"}

670 )

671 tester.run(butler, self)

672

673 def test_lsstcam_drp(self):

674 butler = self.makeButler(writeable=True)

675 tester = PipelineStepTester(

676 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP.yaml"),

677 [

678 "#stage1-single-visit",

679 "#stage2-recalibrate",

680 "#stage3-coadd",

681 "#stage4-measure-variability",

682 ],

683 initial_dataset_types=REFCATS,

684 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

685 expected_outputs=COMMON_V2_OUTPUTS | {"source"}

686 )

687 tester.run(butler, self)

688

689 def test_lsstcam_drp_compat(self):

690 butler = self.makeButler(writeable=True)

691 tester = PipelineStepTester(

692 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-compat.yaml"),

693 [

694 "#stage1-single-visit",

695 "#stage2-recalibrate",

696 "#stage3-coadd",

697 "#stage4-measure-variability",

698 ],

699 initial_dataset_types=REFCATS,

700 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"fgcmLookUpTable"},

701 expected_outputs=COMMON_V2_OUTPUTS | {"source2"},

702 )

703 tester.run(butler, self)

704

705 def test_lsstcam_drp_fl(self):

706 butler = self.makeButler(writeable=True)

707 tester = PipelineStepTester(

708 os.path.join(PIPELINES_DIR, "LSSTCam", "DRP-FL.yaml"),

709 [

710 "#stage1-single-visit",

711 "#stage2-recalibrate",

712 "#stage3-coadd",

713 "#stage4-measure-variability",

714 ],

715 initial_dataset_types=REFCATS,

716 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

717 # Check for some basic outputs

718 expected_outputs={"preliminary_visit_image",

719 "preliminary_visit_summary",

720 "deep_coadd"}

721 )

722 tester.run(butler, self)

723

724 def test_lsstcam_nv_fl(self):

725 butler = self.makeButler(writeable=True)

726 tester = PipelineStepTester(

727 os.path.join(PIPELINES_DIR, "LSSTCam", "nightly-validation-FL.yaml"),

728 [

729 "#stage1-single-visit",

730 "#stage2-recalibrate",

731 "#stage3-coadd",

732 ],

733 initial_dataset_types=REFCATS,

734 expected_inputs=COMMON_INPUTS | LSSTCAM_INPUTS | {"sky_frame_background"},

735 # Check for some basic outputs

736 expected_outputs={"preliminary_visit_image",

737 "preliminary_visit_summary",

738 "deep_coadd_predetection"}

739 )

740 tester.run(butler, self)

741

742 def test_lsstcam_imsim_drp_ci_imsim(self):

743 butler = self.makeButler(writeable=True)

744 tester = PipelineStepTester(

745 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"),

746 [""],

747 initial_dataset_types=REFCATS,

748 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

749 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

750 )

751 tester.run(butler, self)

752

753 def test_lsstcam_imsim_drp_test_med_1(self):

754 butler = self.makeButler(writeable=True)

755 tester = PipelineStepTester(

756 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"),

757 [f"#step{N}" for N in range(1, 8)],

758 initial_dataset_types=REFCATS,

759 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

760 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

761 )

762 tester.run(butler, self)

763

764 def test_lsstcam_imsim_drp(self):

765 butler = self.makeButler(writeable=True)

766 tester = PipelineStepTester(

767 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP.yaml"),

768 [f"#step{N}" for N in range(1, 8)],

769 initial_dataset_types=REFCATS,

770 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

771 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS,

772 )

773 tester.run(butler, self)

774

775 def test_lsstcam_imsim_nightly_validation(self):

776 butler = self.makeButler(writeable=True)

777 tester = PipelineStepTester(

778 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "nightly-validation.yaml"),

779 [

780 "#step1",

781 "#step2",

782 "#step3",

783 "#step7",

784 ],

785 initial_dataset_types=REFCATS,

786 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS,

787 expected_outputs=COMMON_OUTPUTS,

788 )

789 tester.run(butler, self)

790

791 def test_lsstcomcamsim_drp(self):

792 butler = self.makeButler(writeable=True)

793 tester = PipelineStepTester(

794 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "DRP.yaml"),

795 [

796 "#step1",

797 "#step2a",

798 "#step2b",

799 "#step2c",

800 "#step2d",

801 "#step2e",

802 "#step3",

803 "#step4",

804 "#step5",

805 "#step6",

806 "#step7",

807 ],

808 initial_dataset_types=REFCATS,

809 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS | {"fgcmLookUpTable"},

810 expected_outputs=COMMON_OUTPUTS,

811 )

812 tester.run(butler, self)

813

814 def test_lsstcomcamsim_nightly_validation(self):

815 butler = self.makeButler(writeable=True)

816 tester = PipelineStepTester(

817 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "nightly-validation.yaml"),

818 [

819 "#step1",

820 "#step2a",

821 "#nightlyRollup",

822 "#step2b",

823 "#step2d",

824 "#step2e",

825 "#step3",

826 "#step7",

827 ],

828 initial_dataset_types=REFCATS,

829 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

830 expected_outputs=COMMON_OUTPUTS,

831 )

832 tester.run(butler, self)

833

834 def test_lsstcomcamsim_quickLook(self):

835 butler = self.makeButler(writeable=True)

836 tester = PipelineStepTester(

837 os.path.join(PIPELINES_DIR, "LSSTComCamSim", "quickLook.yaml"),

838 [

839 "#step1",

840 "#step2a",

841 "#nightlyRollup",

842 ],

843 initial_dataset_types=REFCATS,

844 expected_inputs=COMMON_INPUTS | LSSTCOMCAMSIM_INPUTS,

845 expected_outputs=QUICKLOOK_OUTPUTS,

846 )

847 tester.run(butler, self)

848

849 def test_lsstcomcam_drp(self):

850 butler = self.makeButler(writeable=True)

851 tester = PipelineStepTester(

852 os.path.join(PIPELINES_DIR, "LSSTComCam", "DRP.yaml"),

853 [

854 "#step1",

855 "#step2a",

856 "#step2b",

857 "#step2c",

858 "#step2d",

859 "#step2e",

860 "#step3a",

861 "#step3b",

862 "#step4",

863 "#step5",

864 "#step6",

865 "#step7",

866 ],

867 initial_dataset_types=REFCATS,

868 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS | {"fgcmLookUpTable"},

869 expected_outputs=COMMON_OUTPUTS,

870 )

871 tester.run(butler, self)

872

873 def test_lsstcomcam_nightly_validation(self):

874 butler = self.makeButler(writeable=True)

875 tester = PipelineStepTester(

876 os.path.join(PIPELINES_DIR, "LSSTComCam", "nightly-validation.yaml"),

877 [

878 "#step1",

879 "#step2a",

880 "#nightlyRollup",

881 "#step2b",

882 "#step2d",

883 "#step2e",

884 "#step3a",

885 "#step3b",

886 "#step7",

887 ],

888 initial_dataset_types=REFCATS,

889 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

890 expected_outputs=COMMON_OUTPUTS,

891 )

892 tester.run(butler, self)

893

894 def test_lsstcomcam_quickLook(self):

895 butler = self.makeButler(writeable=True)

896 tester = PipelineStepTester(

897 os.path.join(PIPELINES_DIR, "LSSTComCam", "quickLook.yaml"),

898 [

899 "#step1",

900 "#step2a",

901 "#nightlyRollup",

902 ],

903 initial_dataset_types=REFCATS,

904 expected_inputs=COMMON_INPUTS | LSSTCOMCAM_INPUTS,

905 expected_outputs=QUICKLOOK_OUTPUTS,

906 )

907 tester.run(butler, self)

908

909

910if __name__ == "__main__": 910 ↛ 911line 910 didn't jump to line 911 because the condition on line 910 was never true

911 unittest.main()