Coverage for tests/test_pipelines.py: 43%

63 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-23 14:00 +0000

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, Config 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "gaia_dr3_20230707", 

59 "linearizer", 

60 "ps1_pv3_3pi_20170110", 

61 "sky", 

62 "transmission_atmosphere", 

63 "transmission_filter", 

64 "transmission_optics", 

65 "transmission_sensor", 

66 "yBackground", 

67} 

68 

69# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

70LSSTCAM_IMSIM_INPUTS = { 

71 "bfk", 

72 "cal_ref_cat_2_2", 

73 "truth_summary", 

74} 

75 

76# a selection of mostly common outputs 

77COMMON_OUTPUTS = { 

78 "calexp", 

79 "calexpBackground", 

80 "ccdVisitTable", 

81 "deepCoaddVisits", 

82 "deepCoadd", 

83 "deepCoadd_calexp", 

84 "deepCoadd_calexp_background", 

85 "deepCoadd_det", 

86 "deepCoadd_directWarp", 

87 "deepCoadd_forced_src", 

88 "deepCoadd_inputMap", 

89 "deepCoadd_meas", 

90 "deepCoadd_measMatch", 

91 "deepCoadd_measMatchFull", 

92 "deepCoadd_mergeDet", 

93 "deepCoadd_nImage", 

94 "deepCoadd_obj", 

95 "deepCoadd_psfMatchedWarp", 

96 "deepCoadd_ref", 

97 "deepCoadd_scarletModelData", 

98 "finalized_src_table", 

99 "finalVisitSummary", 

100 "icExp", 

101 "icExpBackground", 

102 "icSrc", 

103 "isolated_star_cat", 

104 "isolated_star_sources", 

105 "objectTable", 

106 "objectTable_tract", 

107 "postISRCCD", 

108 "source", 

109 "sourceTable", 

110 "sourceTable_visit", 

111 "src", 

112 "srcMatch", 

113 "visitSummary", 

114 "visitTable", 

115} 

116 

117# HSC common outputs, in addition to COMMON_OUTPUTS 

118HSC_OUTPUTS = { 

119 "calexp_skyCorr_visit_mosaic", 

120 "calexpBackground_skyCorr_visit_mosaic", 

121 "forced_src", 

122 "preSource", 

123 "preSourceTable", 

124 "preSourceTable_visit", 

125 "skyCorr", 

126 "srcMatchFull", 

127} 

128 

129# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

130LSSTCAM_IMSIM_OUTPUTS = { 

131 "diaObjectTable_tract", 

132 "diaSourceTable", 

133 "diaSourceTable_tract", 

134 "forcedSourceOnDiaObjectTable", 

135 "forcedSourceOnDiaObjectTable_tract", 

136 "forcedSourceTable", 

137 "forcedSourceTable_tract", 

138 "forced_diff", 

139 "forced_diff_diaObject", 

140 "forced_src", 

141 "forced_src_diaObject", 

142 "goodSeeingCoadd", 

143 "goodSeeingCoadd_nImage", 

144 "goodSeeingDiff_assocDiaSrcTable", 

145 "goodSeeingDiff_diaObjTable", 

146 "goodSeeingDiff_diaSrc", 

147 "goodSeeingDiff_diaSrcTable", 

148 "goodSeeingDiff_differenceExp", 

149 "goodSeeingDiff_differenceTempExp", 

150 "goodSeeingDiff_fullDiaObjTable", 

151 "goodSeeingDiff_matchedExp", 

152 "goodSeeingDiff_templateExp", 

153 "goodSeeingVisits", 

154 "mergedForcedSource", 

155 "mergedForcedSourceOnDiaObject", 

156} 

157 

158 

159class PipelineTestCase(unittest.TestCase): 

160 def setUp(self): 

161 self.root = makeTestTempDir(TEST_DIR) 

162 self.maxDiff = None 

163 

164 def tearDown(self): 

165 removeTestTempDir(self.root) 

166 

167 def makeButler(self, **kwargs: Any) -> Butler: 

168 """Return new Butler instance on each call.""" 

169 config = Config() 

170 

171 # make separate temporary directory for registry of this instance 

172 tmpdir = tempfile.mkdtemp(dir=self.root) 

173 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

174 config = Butler.makeRepo(self.root, config) 

175 butler = Butler.from_config(config, **kwargs) 

176 DatastoreMock.apply(butler) 

177 return butler 

178 

179 def test_decam_drp_merian(self): 

180 butler = self.makeButler(writeable=True) 

181 tester0 = PipelineStepTester( 

182 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

183 ["#step0"], 

184 [ 

185 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

186 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

187 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

188 ], 

189 expected_inputs={ 

190 "camera", 

191 "raw", 

192 }, 

193 expected_outputs={"overscanRaw"}, 

194 ) 

195 tester0.run(butler, self) 

196 tester = PipelineStepTester( 

197 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

198 [ 

199 "#step1", 

200 "#step2a", 

201 "#step2b", 

202 "#step2d", 

203 "#step2e", 

204 "#step3", 

205 ], 

206 [ 

207 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

208 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

209 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

210 ], 

211 expected_inputs=COMMON_INPUTS 

212 | { 

213 "defects", 

214 "gaia_dr2_20200414", 

215 "gaia_dr3_20230707", 

216 "linearizer", 

217 "overscanRaw", 

218 "ps1_pv3_3pi_20170110", 

219 }, 

220 expected_outputs=COMMON_OUTPUTS 

221 | { 

222 "goodSeeingCoadd", 

223 "goodSeeingCoadd_nImage", 

224 "goodSeeingVisits", 

225 "preSource", 

226 "preSourceTable", 

227 "preSourceTable_visit", 

228 "srcMatchFull", 

229 }, 

230 ) 

231 tester.run(butler, self) 

232 

233 def test_hsc_drp_ci_hsc(self): 

234 butler = self.makeButler(writeable=True) 

235 tester = PipelineStepTester( 

236 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

237 [""], 

238 [ 

239 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

240 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

241 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

242 ], 

243 expected_inputs=COMMON_INPUTS 

244 | HSC_INPUTS 

245 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

246 expected_outputs=COMMON_OUTPUTS 

247 | HSC_OUTPUTS 

248 | { 

249 "diaObjectTable_tract", 

250 "diaSourceTable", 

251 "diaSourceTable_tract", 

252 "forcedSourceOnDiaObjectTable", 

253 "forcedSourceOnDiaObjectTable_tract", 

254 "forcedSourceTable", 

255 "forcedSourceTable_tract", 

256 "forced_diff", 

257 "forced_diff_diaObject", 

258 "forced_src_diaObject", 

259 "goodSeeingCoadd", 

260 "goodSeeingCoadd_nImage", 

261 "goodSeeingDiff_assocDiaSrcTable", 

262 "goodSeeingDiff_diaObjTable", 

263 "goodSeeingDiff_diaSrc", 

264 "goodSeeingDiff_diaSrcTable", 

265 "goodSeeingDiff_differenceExp", 

266 "goodSeeingDiff_differenceTempExp", 

267 "goodSeeingDiff_fullDiaObjTable", 

268 "goodSeeingDiff_matchedExp", 

269 "goodSeeingDiff_templateExp", 

270 "goodSeeingVisits", 

271 "mergedForcedSource", 

272 "mergedForcedSourceOnDiaObject", 

273 "objectTable_tract_astrometryRefCat_match", 

274 "sourceTable_visit_astrometryRefCat_match", 

275 }, 

276 ) 

277 tester.run(butler, self) 

278 

279 def test_hsc_drp_prod(self): 

280 butler = self.makeButler(writeable=True) 

281 tester = PipelineStepTester( 

282 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

283 [ 

284 "#step1", 

285 "#step2a", 

286 "#step2b", 

287 "#step2c", 

288 "#step2d", 

289 "#step2e", 

290 "#step3", 

291 "#step4", 

292 "#step7", 

293 ], 

294 [ 

295 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

296 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

297 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

298 ], 

299 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

300 expected_outputs=COMMON_OUTPUTS 

301 | HSC_OUTPUTS 

302 | { 

303 "fgcmAtmosphereParameters6", 

304 "fgcmFitParameters0", 

305 "fgcmFitParameters1", 

306 "fgcmFitParameters2", 

307 "fgcmFitParameters3", 

308 "fgcmFitParameters4", 

309 "fgcmFitParameters5", 

310 "fgcmFitParameters6", 

311 "fgcmFlaggedStars0", 

312 "fgcmFlaggedStars1", 

313 "fgcmFlaggedStars2", 

314 "fgcmFlaggedStars3", 

315 "fgcmFlaggedStars4", 

316 "fgcmFlaggedStars5", 

317 "fgcmFlaggedStars6", 

318 "fgcm_reference_stars", 

319 "fgcmStandardStars6", 

320 "fgcm_star_ids", 

321 "fgcm_star_observations", 

322 "fgcmZeropoints6", 

323 "transmission_atmosphere_fgcm", 

324 }, 

325 ) 

326 tester.run(butler, self) 

327 

328 def test_hsc_drp_rc2(self): 

329 butler = self.makeButler(writeable=True) 

330 tester = PipelineStepTester( 

331 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

332 [ 

333 "#step1", 

334 "#step2a", 

335 "#step2b", 

336 "#step2cde", 

337 "#step3", 

338 "#step4", 

339 "#step5", 

340 "#step6", 

341 "#step7", 

342 ], 

343 [ 

344 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

345 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

346 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

347 ], 

348 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

349 expected_outputs=COMMON_OUTPUTS 

350 | HSC_OUTPUTS 

351 | { 

352 "diaObjectTable_tract", 

353 "diaSourceTable", 

354 "diaSourceTable_tract", 

355 "fgcmAtmosphereParameters4", 

356 "fgcmFitParameters0", 

357 "fgcmFitParameters1", 

358 "fgcmFitParameters2", 

359 "fgcmFitParameters3", 

360 "fgcmFitParameters4", 

361 "fgcmFlaggedStars0", 

362 "fgcmFlaggedStars1", 

363 "fgcmFlaggedStars2", 

364 "fgcmFlaggedStars3", 

365 "fgcmFlaggedStars4", 

366 "fgcm_reference_stars", 

367 "fgcmStandardStars4", 

368 "fgcm_star_ids", 

369 "fgcm_star_observations", 

370 "fgcmZeropoints4", 

371 "forcedSourceOnDiaObjectTable", 

372 "forcedSourceOnDiaObjectTable_tract", 

373 "forcedSourceTable", 

374 "forcedSourceTable_tract", 

375 "forced_diff", 

376 "forced_diff_diaObject", 

377 "forced_src_diaObject", 

378 "goodSeeingCoadd", 

379 "goodSeeingCoadd_nImage", 

380 "goodSeeingDiff_assocDiaSrcTable", 

381 "goodSeeingDiff_diaObjTable", 

382 "goodSeeingDiff_diaSrc", 

383 "goodSeeingDiff_diaSrcTable", 

384 "goodSeeingDiff_differenceExp", 

385 "goodSeeingDiff_differenceTempExp", 

386 "goodSeeingDiff_fullDiaObjTable", 

387 "goodSeeingDiff_matchedExp", 

388 "goodSeeingDiff_templateExp", 

389 "goodSeeingVisits", 

390 "mergedForcedSource", 

391 "mergedForcedSourceOnDiaObject", 

392 "transmission_atmosphere_fgcm", 

393 }, 

394 ) 

395 tester.run(butler, self) 

396 

397 def test_hsc_drp_rc2_subset(self): 

398 butler = self.makeButler(writeable=True) 

399 tester = PipelineStepTester( 

400 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"), 

401 [ 

402 "#nightlyStep1", 

403 "#nightlyStep2a", 

404 "#nightlyStep2b", 

405 "#nightlyStep2c", 

406 "#nightlyStep2d", 

407 "#nightlyStep3", 

408 "#nightlyStep4", 

409 "#nightlyStep5", 

410 ], 

411 [ 

412 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

413 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

414 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

415 ], 

416 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

417 expected_outputs=COMMON_OUTPUTS 

418 | HSC_OUTPUTS 

419 | { 

420 "fgcmAtmosphereParameters4", 

421 "fgcmFitParameters0", 

422 "fgcmFitParameters1", 

423 "fgcmFitParameters2", 

424 "fgcmFitParameters3", 

425 "fgcmFitParameters4", 

426 "fgcmFlaggedStars0", 

427 "fgcmFlaggedStars1", 

428 "fgcmFlaggedStars2", 

429 "fgcmFlaggedStars3", 

430 "fgcmFlaggedStars4", 

431 "fgcm_reference_stars", 

432 "fgcmStandardStars4", 

433 "fgcm_star_ids", 

434 "fgcm_star_observations", 

435 "fgcmZeropoints4", 

436 "transmission_atmosphere_fgcm", 

437 }, 

438 ) 

439 tester.run(butler, self) 

440 

441 def test_lsstcam_imsim_drp_ci_imsim(self): 

442 butler = self.makeButler(writeable=True) 

443 tester = PipelineStepTester( 

444 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

445 [f"#step{N}" for N in range(1, 9)], 

446 [ 

447 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False), 

448 ], 

449 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

450 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

451 ) 

452 tester.run(butler, self) 

453 

454 def test_lsstcam_imsim_drp_test_med_1(self): 

455 butler = self.makeButler(writeable=True) 

456 tester = PipelineStepTester( 

457 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

458 [f"#step{N}" for N in range(1, 9)], 

459 [ 

460 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False), 

461 ], 

462 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

463 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

464 ) 

465 tester.run(butler, self) 

466 

467 

468if __name__ == "__main__": 468 ↛ 469line 468 didn't jump to line 469, because the condition on line 468 was never true

469 unittest.main()