Coverage for tests/test_pipelines.py: 42%

65 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-07 12:12 +0000

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "gaia_dr3_20230707", 

59 "linearizer", 

60 "ps1_pv3_3pi_20170110", 

61 "sky", 

62 "transmission_atmosphere", 

63 "transmission_filter", 

64 "transmission_optics", 

65 "transmission_sensor", 

66 "yBackground", 

67} 

68 

69# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

70LSSTCAM_IMSIM_INPUTS = { 

71 "bfk", 

72 "cal_ref_cat_2_2", 

73 "truth_summary", 

74} 

75 

76# a selection of mostly common outputs 

77COMMON_OUTPUTS = { 

78 "calexp", 

79 "calexpBackground", 

80 "ccdVisitTable", 

81 "deepCoadd", 

82 "deepCoadd_calexp", 

83 "deepCoadd_calexp_background", 

84 "deepCoadd_det", 

85 "deepCoadd_directWarp", 

86 "deepCoadd_forced_src", 

87 "deepCoadd_inputMap", 

88 "deepCoadd_meas", 

89 "deepCoadd_measMatch", 

90 "deepCoadd_measMatchFull", 

91 "deepCoadd_mergeDet", 

92 "deepCoadd_nImage", 

93 "deepCoadd_obj", 

94 "deepCoadd_psfMatchedWarp", 

95 "deepCoadd_ref", 

96 "deepCoadd_scarletModelData", 

97 "finalized_src_table", 

98 "finalVisitSummary", 

99 "icExp", 

100 "icExpBackground", 

101 "icSrc", 

102 "isolated_star_cat", 

103 "isolated_star_sources", 

104 "objectTable", 

105 "objectTable_tract", 

106 "postISRCCD", 

107 "source", 

108 "sourceTable", 

109 "sourceTable_visit", 

110 "src", 

111 "srcMatch", 

112 "visitSummary", 

113 "visitTable", 

114} 

115 

116# HSC common outputs, in addition to COMMON_OUTPUTS 

117HSC_OUTPUTS = { 

118 "calexp_skyCorr_visit_mosaic", 

119 "calexpBackground_skyCorr_visit_mosaic", 

120 "forced_src", 

121 "preSource", 

122 "preSourceTable", 

123 "preSourceTable_visit", 

124 "skyCorr", 

125 "srcMatchFull", 

126} 

127 

128# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

129LSSTCAM_IMSIM_OUTPUTS = { 

130 "diaObjectTable_tract", 

131 "diaSourceTable", 

132 "diaSourceTable_tract", 

133 "forcedSourceOnDiaObjectTable", 

134 "forcedSourceOnDiaObjectTable_tract", 

135 "forcedSourceTable", 

136 "forcedSourceTable_tract", 

137 "forced_diff", 

138 "forced_diff_diaObject", 

139 "forced_src", 

140 "forced_src_diaObject", 

141 "goodSeeingCoadd", 

142 "goodSeeingCoadd_nImage", 

143 "goodSeeingDiff_assocDiaSrcTable", 

144 "goodSeeingDiff_diaObjTable", 

145 "goodSeeingDiff_diaSrc", 

146 "goodSeeingDiff_diaSrcTable", 

147 "goodSeeingDiff_differenceExp", 

148 "goodSeeingDiff_differenceTempExp", 

149 "goodSeeingDiff_fullDiaObjTable", 

150 "goodSeeingDiff_matchedExp", 

151 "goodSeeingDiff_templateExp", 

152 "goodSeeingVisits", 

153 "mergedForcedSource", 

154 "mergedForcedSourceOnDiaObject", 

155} 

156 

157 

158class PipelineTestCase(unittest.TestCase): 

159 def setUp(self): 

160 self.root = makeTestTempDir(TEST_DIR) 

161 self.maxDiff = None 

162 

163 def tearDown(self): 

164 removeTestTempDir(self.root) 

165 

166 def makeButler(self, **kwargs: Any) -> Butler: 

167 """Return new Butler instance on each call.""" 

168 config = ButlerConfig() 

169 

170 # make separate temporary directory for registry of this instance 

171 tmpdir = tempfile.mkdtemp(dir=self.root) 

172 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

173 config["root"] = self.root 

174 

175 # have to make a registry first 

176 registryConfig = RegistryConfig(config.get("registry")) 

177 Registry.createFromConfig(registryConfig) 

178 

179 butler = Butler(config, **kwargs) 

180 DatastoreMock.apply(butler) 

181 return butler 

182 

183 def test_decam_drp_merian(self): 

184 butler = self.makeButler(writeable=True) 

185 tester0 = PipelineStepTester( 

186 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

187 ["#step0"], 

188 [ 

189 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

190 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

191 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

192 ], 

193 expected_inputs={ 

194 "camera", 

195 "raw", 

196 }, 

197 expected_outputs={"overscanRaw"}, 

198 ) 

199 tester0.run(butler, self) 

200 tester = PipelineStepTester( 

201 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

202 [ 

203 "#step1", 

204 "#step2a", 

205 "#step2b", 

206 "#step2d", 

207 "#step2e", 

208 "#step3", 

209 ], 

210 [ 

211 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

212 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

213 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

214 ], 

215 expected_inputs=COMMON_INPUTS 

216 | { 

217 "defects", 

218 "gaia_dr2_20200414", 

219 "gaia_dr3_20230707", 

220 "linearizer", 

221 "overscanRaw", 

222 "ps1_pv3_3pi_20170110", 

223 }, 

224 expected_outputs=COMMON_OUTPUTS 

225 | { 

226 "goodSeeingCoadd", 

227 "goodSeeingCoadd_nImage", 

228 "goodSeeingVisits", 

229 "preSource", 

230 "preSourceTable", 

231 "preSourceTable_visit", 

232 "srcMatchFull", 

233 }, 

234 ) 

235 tester.run(butler, self) 

236 

237 def test_hsc_drp_ci_hsc(self): 

238 butler = self.makeButler(writeable=True) 

239 tester = PipelineStepTester( 

240 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

241 [""], 

242 [ 

243 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

244 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

245 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

246 ], 

247 expected_inputs=COMMON_INPUTS 

248 | HSC_INPUTS 

249 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

250 expected_outputs=COMMON_OUTPUTS 

251 | HSC_OUTPUTS 

252 | { 

253 "diaObjectTable_tract", 

254 "diaSourceTable", 

255 "diaSourceTable_tract", 

256 "forcedSourceOnDiaObjectTable", 

257 "forcedSourceOnDiaObjectTable_tract", 

258 "forcedSourceTable", 

259 "forcedSourceTable_tract", 

260 "forced_diff", 

261 "forced_diff_diaObject", 

262 "forced_src_diaObject", 

263 "goodSeeingCoadd", 

264 "goodSeeingCoadd_nImage", 

265 "goodSeeingDiff_assocDiaSrcTable", 

266 "goodSeeingDiff_diaObjTable", 

267 "goodSeeingDiff_diaSrc", 

268 "goodSeeingDiff_diaSrcTable", 

269 "goodSeeingDiff_differenceExp", 

270 "goodSeeingDiff_differenceTempExp", 

271 "goodSeeingDiff_fullDiaObjTable", 

272 "goodSeeingDiff_matchedExp", 

273 "goodSeeingDiff_templateExp", 

274 "goodSeeingVisits", 

275 "mergedForcedSource", 

276 "mergedForcedSourceOnDiaObject", 

277 "objectTable_tract_astrometryRefCat_match", 

278 "sourceTable_visit_astrometryRefCat_match", 

279 }, 

280 ) 

281 tester.run(butler, self) 

282 

283 def test_hsc_drp_prod(self): 

284 butler = self.makeButler(writeable=True) 

285 tester = PipelineStepTester( 

286 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

287 [ 

288 "#step1", 

289 "#step2a", 

290 "#step2b", 

291 "#step2c", 

292 "#step2d", 

293 "#step2e", 

294 "#step3", 

295 "#step4", 

296 "#step7", 

297 ], 

298 [ 

299 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

300 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

301 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

302 ], 

303 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

304 expected_outputs=COMMON_OUTPUTS 

305 | HSC_OUTPUTS 

306 | { 

307 "fgcmAtmosphereParameters6", 

308 "fgcmFitParameters0", 

309 "fgcmFitParameters1", 

310 "fgcmFitParameters2", 

311 "fgcmFitParameters3", 

312 "fgcmFitParameters4", 

313 "fgcmFitParameters5", 

314 "fgcmFitParameters6", 

315 "fgcmFlaggedStars0", 

316 "fgcmFlaggedStars1", 

317 "fgcmFlaggedStars2", 

318 "fgcmFlaggedStars3", 

319 "fgcmFlaggedStars4", 

320 "fgcmFlaggedStars5", 

321 "fgcmFlaggedStars6", 

322 "fgcm_reference_stars", 

323 "fgcmStandardStars6", 

324 "fgcm_star_ids", 

325 "fgcm_star_observations", 

326 "fgcmZeropoints6", 

327 "transmission_atmosphere_fgcm", 

328 }, 

329 ) 

330 tester.run(butler, self) 

331 

332 def test_hsc_drp_rc2(self): 

333 butler = self.makeButler(writeable=True) 

334 tester = PipelineStepTester( 

335 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

336 [ 

337 "#step1", 

338 "#step2a", 

339 "#step2b", 

340 "#step2cde", 

341 "#step3", 

342 "#step4", 

343 "#step5", 

344 "#step6", 

345 "#step7", 

346 ], 

347 [ 

348 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

349 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

350 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

351 ], 

352 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

353 expected_outputs=COMMON_OUTPUTS 

354 | HSC_OUTPUTS 

355 | { 

356 "diaObjectTable_tract", 

357 "diaSourceTable", 

358 "diaSourceTable_tract", 

359 "fgcmAtmosphereParameters4", 

360 "fgcmFitParameters0", 

361 "fgcmFitParameters1", 

362 "fgcmFitParameters2", 

363 "fgcmFitParameters3", 

364 "fgcmFitParameters4", 

365 "fgcmFlaggedStars0", 

366 "fgcmFlaggedStars1", 

367 "fgcmFlaggedStars2", 

368 "fgcmFlaggedStars3", 

369 "fgcmFlaggedStars4", 

370 "fgcm_reference_stars", 

371 "fgcmStandardStars4", 

372 "fgcm_star_ids", 

373 "fgcm_star_observations", 

374 "fgcmZeropoints4", 

375 "forcedSourceOnDiaObjectTable", 

376 "forcedSourceOnDiaObjectTable_tract", 

377 "forcedSourceTable", 

378 "forcedSourceTable_tract", 

379 "forced_diff", 

380 "forced_diff_diaObject", 

381 "forced_src_diaObject", 

382 "goodSeeingCoadd", 

383 "goodSeeingCoadd_nImage", 

384 "goodSeeingDiff_assocDiaSrcTable", 

385 "goodSeeingDiff_diaObjTable", 

386 "goodSeeingDiff_diaSrc", 

387 "goodSeeingDiff_diaSrcTable", 

388 "goodSeeingDiff_differenceExp", 

389 "goodSeeingDiff_differenceTempExp", 

390 "goodSeeingDiff_fullDiaObjTable", 

391 "goodSeeingDiff_matchedExp", 

392 "goodSeeingDiff_templateExp", 

393 "goodSeeingVisits", 

394 "mergedForcedSource", 

395 "mergedForcedSourceOnDiaObject", 

396 "transmission_atmosphere_fgcm", 

397 }, 

398 ) 

399 tester.run(butler, self) 

400 

401 def test_hsc_drp_rc2_subset(self): 

402 butler = self.makeButler(writeable=True) 

403 tester = PipelineStepTester( 

404 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"), 

405 [ 

406 "#nightlyStep1", 

407 "#nightlyStep2a", 

408 "#nightlyStep2b", 

409 "#nightlyStep2c", 

410 "#nightlyStep2d", 

411 "#nightlyStep3", 

412 "#nightlyStep4", 

413 "#nightlyStep5", 

414 ], 

415 [ 

416 ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False), 

417 ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False), 

418 ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False), 

419 ], 

420 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

421 expected_outputs=COMMON_OUTPUTS 

422 | HSC_OUTPUTS 

423 | { 

424 "fgcmAtmosphereParameters4", 

425 "fgcmFitParameters0", 

426 "fgcmFitParameters1", 

427 "fgcmFitParameters2", 

428 "fgcmFitParameters3", 

429 "fgcmFitParameters4", 

430 "fgcmFlaggedStars0", 

431 "fgcmFlaggedStars1", 

432 "fgcmFlaggedStars2", 

433 "fgcmFlaggedStars3", 

434 "fgcmFlaggedStars4", 

435 "fgcm_reference_stars", 

436 "fgcmStandardStars4", 

437 "fgcm_star_ids", 

438 "fgcm_star_observations", 

439 "fgcmZeropoints4", 

440 "transmission_atmosphere_fgcm", 

441 }, 

442 ) 

443 tester.run(butler, self) 

444 

445 def test_lsstcam_imsim_drp_ci_imsim(self): 

446 butler = self.makeButler(writeable=True) 

447 tester = PipelineStepTester( 

448 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

449 [f"#step{N}" for N in range(1, 9)], 

450 [ 

451 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False), 

452 ], 

453 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

454 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

455 ) 

456 tester.run(butler, self) 

457 

458 def test_lsstcam_imsim_drp_test_med_1(self): 

459 butler = self.makeButler(writeable=True) 

460 tester = PipelineStepTester( 

461 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

462 [f"#step{N}" for N in range(1, 9)], 

463 [ 

464 ("cal_ref_cat_2_2", {"htm7"}, "SimpleCatalog", False), 

465 ], 

466 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

467 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

468 ) 

469 tester.run(butler, self) 

470 

471 

472if __name__ == "__main__": 472 ↛ 473line 472 didn't jump to line 473, because the condition on line 472 was never true

473 unittest.main()