Coverage for tests/test_pipelines.py: 42%

65 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-16 03:19 -0700

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "linearizer", 

59 "ps1_pv3_3pi_20170110", 

60 "sky", 

61 "transmission_atmosphere", 

62 "transmission_filter", 

63 "transmission_optics", 

64 "transmission_sensor", 

65 "yBackground", 

66} 

67 

68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

69LSSTCAM_IMSIM_INPUTS = { 

70 "bfk", 

71 "cal_ref_cat_2_2", 

72 "truth_summary", 

73} 

74 

75# a selection of mostly common outputs 

76COMMON_OUTPUTS = { 

77 "calexp", 

78 "calexpBackground", 

79 "ccdVisitTable", 

80 "deepCoadd", 

81 "deepCoadd_calexp", 

82 "deepCoadd_calexp_background", 

83 "deepCoadd_det", 

84 "deepCoadd_directWarp", 

85 "deepCoadd_forced_src", 

86 "deepCoadd_inputMap", 

87 "deepCoadd_meas", 

88 "deepCoadd_measMatch", 

89 "deepCoadd_measMatchFull", 

90 "deepCoadd_mergeDet", 

91 "deepCoadd_nImage", 

92 "deepCoadd_obj", 

93 "deepCoadd_psfMatchedWarp", 

94 "deepCoadd_ref", 

95 "deepCoadd_scarletModelData", 

96 "finalized_src_table", 

97 "finalVisitSummary", 

98 "icExp", 

99 "icExpBackground", 

100 "icSrc", 

101 "isolated_star_cat", 

102 "isolated_star_sources", 

103 "objectTable", 

104 "objectTable_tract", 

105 "postISRCCD", 

106 "source", 

107 "sourceTable", 

108 "sourceTable_visit", 

109 "src", 

110 "srcMatch", 

111 "visitSummary", 

112 "visitTable", 

113} 

114 

115# HSC common outputs, in addition to COMMON_OUTPUTS 

116HSC_OUTPUTS = { 

117 "calexp_skyCorr_visit_mosaic", 

118 "calexpBackground_skyCorr_visit_mosaic", 

119 "forced_src", 

120 "preSource", 

121 "preSourceTable", 

122 "preSourceTable_visit", 

123 "skyCorr", 

124 "srcMatchFull", 

125} 

126 

127# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

128LSSTCAM_IMSIM_OUTPUTS = { 

129 "diaObjectTable_tract", 

130 "diaSourceTable", 

131 "diaSourceTable_tract", 

132 "forcedSourceOnDiaObjectTable", 

133 "forcedSourceOnDiaObjectTable_tract", 

134 "forcedSourceTable", 

135 "forcedSourceTable_tract", 

136 "forced_diff", 

137 "forced_diff_diaObject", 

138 "forced_src", 

139 "forced_src_diaObject", 

140 "goodSeeingCoadd", 

141 "goodSeeingCoadd_nImage", 

142 "goodSeeingDiff_assocDiaSrcTable", 

143 "goodSeeingDiff_diaObjTable", 

144 "goodSeeingDiff_diaSrc", 

145 "goodSeeingDiff_diaSrcTable", 

146 "goodSeeingDiff_differenceExp", 

147 "goodSeeingDiff_differenceTempExp", 

148 "goodSeeingDiff_fullDiaObjTable", 

149 "goodSeeingDiff_matchedExp", 

150 "goodSeeingDiff_templateExp", 

151 "goodSeeingVisits", 

152 "mergedForcedSource", 

153 "mergedForcedSourceOnDiaObject", 

154} 

155 

156 

157class PipelineTestCase(unittest.TestCase): 

158 def setUp(self): 

159 self.root = makeTestTempDir(TEST_DIR) 

160 self.maxDiff = None 

161 

162 def tearDown(self): 

163 removeTestTempDir(self.root) 

164 

165 def makeButler(self, **kwargs: Any) -> Butler: 

166 """Return new Butler instance on each call.""" 

167 config = ButlerConfig() 

168 

169 # make separate temporary directory for registry of this instance 

170 tmpdir = tempfile.mkdtemp(dir=self.root) 

171 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

172 config["root"] = self.root 

173 

174 # have to make a registry first 

175 registryConfig = RegistryConfig(config.get("registry")) 

176 Registry.createFromConfig(registryConfig) 

177 

178 butler = Butler(config, **kwargs) 

179 DatastoreMock.apply(butler) 

180 return butler 

181 

182 def test_decam_drp_merian(self): 

183 butler = self.makeButler(writeable=True) 

184 tester0 = PipelineStepTester( 

185 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

186 ["#step0"], 

187 [ 

188 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

189 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

190 ], 

191 expected_inputs={ 

192 "camera", 

193 "raw", 

194 }, 

195 expected_outputs={"overscanRaw"}, 

196 ) 

197 tester0.run(butler, self) 

198 tester = PipelineStepTester( 

199 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

200 [ 

201 "#step1", 

202 "#step2a", 

203 "#step2b", 

204 "#step2d", 

205 "#step2e", 

206 "#step3", 

207 ], 

208 [ 

209 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

210 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

211 ], 

212 expected_inputs=COMMON_INPUTS 

213 | { 

214 "defects", 

215 "gaia_dr2_20200414", 

216 "linearizer", 

217 "overscanRaw", 

218 "ps1_pv3_3pi_20170110", 

219 }, 

220 expected_outputs=COMMON_OUTPUTS 

221 | { 

222 "goodSeeingCoadd", 

223 "goodSeeingCoadd_nImage", 

224 "goodSeeingVisits", 

225 "preSource", 

226 "preSourceTable", 

227 "preSourceTable_visit", 

228 "srcMatchFull", 

229 }, 

230 ) 

231 tester.run(butler, self) 

232 

233 def test_hsc_drp_ci_hsc(self): 

234 butler = self.makeButler(writeable=True) 

235 tester = PipelineStepTester( 

236 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

237 [""], 

238 [ 

239 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

240 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

241 ], 

242 expected_inputs=COMMON_INPUTS 

243 | HSC_INPUTS 

244 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

245 expected_outputs=COMMON_OUTPUTS 

246 | HSC_OUTPUTS 

247 | { 

248 "diaObjectTable_tract", 

249 "diaSourceTable", 

250 "diaSourceTable_tract", 

251 "forcedSourceOnDiaObjectTable", 

252 "forcedSourceOnDiaObjectTable_tract", 

253 "forcedSourceTable", 

254 "forcedSourceTable_tract", 

255 "forced_diff", 

256 "forced_diff_diaObject", 

257 "forced_src_diaObject", 

258 "goodSeeingCoadd", 

259 "goodSeeingCoadd_nImage", 

260 "goodSeeingDiff_assocDiaSrcTable", 

261 "goodSeeingDiff_diaObjTable", 

262 "goodSeeingDiff_diaSrc", 

263 "goodSeeingDiff_diaSrcTable", 

264 "goodSeeingDiff_differenceExp", 

265 "goodSeeingDiff_differenceTempExp", 

266 "goodSeeingDiff_fullDiaObjTable", 

267 "goodSeeingDiff_matchedExp", 

268 "goodSeeingDiff_templateExp", 

269 "goodSeeingVisits", 

270 "mergedForcedSource", 

271 "mergedForcedSourceOnDiaObject", 

272 "objectTable_tract_astrometryRefCat_match", 

273 "sourceTable_visit_astrometryRefCat_match", 

274 }, 

275 ) 

276 tester.run(butler, self) 

277 

278 def test_hsc_drp_prod(self): 

279 butler = self.makeButler(writeable=True) 

280 tester = PipelineStepTester( 

281 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

282 [ 

283 "#step1", 

284 "#step2a", 

285 "#step2b", 

286 "#step2c", 

287 "#step2d", 

288 "#step2e", 

289 "#step3", 

290 "#step4", 

291 "#step7", 

292 ], 

293 [ 

294 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

295 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

296 ], 

297 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

298 expected_outputs=COMMON_OUTPUTS 

299 | HSC_OUTPUTS 

300 | { 

301 "fgcmAtmosphereParameters6", 

302 "fgcmFitParameters0", 

303 "fgcmFitParameters1", 

304 "fgcmFitParameters2", 

305 "fgcmFitParameters3", 

306 "fgcmFitParameters4", 

307 "fgcmFitParameters5", 

308 "fgcmFitParameters6", 

309 "fgcmFlaggedStars0", 

310 "fgcmFlaggedStars1", 

311 "fgcmFlaggedStars2", 

312 "fgcmFlaggedStars3", 

313 "fgcmFlaggedStars4", 

314 "fgcmFlaggedStars5", 

315 "fgcmFlaggedStars6", 

316 "fgcmReferenceStars", 

317 "fgcmStandardStars6", 

318 "fgcmStarIds", 

319 "fgcmStarIndices", 

320 "fgcmStarObservations", 

321 "fgcmZeropoints6", 

322 "transmission_atmosphere_fgcm", 

323 }, 

324 ) 

325 tester.run(butler, self) 

326 

327 def test_hsc_drp_rc2(self): 

328 butler = self.makeButler(writeable=True) 

329 tester = PipelineStepTester( 

330 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

331 [ 

332 "#step1", 

333 "#step2a", 

334 "#step2b", 

335 "#step2cde", 

336 "#step3", 

337 "#step4", 

338 "#step5", 

339 "#step6", 

340 "#step7", 

341 ], 

342 [ 

343 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

344 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

345 ], 

346 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

347 expected_outputs=COMMON_OUTPUTS 

348 | HSC_OUTPUTS 

349 | { 

350 "diaObjectTable_tract", 

351 "diaSourceTable", 

352 "diaSourceTable_tract", 

353 "fgcmAtmosphereParameters4", 

354 "fgcmFitParameters0", 

355 "fgcmFitParameters1", 

356 "fgcmFitParameters2", 

357 "fgcmFitParameters3", 

358 "fgcmFitParameters4", 

359 "fgcmFlaggedStars0", 

360 "fgcmFlaggedStars1", 

361 "fgcmFlaggedStars2", 

362 "fgcmFlaggedStars3", 

363 "fgcmFlaggedStars4", 

364 "fgcmReferenceStars", 

365 "fgcmStandardStars4", 

366 "fgcmStarIds", 

367 "fgcmStarIndices", 

368 "fgcmStarObservations", 

369 "fgcmZeropoints4", 

370 "forcedSourceOnDiaObjectTable", 

371 "forcedSourceOnDiaObjectTable_tract", 

372 "forcedSourceTable", 

373 "forcedSourceTable_tract", 

374 "forced_diff", 

375 "forced_diff_diaObject", 

376 "forced_src_diaObject", 

377 "goodSeeingCoadd", 

378 "goodSeeingCoadd_nImage", 

379 "goodSeeingDiff_assocDiaSrcTable", 

380 "goodSeeingDiff_diaObjTable", 

381 "goodSeeingDiff_diaSrc", 

382 "goodSeeingDiff_diaSrcTable", 

383 "goodSeeingDiff_differenceExp", 

384 "goodSeeingDiff_differenceTempExp", 

385 "goodSeeingDiff_fullDiaObjTable", 

386 "goodSeeingDiff_matchedExp", 

387 "goodSeeingDiff_templateExp", 

388 "goodSeeingVisits", 

389 "mergedForcedSource", 

390 "mergedForcedSourceOnDiaObject", 

391 "transmission_atmosphere_fgcm", 

392 }, 

393 ) 

394 tester.run(butler, self) 

395 

396 def test_hsc_drp_rc2_subset(self): 

397 butler = self.makeButler(writeable=True) 

398 tester = PipelineStepTester( 

399 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2_subset.yaml"), 

400 [ 

401 "#nightlyStep1", 

402 "#nightlyStep2a", 

403 "#nightlyStep2b", 

404 "#nightlyStep2c", 

405 "#nightlyStep2d", 

406 "#nightlyStep3", 

407 "#nightlyStep4", 

408 "#nightlyStep5", 

409 ], 

410 [ 

411 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

412 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

413 ], 

414 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

415 expected_outputs=COMMON_OUTPUTS 

416 | HSC_OUTPUTS 

417 | { 

418 "fgcmAtmosphereParameters4", 

419 "fgcmFitParameters0", 

420 "fgcmFitParameters1", 

421 "fgcmFitParameters2", 

422 "fgcmFitParameters3", 

423 "fgcmFitParameters4", 

424 "fgcmFlaggedStars0", 

425 "fgcmFlaggedStars1", 

426 "fgcmFlaggedStars2", 

427 "fgcmFlaggedStars3", 

428 "fgcmFlaggedStars4", 

429 "fgcmReferenceStars", 

430 "fgcmStandardStars4", 

431 "fgcmStarIds", 

432 "fgcmStarIndices", 

433 "fgcmStarObservations", 

434 "fgcmZeropoints4", 

435 "transmission_atmosphere_fgcm", 

436 }, 

437 ) 

438 tester.run(butler, self) 

439 

440 def test_lsstcam_imsim_drp_ci_imsim(self): 

441 butler = self.makeButler(writeable=True) 

442 tester = PipelineStepTester( 

443 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

444 [f"#step{N}" for N in range(1, 9)], 

445 [ 

446 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

447 ], 

448 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

449 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

450 ) 

451 tester.run(butler, self) 

452 

453 def test_lsstcam_imsim_drp_test_med_1(self): 

454 butler = self.makeButler(writeable=True) 

455 tester = PipelineStepTester( 

456 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

457 [f"#step{N}" for N in range(1, 9)], 

458 [ 

459 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

460 ], 

461 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

462 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

463 ) 

464 tester.run(butler, self) 

465 

466 

467if __name__ == "__main__": 467 ↛ 468line 467 didn't jump to line 468, because the condition on line 467 was never true

468 unittest.main()