Coverage for tests/test_pipelines.py: 40%

65 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 20:43 -0800

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "linearizer", 

59 "ps1_pv3_3pi_20170110", 

60 "sky", 

61 "transmission_atmosphere", 

62 "transmission_filter", 

63 "transmission_optics", 

64 "transmission_sensor", 

65 "yBackground", 

66} 

67 

68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

69LSSTCAM_IMSIM_INPUTS = { 

70 "bfk", 

71 "cal_ref_cat_2_2", 

72} 

73 

74# a selection of mostly common outputs 

75COMMON_OUTPUTS = { 

76 "calexp", 

77 "calexpBackground", 

78 "ccdVisitTable", 

79 "deepCoadd", 

80 "deepCoadd_calexp", 

81 "deepCoadd_calexp_background", 

82 "deepCoadd_det", 

83 "deepCoadd_directWarp", 

84 "deepCoadd_forced_src", 

85 "deepCoadd_inputMap", 

86 "deepCoadd_meas", 

87 "deepCoadd_measMatch", 

88 "deepCoadd_measMatchFull", 

89 "deepCoadd_mergeDet", 

90 "deepCoadd_nImage", 

91 "deepCoadd_obj", 

92 "deepCoadd_psfMatchedWarp", 

93 "deepCoadd_ref", 

94 "deepCoadd_scarletModelData", 

95 "finalized_src_table", 

96 "icExp", 

97 "icExpBackground", 

98 "icSrc", 

99 "isolated_star_cat", 

100 "isolated_star_sources", 

101 "objectTable", 

102 "objectTable_tract", 

103 "postISRCCD", 

104 "source", 

105 "sourceTable", 

106 "sourceTable_visit", 

107 "src", 

108 "srcMatch", 

109 "visitSummary", 

110 "visitTable", 

111} 

112 

113# HSC common outputs, in addition to COMMON_OUTPUTS 

114HSC_OUTPUTS = { 

115 "calexp_camera", 

116 "forced_src", 

117 "preSource", 

118 "preSourceTable", 

119 "preSourceTable_visit", 

120 "skyCorr", 

121 "srcMatchFull", 

122} 

123 

124# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

125LSSTCAM_IMSIM_OUTPUTS = { 

126 "diaObjectTable_tract", 

127 "diaSourceTable", 

128 "diaSourceTable_tract", 

129 "forcedSourceOnDiaObjectTable", 

130 "forcedSourceOnDiaObjectTable_tract", 

131 "forcedSourceTable", 

132 "forcedSourceTable_tract", 

133 "forced_diff", 

134 "forced_diff_diaObject", 

135 "forced_src", 

136 "forced_src_diaObject", 

137 "goodSeeingCoadd", 

138 "goodSeeingCoadd_nImage", 

139 "goodSeeingDiff_assocDiaSrcTable", 

140 "goodSeeingDiff_diaObjTable", 

141 "goodSeeingDiff_diaSrc", 

142 "goodSeeingDiff_diaSrcTable", 

143 "goodSeeingDiff_differenceExp", 

144 "goodSeeingDiff_differenceTempExp", 

145 "goodSeeingDiff_fullDiaObjTable", 

146 "goodSeeingDiff_matchedExp", 

147 "goodSeeingDiff_templateExp", 

148 "goodSeeingVisits", 

149 "mergedForcedSource", 

150 "mergedForcedSourceOnDiaObject", 

151} 

152 

153 

154class PipelineTestCase(unittest.TestCase): 

155 def setUp(self): 

156 self.root = makeTestTempDir(TEST_DIR) 

157 self.maxDiff = None 

158 

159 def tearDown(self): 

160 removeTestTempDir(self.root) 

161 

162 def makeButler(self, **kwargs: Any) -> Butler: 

163 """Return new Butler instance on each call.""" 

164 config = ButlerConfig() 

165 

166 # make separate temporary directory for registry of this instance 

167 tmpdir = tempfile.mkdtemp(dir=self.root) 

168 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

169 config["root"] = self.root 

170 

171 # have to make a registry first 

172 registryConfig = RegistryConfig(config.get("registry")) 

173 Registry.createFromConfig(registryConfig) 

174 

175 butler = Butler(config, **kwargs) 

176 DatastoreMock.apply(butler) 

177 return butler 

178 

179 def test_decam_drp_merian(self): 

180 butler = self.makeButler(writeable=True) 

181 tester0 = PipelineStepTester( 

182 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

183 ["#step0"], 

184 [ 

185 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

186 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

187 ], 

188 expected_inputs={ 

189 "camera", 

190 "raw", 

191 }, 

192 expected_outputs={"overscanRaw"}, 

193 ) 

194 tester0.run(butler, self) 

195 tester = PipelineStepTester( 

196 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

197 [ 

198 "#step1", 

199 "#step2a", 

200 "#step2b", 

201 "#step2c", 

202 "#step2d", 

203 "#step3", 

204 ], 

205 [ 

206 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

207 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

208 ], 

209 expected_inputs=COMMON_INPUTS 

210 | { 

211 "defects", 

212 "gaia_dr2_20200414", 

213 "linearizer", 

214 "overscanRaw", 

215 "ps1_pv3_3pi_20170110", 

216 }, 

217 expected_outputs=COMMON_OUTPUTS 

218 | { 

219 "goodSeeingCoadd", 

220 "goodSeeingCoadd_nImage", 

221 "goodSeeingVisits", 

222 "preSource", 

223 "preSourceTable", 

224 "preSourceTable_visit", 

225 "srcMatchFull", 

226 }, 

227 ) 

228 tester.run(butler, self) 

229 

230 def test_hsc_drp_ci_hsc(self): 

231 butler = self.makeButler(writeable=True) 

232 tester = PipelineStepTester( 

233 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

234 [""], 

235 [ 

236 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

237 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

238 ], 

239 expected_inputs=COMMON_INPUTS 

240 | HSC_INPUTS 

241 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

242 expected_outputs=COMMON_OUTPUTS 

243 | HSC_OUTPUTS 

244 | { 

245 "diaObjectTable_tract", 

246 "diaSourceTable", 

247 "diaSourceTable_tract", 

248 "forcedSourceOnDiaObjectTable", 

249 "forcedSourceOnDiaObjectTable_tract", 

250 "forcedSourceTable", 

251 "forcedSourceTable_tract", 

252 "forced_diff", 

253 "forced_diff_diaObject", 

254 "forced_src_diaObject", 

255 "goodSeeingCoadd", 

256 "goodSeeingCoadd_nImage", 

257 "goodSeeingDiff_assocDiaSrcTable", 

258 "goodSeeingDiff_diaObjTable", 

259 "goodSeeingDiff_diaSrc", 

260 "goodSeeingDiff_diaSrcTable", 

261 "goodSeeingDiff_differenceExp", 

262 "goodSeeingDiff_differenceTempExp", 

263 "goodSeeingDiff_fullDiaObjTable", 

264 "goodSeeingDiff_matchedExp", 

265 "goodSeeingDiff_templateExp", 

266 "goodSeeingVisits", 

267 "mergedForcedSource", 

268 "mergedForcedSourceOnDiaObject", 

269 "objectTable_tract_astrometryRefCat_match", 

270 "sourceTable_visit_astrometryRefCat_match", 

271 }, 

272 ) 

273 tester.run(butler, self) 

274 

275 def test_hsc_drp_prod(self): 

276 butler = self.makeButler(writeable=True) 

277 tester = PipelineStepTester( 

278 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

279 [ 

280 "#step1", 

281 "#step2a", 

282 "#step2b", 

283 "#step2c", 

284 "#step2d", 

285 "#step3", 

286 "#step4", 

287 "#step7", 

288 ], 

289 [ 

290 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

291 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

292 ], 

293 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

294 expected_outputs=COMMON_OUTPUTS 

295 | HSC_OUTPUTS 

296 | { 

297 "fgcmAtmosphereParameters6", 

298 "fgcmFitParameters0", 

299 "fgcmFitParameters1", 

300 "fgcmFitParameters2", 

301 "fgcmFitParameters3", 

302 "fgcmFitParameters4", 

303 "fgcmFitParameters5", 

304 "fgcmFitParameters6", 

305 "fgcmFlaggedStars0", 

306 "fgcmFlaggedStars1", 

307 "fgcmFlaggedStars2", 

308 "fgcmFlaggedStars3", 

309 "fgcmFlaggedStars4", 

310 "fgcmFlaggedStars5", 

311 "fgcmFlaggedStars6", 

312 "fgcmReferenceStars", 

313 "fgcmStandardStars6", 

314 "fgcmStarIds", 

315 "fgcmStarIndices", 

316 "fgcmStarObservations", 

317 "fgcmZeropoints6", 

318 "transmission_atmosphere_fgcm", 

319 }, 

320 ) 

321 tester.run(butler, self) 

322 

323 def test_hsc_drp_rc2(self): 

324 butler = self.makeButler(writeable=True) 

325 tester = PipelineStepTester( 

326 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

327 [f"#step{N}" for N in range(1, 8)], 

328 [ 

329 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

330 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

331 ], 

332 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

333 expected_outputs=COMMON_OUTPUTS 

334 | HSC_OUTPUTS 

335 | { 

336 "diaObjectTable_tract", 

337 "diaSourceTable", 

338 "diaSourceTable_tract", 

339 "fgcmAtmosphereParameters4", 

340 "fgcmFitParameters0", 

341 "fgcmFitParameters1", 

342 "fgcmFitParameters2", 

343 "fgcmFitParameters3", 

344 "fgcmFitParameters4", 

345 "fgcmFlaggedStars0", 

346 "fgcmFlaggedStars1", 

347 "fgcmFlaggedStars2", 

348 "fgcmFlaggedStars3", 

349 "fgcmFlaggedStars4", 

350 "fgcmReferenceStars", 

351 "fgcmStandardStars4", 

352 "fgcmStarIds", 

353 "fgcmStarIndices", 

354 "fgcmStarObservations", 

355 "fgcmZeropoints4", 

356 "forcedSourceOnDiaObjectTable", 

357 "forcedSourceOnDiaObjectTable_tract", 

358 "forcedSourceTable", 

359 "forcedSourceTable_tract", 

360 "forced_diff", 

361 "forced_diff_diaObject", 

362 "forced_src_diaObject", 

363 "goodSeeingCoadd", 

364 "goodSeeingCoadd_nImage", 

365 "goodSeeingDiff_assocDiaSrcTable", 

366 "goodSeeingDiff_diaObjTable", 

367 "goodSeeingDiff_diaSrc", 

368 "goodSeeingDiff_diaSrcTable", 

369 "goodSeeingDiff_differenceExp", 

370 "goodSeeingDiff_differenceTempExp", 

371 "goodSeeingDiff_fullDiaObjTable", 

372 "goodSeeingDiff_matchedExp", 

373 "goodSeeingDiff_templateExp", 

374 "goodSeeingVisits", 

375 "mergedForcedSource", 

376 "mergedForcedSourceOnDiaObject", 

377 "transmission_atmosphere_fgcm", 

378 }, 

379 ) 

380 tester.run(butler, self) 

381 

382 def test_lsstcam_imsim_drp_ci_imsim(self): 

383 butler = self.makeButler(writeable=True) 

384 tester = PipelineStepTester( 

385 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

386 [f"#step{N}" for N in range(1, 8)], 

387 [ 

388 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

389 ], 

390 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

391 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

392 ) 

393 tester.run(butler, self) 

394 

395 def test_lsstcam_imsim_drp_dp02(self): 

396 butler = self.makeButler(writeable=True) 

397 tester = PipelineStepTester( 

398 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-DP0.2.yaml"), 

399 [f"#step{N}" for N in range(1, 8)], 

400 [ 

401 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

402 ], 

403 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

404 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

405 ) 

406 tester.run(butler, self) 

407 

408 def test_lsstcam_imsim_drp_test_med_1(self): 

409 butler = self.makeButler(writeable=True) 

410 tester = PipelineStepTester( 

411 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

412 [f"#step{N}" for N in range(1, 8)], 

413 [ 

414 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

415 ], 

416 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

417 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

418 ) 

419 tester.run(butler, self) 

420 

421 

422if __name__ == "__main__": 422 ↛ 423line 422 didn't jump to line 423, because the condition on line 422 was never true

423 unittest.main()