Coverage for tests/test_pipelines.py: 40%

65 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-04 03:25 -0800

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "linearizer", 

59 "ps1_pv3_3pi_20170110", 

60 "sky", 

61 "transmission_atmosphere", 

62 "transmission_filter", 

63 "transmission_optics", 

64 "transmission_sensor", 

65 "yBackground", 

66} 

67 

68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

69LSSTCAM_IMSIM_INPUTS = { 

70 "bfk", 

71 "cal_ref_cat_2_2", 

72} 

73 

74# a selection of mostly common outputs 

75COMMON_OUTPUTS = { 

76 "calexp", 

77 "calexpBackground", 

78 "ccdVisitTable", 

79 "deepCoadd", 

80 "deepCoadd_calexp", 

81 "deepCoadd_calexp_background", 

82 "deepCoadd_det", 

83 "deepCoadd_directWarp", 

84 "deepCoadd_forced_src", 

85 "deepCoadd_inputMap", 

86 "deepCoadd_meas", 

87 "deepCoadd_measMatch", 

88 "deepCoadd_measMatchFull", 

89 "deepCoadd_mergeDet", 

90 "deepCoadd_nImage", 

91 "deepCoadd_obj", 

92 "deepCoadd_psfMatchedWarp", 

93 "deepCoadd_ref", 

94 "deepCoadd_scarletModelData", 

95 "finalized_src_table", 

96 "icExp", 

97 "icExpBackground", 

98 "icSrc", 

99 "isolated_star_cat", 

100 "isolated_star_sources", 

101 "objectTable", 

102 "objectTable_tract", 

103 "postISRCCD", 

104 "source", 

105 "sourceTable", 

106 "sourceTable_visit", 

107 "src", 

108 "srcMatch", 

109 "visitSummary", 

110 "visitTable", 

111} 

112 

113# HSC common outputs, in addition to COMMON_OUTPUTS 

114HSC_OUTPUTS = { 

115 "calexp_skyCorr_visit_mosaic", 

116 "calexpBackground_skyCorr_visit_mosaic", 

117 "forced_src", 

118 "preSource", 

119 "preSourceTable", 

120 "preSourceTable_visit", 

121 "skyCorr", 

122 "srcMatchFull", 

123} 

124 

125# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

126LSSTCAM_IMSIM_OUTPUTS = { 

127 "diaObjectTable_tract", 

128 "diaSourceTable", 

129 "diaSourceTable_tract", 

130 "forcedSourceOnDiaObjectTable", 

131 "forcedSourceOnDiaObjectTable_tract", 

132 "forcedSourceTable", 

133 "forcedSourceTable_tract", 

134 "forced_diff", 

135 "forced_diff_diaObject", 

136 "forced_src", 

137 "forced_src_diaObject", 

138 "goodSeeingCoadd", 

139 "goodSeeingCoadd_nImage", 

140 "goodSeeingDiff_assocDiaSrcTable", 

141 "goodSeeingDiff_diaObjTable", 

142 "goodSeeingDiff_diaSrc", 

143 "goodSeeingDiff_diaSrcTable", 

144 "goodSeeingDiff_differenceExp", 

145 "goodSeeingDiff_differenceTempExp", 

146 "goodSeeingDiff_fullDiaObjTable", 

147 "goodSeeingDiff_matchedExp", 

148 "goodSeeingDiff_templateExp", 

149 "goodSeeingVisits", 

150 "mergedForcedSource", 

151 "mergedForcedSourceOnDiaObject", 

152} 

153 

154 

155class PipelineTestCase(unittest.TestCase): 

156 def setUp(self): 

157 self.root = makeTestTempDir(TEST_DIR) 

158 self.maxDiff = None 

159 

160 def tearDown(self): 

161 removeTestTempDir(self.root) 

162 

163 def makeButler(self, **kwargs: Any) -> Butler: 

164 """Return new Butler instance on each call.""" 

165 config = ButlerConfig() 

166 

167 # make separate temporary directory for registry of this instance 

168 tmpdir = tempfile.mkdtemp(dir=self.root) 

169 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

170 config["root"] = self.root 

171 

172 # have to make a registry first 

173 registryConfig = RegistryConfig(config.get("registry")) 

174 Registry.createFromConfig(registryConfig) 

175 

176 butler = Butler(config, **kwargs) 

177 DatastoreMock.apply(butler) 

178 return butler 

179 

180 def test_decam_drp_merian(self): 

181 butler = self.makeButler(writeable=True) 

182 tester0 = PipelineStepTester( 

183 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

184 ["#step0"], 

185 [ 

186 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

187 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

188 ], 

189 expected_inputs={ 

190 "camera", 

191 "raw", 

192 }, 

193 expected_outputs={"overscanRaw"}, 

194 ) 

195 tester0.run(butler, self) 

196 tester = PipelineStepTester( 

197 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

198 [ 

199 "#step1", 

200 "#step2a", 

201 "#step2b", 

202 "#step2c", 

203 "#step2d", 

204 "#step3", 

205 ], 

206 [ 

207 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

208 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

209 ], 

210 expected_inputs=COMMON_INPUTS 

211 | { 

212 "defects", 

213 "gaia_dr2_20200414", 

214 "linearizer", 

215 "overscanRaw", 

216 "ps1_pv3_3pi_20170110", 

217 }, 

218 expected_outputs=COMMON_OUTPUTS 

219 | { 

220 "goodSeeingCoadd", 

221 "goodSeeingCoadd_nImage", 

222 "goodSeeingVisits", 

223 "preSource", 

224 "preSourceTable", 

225 "preSourceTable_visit", 

226 "srcMatchFull", 

227 }, 

228 ) 

229 tester.run(butler, self) 

230 

231 def test_hsc_drp_ci_hsc(self): 

232 butler = self.makeButler(writeable=True) 

233 tester = PipelineStepTester( 

234 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

235 [""], 

236 [ 

237 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

238 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

239 ], 

240 expected_inputs=COMMON_INPUTS 

241 | HSC_INPUTS 

242 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

243 expected_outputs=COMMON_OUTPUTS 

244 | HSC_OUTPUTS 

245 | { 

246 "diaObjectTable_tract", 

247 "diaSourceTable", 

248 "diaSourceTable_tract", 

249 "forcedSourceOnDiaObjectTable", 

250 "forcedSourceOnDiaObjectTable_tract", 

251 "forcedSourceTable", 

252 "forcedSourceTable_tract", 

253 "forced_diff", 

254 "forced_diff_diaObject", 

255 "forced_src_diaObject", 

256 "goodSeeingCoadd", 

257 "goodSeeingCoadd_nImage", 

258 "goodSeeingDiff_assocDiaSrcTable", 

259 "goodSeeingDiff_diaObjTable", 

260 "goodSeeingDiff_diaSrc", 

261 "goodSeeingDiff_diaSrcTable", 

262 "goodSeeingDiff_differenceExp", 

263 "goodSeeingDiff_differenceTempExp", 

264 "goodSeeingDiff_fullDiaObjTable", 

265 "goodSeeingDiff_matchedExp", 

266 "goodSeeingDiff_templateExp", 

267 "goodSeeingVisits", 

268 "mergedForcedSource", 

269 "mergedForcedSourceOnDiaObject", 

270 "objectTable_tract_astrometryRefCat_match", 

271 "sourceTable_visit_astrometryRefCat_match", 

272 }, 

273 ) 

274 tester.run(butler, self) 

275 

276 def test_hsc_drp_prod(self): 

277 butler = self.makeButler(writeable=True) 

278 tester = PipelineStepTester( 

279 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

280 [ 

281 "#step1", 

282 "#step2a", 

283 "#step2b", 

284 "#step2c", 

285 "#step2d", 

286 "#step3", 

287 "#step4", 

288 "#step7", 

289 ], 

290 [ 

291 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

292 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

293 ], 

294 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

295 expected_outputs=COMMON_OUTPUTS 

296 | HSC_OUTPUTS 

297 | { 

298 "fgcmAtmosphereParameters6", 

299 "fgcmFitParameters0", 

300 "fgcmFitParameters1", 

301 "fgcmFitParameters2", 

302 "fgcmFitParameters3", 

303 "fgcmFitParameters4", 

304 "fgcmFitParameters5", 

305 "fgcmFitParameters6", 

306 "fgcmFlaggedStars0", 

307 "fgcmFlaggedStars1", 

308 "fgcmFlaggedStars2", 

309 "fgcmFlaggedStars3", 

310 "fgcmFlaggedStars4", 

311 "fgcmFlaggedStars5", 

312 "fgcmFlaggedStars6", 

313 "fgcmReferenceStars", 

314 "fgcmStandardStars6", 

315 "fgcmStarIds", 

316 "fgcmStarIndices", 

317 "fgcmStarObservations", 

318 "fgcmZeropoints6", 

319 "transmission_atmosphere_fgcm", 

320 }, 

321 ) 

322 tester.run(butler, self) 

323 

324 def test_hsc_drp_rc2(self): 

325 butler = self.makeButler(writeable=True) 

326 tester = PipelineStepTester( 

327 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

328 [f"#step{N}" for N in range(1, 8)], 

329 [ 

330 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

331 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

332 ], 

333 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

334 expected_outputs=COMMON_OUTPUTS 

335 | HSC_OUTPUTS 

336 | { 

337 "diaObjectTable_tract", 

338 "diaSourceTable", 

339 "diaSourceTable_tract", 

340 "fgcmAtmosphereParameters4", 

341 "fgcmFitParameters0", 

342 "fgcmFitParameters1", 

343 "fgcmFitParameters2", 

344 "fgcmFitParameters3", 

345 "fgcmFitParameters4", 

346 "fgcmFlaggedStars0", 

347 "fgcmFlaggedStars1", 

348 "fgcmFlaggedStars2", 

349 "fgcmFlaggedStars3", 

350 "fgcmFlaggedStars4", 

351 "fgcmReferenceStars", 

352 "fgcmStandardStars4", 

353 "fgcmStarIds", 

354 "fgcmStarIndices", 

355 "fgcmStarObservations", 

356 "fgcmZeropoints4", 

357 "forcedSourceOnDiaObjectTable", 

358 "forcedSourceOnDiaObjectTable_tract", 

359 "forcedSourceTable", 

360 "forcedSourceTable_tract", 

361 "forced_diff", 

362 "forced_diff_diaObject", 

363 "forced_src_diaObject", 

364 "goodSeeingCoadd", 

365 "goodSeeingCoadd_nImage", 

366 "goodSeeingDiff_assocDiaSrcTable", 

367 "goodSeeingDiff_diaObjTable", 

368 "goodSeeingDiff_diaSrc", 

369 "goodSeeingDiff_diaSrcTable", 

370 "goodSeeingDiff_differenceExp", 

371 "goodSeeingDiff_differenceTempExp", 

372 "goodSeeingDiff_fullDiaObjTable", 

373 "goodSeeingDiff_matchedExp", 

374 "goodSeeingDiff_templateExp", 

375 "goodSeeingVisits", 

376 "mergedForcedSource", 

377 "mergedForcedSourceOnDiaObject", 

378 "transmission_atmosphere_fgcm", 

379 }, 

380 ) 

381 tester.run(butler, self) 

382 

383 def test_lsstcam_imsim_drp_ci_imsim(self): 

384 butler = self.makeButler(writeable=True) 

385 tester = PipelineStepTester( 

386 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

387 [f"#step{N}" for N in range(1, 8)], 

388 [ 

389 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

390 ], 

391 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

392 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

393 ) 

394 tester.run(butler, self) 

395 

396 def test_lsstcam_imsim_drp_dp02(self): 

397 butler = self.makeButler(writeable=True) 

398 tester = PipelineStepTester( 

399 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-DP0.2.yaml"), 

400 [f"#step{N}" for N in range(1, 8)], 

401 [ 

402 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

403 ], 

404 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

405 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

406 ) 

407 tester.run(butler, self) 

408 

409 def test_lsstcam_imsim_drp_test_med_1(self): 

410 butler = self.makeButler(writeable=True) 

411 tester = PipelineStepTester( 

412 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

413 [f"#step{N}" for N in range(1, 8)], 

414 [ 

415 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

416 ], 

417 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

418 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

419 ) 

420 tester.run(butler, self) 

421 

422 

423if __name__ == "__main__": 423 ↛ 424line 423 didn't jump to line 424, because the condition on line 423 was never true

424 unittest.main()