Coverage for tests/test_pipelines.py: 43%

61 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-14 04:16 -0800

1# This file is part of drp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for tracking edits to drp_pipe pipelines. 

23""" 

24 

25import os 

26import tempfile 

27import unittest 

28from typing import Any 

29 

30from lsst.daf.butler import Butler, ButlerConfig, Registry, RegistryConfig 

31from lsst.daf.butler.tests import DatastoreMock 

32from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

33from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester 

34 

35PIPELINES_DIR = os.path.join(os.path.dirname(__file__), "..", "pipelines") 

36TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38# mostly common inputs 

39COMMON_INPUTS = { 

40 "bfKernel", 

41 "bias", 

42 "camera", 

43 "crosstalk", 

44 "dark", 

45 "flat", 

46 "fringe", 

47 "isrOverscanCorrected", 

48 "raw", 

49 "skyMap", 

50} 

51 

52# HSC common inputs, in addition to COMMON_INPUTS 

53HSC_INPUTS = { 

54 "brightObjectMask", 

55 "brighterFatterKernel", 

56 "defects", 

57 "gaia_dr2_20200414", 

58 "linearizer", 

59 "ps1_pv3_3pi_20170110", 

60 "sky", 

61 "transmission_atmosphere", 

62 "transmission_filter", 

63 "transmission_optics", 

64 "transmission_sensor", 

65 "yBackground", 

66} 

67 

68# LSSTCam-imSim common inputs, in addition to COMMON_INPUTS 

69LSSTCAM_IMSIM_INPUTS = { 

70 "bfk", 

71 "cal_ref_cat_2_2", 

72} 

73 

74# a selection of mostly common outputs 

75COMMON_OUTPUTS = { 

76 "calexp", 

77 "calexpBackground", 

78 "ccdVisitTable", 

79 "deepCoadd", 

80 "deepCoadd_calexp", 

81 "deepCoadd_calexp_background", 

82 "deepCoadd_det", 

83 "deepCoadd_directWarp", 

84 "deepCoadd_forced_src", 

85 "deepCoadd_inputMap", 

86 "deepCoadd_meas", 

87 "deepCoadd_measMatch", 

88 "deepCoadd_measMatchFull", 

89 "deepCoadd_mergeDet", 

90 "deepCoadd_nImage", 

91 "deepCoadd_obj", 

92 "deepCoadd_psfMatchedWarp", 

93 "deepCoadd_ref", 

94 "deepCoadd_scarletModelData", 

95 "finalized_src_table", 

96 "finalVisitSummary", 

97 "icExp", 

98 "icExpBackground", 

99 "icSrc", 

100 "isolated_star_cat", 

101 "isolated_star_sources", 

102 "objectTable", 

103 "objectTable_tract", 

104 "postISRCCD", 

105 "source", 

106 "sourceTable", 

107 "sourceTable_visit", 

108 "src", 

109 "srcMatch", 

110 "visitSummary", 

111 "visitTable", 

112} 

113 

114# HSC common outputs, in addition to COMMON_OUTPUTS 

115HSC_OUTPUTS = { 

116 "calexp_skyCorr_visit_mosaic", 

117 "calexpBackground_skyCorr_visit_mosaic", 

118 "forced_src", 

119 "preSource", 

120 "preSourceTable", 

121 "preSourceTable_visit", 

122 "skyCorr", 

123 "srcMatchFull", 

124} 

125 

126# LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS 

127LSSTCAM_IMSIM_OUTPUTS = { 

128 "diaObjectTable_tract", 

129 "diaSourceTable", 

130 "diaSourceTable_tract", 

131 "forcedSourceOnDiaObjectTable", 

132 "forcedSourceOnDiaObjectTable_tract", 

133 "forcedSourceTable", 

134 "forcedSourceTable_tract", 

135 "forced_diff", 

136 "forced_diff_diaObject", 

137 "forced_src", 

138 "forced_src_diaObject", 

139 "goodSeeingCoadd", 

140 "goodSeeingCoadd_nImage", 

141 "goodSeeingDiff_assocDiaSrcTable", 

142 "goodSeeingDiff_diaObjTable", 

143 "goodSeeingDiff_diaSrc", 

144 "goodSeeingDiff_diaSrcTable", 

145 "goodSeeingDiff_differenceExp", 

146 "goodSeeingDiff_differenceTempExp", 

147 "goodSeeingDiff_fullDiaObjTable", 

148 "goodSeeingDiff_matchedExp", 

149 "goodSeeingDiff_templateExp", 

150 "goodSeeingVisits", 

151 "mergedForcedSource", 

152 "mergedForcedSourceOnDiaObject", 

153} 

154 

155 

156class PipelineTestCase(unittest.TestCase): 

157 def setUp(self): 

158 self.root = makeTestTempDir(TEST_DIR) 

159 self.maxDiff = None 

160 

161 def tearDown(self): 

162 removeTestTempDir(self.root) 

163 

164 def makeButler(self, **kwargs: Any) -> Butler: 

165 """Return new Butler instance on each call.""" 

166 config = ButlerConfig() 

167 

168 # make separate temporary directory for registry of this instance 

169 tmpdir = tempfile.mkdtemp(dir=self.root) 

170 config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3" 

171 config["root"] = self.root 

172 

173 # have to make a registry first 

174 registryConfig = RegistryConfig(config.get("registry")) 

175 Registry.createFromConfig(registryConfig) 

176 

177 butler = Butler(config, **kwargs) 

178 DatastoreMock.apply(butler) 

179 return butler 

180 

181 def test_decam_drp_merian(self): 

182 butler = self.makeButler(writeable=True) 

183 tester0 = PipelineStepTester( 

184 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

185 ["#step0"], 

186 [ 

187 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

188 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

189 ], 

190 expected_inputs={ 

191 "camera", 

192 "raw", 

193 }, 

194 expected_outputs={"overscanRaw"}, 

195 ) 

196 tester0.run(butler, self) 

197 tester = PipelineStepTester( 

198 os.path.join(PIPELINES_DIR, "DECam", "DRP-Merian.yaml"), 

199 [ 

200 "#step1", 

201 "#step2a", 

202 "#step2b", 

203 "#step2d", 

204 "#step2e", 

205 "#step3", 

206 ], 

207 [ 

208 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

209 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

210 ], 

211 expected_inputs=COMMON_INPUTS 

212 | { 

213 "defects", 

214 "gaia_dr2_20200414", 

215 "linearizer", 

216 "overscanRaw", 

217 "ps1_pv3_3pi_20170110", 

218 }, 

219 expected_outputs=COMMON_OUTPUTS 

220 | { 

221 "goodSeeingCoadd", 

222 "goodSeeingCoadd_nImage", 

223 "goodSeeingVisits", 

224 "preSource", 

225 "preSourceTable", 

226 "preSourceTable_visit", 

227 "srcMatchFull", 

228 }, 

229 ) 

230 tester.run(butler, self) 

231 

232 def test_hsc_drp_ci_hsc(self): 

233 butler = self.makeButler(writeable=True) 

234 tester = PipelineStepTester( 

235 os.path.join(PIPELINES_DIR, "HSC", "DRP-ci_hsc.yaml"), 

236 [""], 

237 [ 

238 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

239 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

240 ], 

241 expected_inputs=COMMON_INPUTS 

242 | HSC_INPUTS 

243 | {"jointcalPhotoCalibCatalog", "jointcalSkyWcsCatalog"}, 

244 expected_outputs=COMMON_OUTPUTS 

245 | HSC_OUTPUTS 

246 | { 

247 "diaObjectTable_tract", 

248 "diaSourceTable", 

249 "diaSourceTable_tract", 

250 "forcedSourceOnDiaObjectTable", 

251 "forcedSourceOnDiaObjectTable_tract", 

252 "forcedSourceTable", 

253 "forcedSourceTable_tract", 

254 "forced_diff", 

255 "forced_diff_diaObject", 

256 "forced_src_diaObject", 

257 "goodSeeingCoadd", 

258 "goodSeeingCoadd_nImage", 

259 "goodSeeingDiff_assocDiaSrcTable", 

260 "goodSeeingDiff_diaObjTable", 

261 "goodSeeingDiff_diaSrc", 

262 "goodSeeingDiff_diaSrcTable", 

263 "goodSeeingDiff_differenceExp", 

264 "goodSeeingDiff_differenceTempExp", 

265 "goodSeeingDiff_fullDiaObjTable", 

266 "goodSeeingDiff_matchedExp", 

267 "goodSeeingDiff_templateExp", 

268 "goodSeeingVisits", 

269 "mergedForcedSource", 

270 "mergedForcedSourceOnDiaObject", 

271 "objectTable_tract_astrometryRefCat_match", 

272 "sourceTable_visit_astrometryRefCat_match", 

273 }, 

274 ) 

275 tester.run(butler, self) 

276 

277 def test_hsc_drp_prod(self): 

278 butler = self.makeButler(writeable=True) 

279 tester = PipelineStepTester( 

280 os.path.join(PIPELINES_DIR, "HSC", "DRP-Prod.yaml"), 

281 [ 

282 "#step1", 

283 "#step2a", 

284 "#step2b", 

285 "#step2c", 

286 "#step2d", 

287 "#step2e", 

288 "#step3", 

289 "#step4", 

290 "#step7", 

291 ], 

292 [ 

293 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

294 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

295 ], 

296 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

297 expected_outputs=COMMON_OUTPUTS 

298 | HSC_OUTPUTS 

299 | { 

300 "fgcmAtmosphereParameters6", 

301 "fgcmFitParameters0", 

302 "fgcmFitParameters1", 

303 "fgcmFitParameters2", 

304 "fgcmFitParameters3", 

305 "fgcmFitParameters4", 

306 "fgcmFitParameters5", 

307 "fgcmFitParameters6", 

308 "fgcmFlaggedStars0", 

309 "fgcmFlaggedStars1", 

310 "fgcmFlaggedStars2", 

311 "fgcmFlaggedStars3", 

312 "fgcmFlaggedStars4", 

313 "fgcmFlaggedStars5", 

314 "fgcmFlaggedStars6", 

315 "fgcmReferenceStars", 

316 "fgcmStandardStars6", 

317 "fgcmStarIds", 

318 "fgcmStarIndices", 

319 "fgcmStarObservations", 

320 "fgcmZeropoints6", 

321 "transmission_atmosphere_fgcm", 

322 }, 

323 ) 

324 tester.run(butler, self) 

325 

326 def test_hsc_drp_rc2(self): 

327 butler = self.makeButler(writeable=True) 

328 tester = PipelineStepTester( 

329 os.path.join(PIPELINES_DIR, "HSC", "DRP-RC2.yaml"), 

330 [ 

331 "#step1", 

332 "#step2a", 

333 "#step2b", 

334 "#step2cde", 

335 "#step3", 

336 "#step4", 

337 "#step5", 

338 "#step6", 

339 "#step7", 

340 ], 

341 [ 

342 ("ps1_pv3_3pi_20170110", {"htm7"}, "Catalog", False), 

343 ("gaia_dr2_20200414", {"htm7"}, "Catalog", False), 

344 ], 

345 expected_inputs=COMMON_INPUTS | HSC_INPUTS | {"fgcmLookUpTable"}, 

346 expected_outputs=COMMON_OUTPUTS 

347 | HSC_OUTPUTS 

348 | { 

349 "diaObjectTable_tract", 

350 "diaSourceTable", 

351 "diaSourceTable_tract", 

352 "fgcmAtmosphereParameters4", 

353 "fgcmFitParameters0", 

354 "fgcmFitParameters1", 

355 "fgcmFitParameters2", 

356 "fgcmFitParameters3", 

357 "fgcmFitParameters4", 

358 "fgcmFlaggedStars0", 

359 "fgcmFlaggedStars1", 

360 "fgcmFlaggedStars2", 

361 "fgcmFlaggedStars3", 

362 "fgcmFlaggedStars4", 

363 "fgcmReferenceStars", 

364 "fgcmStandardStars4", 

365 "fgcmStarIds", 

366 "fgcmStarIndices", 

367 "fgcmStarObservations", 

368 "fgcmZeropoints4", 

369 "forcedSourceOnDiaObjectTable", 

370 "forcedSourceOnDiaObjectTable_tract", 

371 "forcedSourceTable", 

372 "forcedSourceTable_tract", 

373 "forced_diff", 

374 "forced_diff_diaObject", 

375 "forced_src_diaObject", 

376 "goodSeeingCoadd", 

377 "goodSeeingCoadd_nImage", 

378 "goodSeeingDiff_assocDiaSrcTable", 

379 "goodSeeingDiff_diaObjTable", 

380 "goodSeeingDiff_diaSrc", 

381 "goodSeeingDiff_diaSrcTable", 

382 "goodSeeingDiff_differenceExp", 

383 "goodSeeingDiff_differenceTempExp", 

384 "goodSeeingDiff_fullDiaObjTable", 

385 "goodSeeingDiff_matchedExp", 

386 "goodSeeingDiff_templateExp", 

387 "goodSeeingVisits", 

388 "mergedForcedSource", 

389 "mergedForcedSourceOnDiaObject", 

390 "transmission_atmosphere_fgcm", 

391 }, 

392 ) 

393 tester.run(butler, self) 

394 

395 def test_lsstcam_imsim_drp_ci_imsim(self): 

396 butler = self.makeButler(writeable=True) 

397 tester = PipelineStepTester( 

398 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-ci_imsim.yaml"), 

399 [f"#step{N}" for N in range(1, 8)], 

400 [ 

401 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

402 ], 

403 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

404 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

405 ) 

406 tester.run(butler, self) 

407 

408 def test_lsstcam_imsim_drp_test_med_1(self): 

409 butler = self.makeButler(writeable=True) 

410 tester = PipelineStepTester( 

411 os.path.join(PIPELINES_DIR, "LSSTCam-imSim", "DRP-test-med-1.yaml"), 

412 [f"#step{N}" for N in range(1, 8)], 

413 [ 

414 ("cal_ref_cat_2_2", {"htm7"}, "Catalog", False), 

415 ], 

416 expected_inputs=COMMON_INPUTS | LSSTCAM_IMSIM_INPUTS, 

417 expected_outputs=COMMON_OUTPUTS | LSSTCAM_IMSIM_OUTPUTS, 

418 ) 

419 tester.run(butler, self) 

420 

421 

422if __name__ == "__main__": 422 ↛ 423line 422 didn't jump to line 423, because the condition on line 422 was never true

423 unittest.main()