Coverage for tests/jointcalTestBase.py: 13%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

145 statements  

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["importRepository", "JointcalTestBase"] 

23 

24import copy 

25import os 

26import shutil 

27import tempfile 

28 

29import lsst.afw.image.utils 

30from lsst.ctrl.mpexec import SimplePipelineExecutor 

31import lsst.daf.butler 

32from lsst.daf.butler.script import ingest_files 

33import lsst.obs.base 

34import lsst.geom 

35from lsst.verify.bin.jobReporter import JobReporter 

36 

37from lsst.jointcal import jointcal, utils 

38 

39 

40def importRepository(instrument, exportPath, exportFile, outputDir=None, 

41 refcats=None, refcatPath=""): 

42 """Import a gen3 test repository into a test path. 

43 

44 Parameters 

45 ---------- 

46 instrument : `str` 

47 Full string name for the instrument. 

48 exportPath : `str` 

49 Path to location of repository to export. 

50 exportFile : `str` 

51 Filename of YAML butler export data, describing the data to import. 

52 outputDir : `str`, or `None` 

53 Root path to put the test repo in; appended with `testrepo/`. 

54 If not supplied, a temporary path is generated with mkdtemp; this path 

55 must be deleted by the calling code, it is not automatically removed. 

56 refcats : `dict` [`str`, `str`] 

57 Mapping of refcat name to relative path to .ecsv ingest file. 

58 refcatPath : `str` 

59 Path prefix for refcat ingest files and files linked therein. 

60 

61 Returns 

62 ------- 

63 repopath : `str` 

64 The path to the newly created butler repo. 

65 """ 

66 if outputDir is None: 

67 repopath = tempfile.mkdtemp() 

68 else: 

69 repopath = os.path.join(outputDir, "testrepo") 

70 

71 # Make the repo and retrieve a writeable Butler 

72 _ = lsst.daf.butler.Butler.makeRepo(repopath) 

73 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

74 

75 instrInstance = lsst.obs.base.utils.getInstrument(instrument) 

76 instrInstance.register(butler.registry) 

77 

78 # Register refcats first, so the `refcats` collection will exist. 

79 if refcats is not None: 

80 for name, file in refcats.items(): 

81 graph = butler.registry.dimensions.extract(["htm7"]) 

82 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog", 

83 universe=butler.registry.dimensions) 

84 butler.registry.registerDatasetType(datasetType) 

85 ingest_files(repopath, name, "refcats", file, prefix=refcatPath) 

86 # New butler to get the refcats collections to be visible 

87 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

88 

89 butler.import_(directory=exportPath, filename=exportFile, 

90 transfer='symlink', 

91 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

92 return repopath 

93 

94 

95class JointcalTestBase: 

96 """ 

97 Base class for jointcal tests, to genericize some test running and setup. 

98 

99 Derive from this first, then from TestCase. 

100 """ 

101 

102 def set_output_dir(self): 

103 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

104 

105 def setUp_base(self, center, radius, 

106 match_radius=0.1*lsst.geom.arcseconds, 

107 input_dir="", 

108 all_visits=None, 

109 other_args=None, 

110 do_plot=False, 

111 log_level=None, 

112 where="", 

113 refcats=None, 

114 refcatPath="", 

115 inputCollections=None): 

116 """ 

117 Call from your child classes's setUp() to get the necessary variables built. 

118 

119 Parameters 

120 ---------- 

121 center : `lsst.geom.SpherePoint` 

122 Center of the reference catalog. 

123 radius : `lsst.geom.Angle` 

124 Radius from center to load reference catalog objects inside. 

125 match_radius : `lsst.geom.Angle` 

126 matching radius when calculating RMS of result. 

127 input_dir : `str` 

128 Directory of input butler repository. 

129 all_visits : `list` [`int`] 

130 List of the available visits to generate the parseAndRun arguments. 

131 other_args : `list` [`str`] 

132 Optional other arguments for the butler dataId. 

133 do_plot : `bool` 

134 Set to True for a comparison plot and some diagnostic numbers. 

135 log_level : `str` 

136 Set to the default log level you want jointcal to produce while the 

137 tests are running. See the developer docs about logging for valid 

138 levels: https://developer.lsst.io/coding/logging.html 

139 where : `str` 

140 Data ID query for pipetask specifying the data to run on. 

141 refcats : `dict` [`str`, `str`] 

142 Mapping of refcat name to relative path to .ecsv ingest file. 

143 refcatPath : `str` 

144 Path prefix for refcat ingest files and files linked therein. 

145 inputCollections : `list` [`str`] 

146 String to use for "-i" input collections (comma delimited). 

147 For example, "refcats,HSC/runs/tests,HSC/calib" 

148 """ 

149 self.path = os.path.dirname(__file__) 

150 

151 self.center = center 

152 self.radius = radius 

153 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True) 

154 self.input_dir = input_dir 

155 self.all_visits = all_visits 

156 if other_args is None: 

157 other_args = [] 

158 self.other_args = other_args 

159 self.do_plot = do_plot 

160 self.log_level = log_level 

161 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match. 

162 # 100 is a balance between good centroids and enough sources. 

163 self.flux_limit = 100 

164 

165 # Individual tests may want to tweak the config that is passed to parseAndRun(). 

166 self.config = None 

167 self.configfiles = [] 

168 

169 # Append `msg` arguments to assert failures. 

170 self.longMessage = True 

171 

172 self.where = where 

173 self.refcats = refcats 

174 self.refcatPath = refcatPath 

175 self.inputCollections = inputCollections 

176 

177 # Ensure that the filter list is reset for each test so that we avoid 

178 # confusion or contamination from other instruments. 

179 lsst.obs.base.FilterDefinitionCollection.reset() 

180 

181 self.set_output_dir() 

182 

183 def tearDown(self): 

184 shutil.rmtree(self.output_dir, ignore_errors=True) 

185 

186 if getattr(self, 'reference', None) is not None: 

187 del self.reference 

188 if getattr(self, 'oldWcsList', None) is not None: 

189 del self.oldWcsList 

190 if getattr(self, 'jointcalTask', None) is not None: 

191 del self.jointcalTask 

192 if getattr(self, 'jointcalStatistics', None) is not None: 

193 del self.jointcalStatistics 

194 if getattr(self, 'config', None) is not None: 

195 del self.config 

196 

197 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1, 

198 metrics=None): 

199 """ 

200 Test parseAndRun for jointcal on nCatalogs. 

201 

202 Checks relative and absolute astrometric error (arcsec) and photometric 

203 repeatability (PA1 from the SRD). 

204 

205 Parameters 

206 ---------- 

207 nCatalogs : `int` 

208 Number of catalogs to run jointcal on. Used to construct the "id" 

209 field for parseAndRun. 

210 dist_rms_relative : `astropy.Quantity` 

211 Minimum relative astrometric rms post-jointcal to pass the test. 

212 dist_rms_absolute : `astropy.Quantity` 

213 Minimum absolute astrometric rms post-jointcal to pass the test. 

214 pa1 : `float` 

215 Minimum PA1 (from Table 14 of the Science Requirements Document: 

216 https://ls.st/LPM-17) post-jointcal to pass the test. 

217 metrics : `dict`, optional 

218 Dictionary of 'metricName': value to test jointcal's result.metrics 

219 against. 

220 

221 Returns 

222 ------- 

223 dataRefs : `list` [`lsst.daf.persistence.ButlerDataRef`] 

224 The dataRefs that were processed. 

225 """ 

226 

227 resultFull = self._runJointcalTask(nCatalogs, metrics=metrics) 

228 result = resultFull.resultList[0].result # shorten this very long thing 

229 

230 def compute_statistics(refObjLoader): 

231 refCat = refObjLoader.loadSkyCircle(self.center, 

232 self.radius, 

233 result.defaultFilter.bandLabel, 

234 epoch=result.epoch).refCat 

235 rms_result = self.jointcalStatistics.compute_rms(result.dataRefs, refCat) 

236 # Make plots before testing, if requested, so we still get plots if tests fail. 

237 if self.do_plot: 

238 self._plotJointcalTask(result.dataRefs, result.oldWcsList) 

239 return rms_result 

240 

241 # we now have different astrometry/photometry refcats, so have to 

242 # do these calculations separately 

243 if self.jointcalStatistics.do_astrometry: 

244 refObjLoader = result.astrometryRefObjLoader 

245 # preserve do_photometry for the next `if` 

246 temp = copy.copy(self.jointcalStatistics.do_photometry) 

247 self.jointcalStatistics.do_photometry = False 

248 rms_result = compute_statistics(refObjLoader) 

249 self.jointcalStatistics.do_photometry = temp # restore do_photometry 

250 

251 if dist_rms_relative is not None and dist_rms_absolute is not None: 

252 self.assertLess(rms_result.dist_relative, dist_rms_relative) 

253 self.assertLess(rms_result.dist_absolute, dist_rms_absolute) 

254 

255 if self.jointcalStatistics.do_photometry: 

256 refObjLoader = result.photometryRefObjLoader 

257 self.jointcalStatistics.do_astrometry = False 

258 rms_result = compute_statistics(refObjLoader) 

259 

260 if pa1 is not None: 

261 self.assertLess(rms_result.pa1, pa1) 

262 

263 return result.dataRefs 

264 

265 def _runJointcalTask(self, nCatalogs, metrics=None): 

266 """ 

267 Run jointcalTask on nCatalogs, with the most basic tests. 

268 Tests for non-empty result list, and that the basic metrics are correct. 

269 

270 Parameters 

271 ---------- 

272 nCatalogs : `int` 

273 Number of catalogs to test on. 

274 metrics : `dict`, optional 

275 Dictionary of 'metricName': value to test jointcal's result.metrics 

276 against. 

277 

278 Returns 

279 ------- 

280 result : `pipe.base.Struct` 

281 The structure returned by jointcalTask.run() 

282 """ 

283 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs]) 

284 if self.log_level is not None: 

285 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level]) 

286 

287 # Place default configfile first so that specific subclass configfiles are applied after 

288 test_config = os.path.join(self.path, 'config/config.py') 

289 configfiles = [test_config] + self.configfiles 

290 

291 args = [self.input_dir, '--output', self.output_dir, 

292 '--clobber-versions', '--clobber-config', 

293 '--doraise', '--configfile', *configfiles, 

294 '--id', 'visit=%s'%visits] 

295 args.extend(self.other_args) 

296 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config) 

297 self.assertNotEqual(result.resultList, [], 'resultList should not be empty') 

298 self.assertEqual(result.resultList[0].exitStatus, 0) 

299 job = result.resultList[0].result.job 

300 self._test_metrics(job.measurements, metrics) 

301 

302 return result 

303 

304 def _plotJointcalTask(self, data_refs, oldWcsList): 

305 """ 

306 Plot the results of a jointcal run. 

307 

308 Parameters 

309 ---------- 

310 data_refs : `list` [`lsst.daf.persistence.ButlerDataRef`] 

311 The dataRefs that were processed. 

312 oldWcsList : `list` [`lsst.afw.image.SkyWcs`] 

313 The original WCS from each dataRef. 

314 """ 

315 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots') 

316 if not os.path.isdir(plot_dir): 

317 os.mkdir(plot_dir) 

318 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir) 

319 print("Plots saved to: {}".format(plot_dir)) 

320 

321 def _test_metrics(self, result, expect): 

322 """Test a dictionary of "metrics" against those returned by jointcal.py 

323 

324 Parameters 

325 ---------- 

326 result : `dict` 

327 Result metric dictionary from jointcal.py 

328 expect : `dict` 

329 Expected metric dictionary; set a value to None to not test it. 

330 """ 

331 for key in result: 

332 with self.subTest(key.metric): 

333 if expect[key.metric] is not None: 

334 value = result[key].quantity.value 

335 if isinstance(value, float): 

336 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

337 else: 

338 self.assertEqual(value, expect[key.metric], msg=key.metric) 

339 

340 def _runPipeline(self, repo, 

341 inputCollections, outputCollection, 

342 configFiles=None, configOptions=None, 

343 registerDatasetTypes=False, whereSuffix=None, 

344 nJobs=1): 

345 """Run a pipeline via the SimplePipelineExecutor. 

346 

347 Parameters 

348 ---------- 

349 repo : `str` 

350 Gen3 Butler repository to read from/write to. 

351 inputCollections : `list` [`str`] 

352 String to use for "-i" input collections (comma delimited). 

353 For example, "refcats,HSC/runs/tests,HSC/calib" 

354 outputCollection : `str` 

355 Name of the output collection to write to. For example, 

356 "HSC/testdata/jointcal" 

357 configFiles : `list` [`str`], optional 

358 List of jointcal config files to use. 

359 configOptions : `dict` [`str`], optional 

360 Individual jointcal config options (field: value) to override. 

361 registerDatasetTypes : bool, optional 

362 Set "--register-dataset-types" when running the pipeline. 

363 whereSuffix : `str`, optional 

364 Additional parameters to the ``where`` pipetask statement. 

365 nJobs : `int`, optional 

366 Number of quanta expected to be run. 

367 

368 Returns 

369 ------- 

370 job : `lsst.verify.Job` 

371 Job containing the metric measurements from this test run. 

372 """ 

373 config = lsst.jointcal.JointcalConfig() 

374 if configFiles: 

375 for file in configFiles: 

376 config.load(file) 

377 if configOptions: 

378 for key, value in configOptions.items(): 

379 setattr(config, key, value) 

380 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where 

381 lsst.daf.butler.cli.cliLog.CliLog.initLog(False) 

382 butler = SimplePipelineExecutor.prep_butler(repo, 

383 inputs=inputCollections, 

384 output=outputCollection, 

385 output_run=outputCollection.replace("all", "jointcal")) 

386 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask, 

387 config=config, 

388 where=where, 

389 butler=butler) 

390 executor.run(register_dataset_types=registerDatasetTypes) 

391 # JobReporter bundles all metrics in the collection into one job. 

392 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run() 

393 # should only ever get one job output in tests, unless specified 

394 self.assertEqual(len(jobs), nJobs) 

395 return list(jobs.values())[0] 

396 

397 def _runGen3Jointcal(self, instrumentClass, instrumentName, 

398 configFiles=None, configOptions=None, whereSuffix=None, 

399 metrics=None, nJobs=1): 

400 """Create a Butler repo and run jointcal on it. 

401 

402 Parameters 

403 ---------- 

404 instrumentClass : `str` 

405 The full module name of the instrument to be registered in the 

406 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

407 instrumentName : `str` 

408 The name of the instrument as it appears in the repo collections. 

409 For example, "HSC". 

410 configFiles : `list` [`str`], optional 

411 List of jointcal config files to use. 

412 configOptions : `dict` [`str`], optional 

413 Individual jointcal config options (field: value) to override. 

414 whereSuffix : `str`, optional 

415 Additional parameters to the ``where`` pipetask statement. 

416 metrics : `dict`, optional 

417 Dictionary of 'metricName': value to test jointcal's result.metrics 

418 against. 

419 nJobs : `int`, optional 

420 Number of quanta expected to be run. 

421 

422 Returns 

423 ------- 

424 repopath : `str` 

425 The path to the newly created butler repo. 

426 """ 

427 repopath = importRepository(instrumentClass, 

428 os.path.join(self.input_dir, "repo"), 

429 os.path.join(self.input_dir, "exports.yaml"), 

430 self.output_dir, 

431 refcats=self.refcats, 

432 refcatPath=self.refcatPath) 

433 configs = [os.path.join(self.path, "config/config-gen3.py")] 

434 configs.extend(self.configfiles or []) 

435 configs.extend(configFiles or []) 

436 job = self._runPipeline(repopath, 

437 self.inputCollections, 

438 f"{instrumentName}/tests/all", 

439 configFiles=configs, 

440 configOptions=configOptions, 

441 registerDatasetTypes=True, 

442 whereSuffix=whereSuffix, 

443 nJobs=nJobs) 

444 

445 if metrics: 

446 self._test_metrics(job.measurements, metrics) 

447 

448 return repopath