Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import copy 

23import os 

24import shutil 

25 

26import click.testing 

27 

28import lsst.afw.image.utils 

29import lsst.ctrl.mpexec.cli.pipetask 

30import lsst.daf.butler 

31import lsst.obs.base 

32import lsst.geom 

33 

34from lsst.jointcal import jointcal, utils 

35 

36 

37class JointcalTestBase: 

38 """ 

39 Base class for jointcal tests, to genericize some test running and setup. 

40 

41 Derive from this first, then from TestCase. 

42 """ 

43 

44 def set_output_dir(self): 

45 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

46 

47 def setUp_base(self, center, radius, 

48 match_radius=0.1*lsst.geom.arcseconds, 

49 input_dir="", 

50 all_visits=None, 

51 other_args=None, 

52 do_plot=False, 

53 log_level=None): 

54 """ 

55 Call from your child classes's setUp() to get the necessary variables built. 

56 

57 Parameters 

58 ---------- 

59 center : `lsst.geom.SpherePoint` 

60 Center of the reference catalog. 

61 radius : `lsst.geom.Angle` 

62 Radius from center to load reference catalog objects inside. 

63 match_radius : `lsst.geom.Angle` 

64 matching radius when calculating RMS of result. 

65 input_dir : `str` 

66 Directory of input butler repository. 

67 all_visits : `list` [`int`] 

68 List of the available visits to generate the parseAndRun arguments. 

69 other_args : `list` [`str`] 

70 Optional other arguments for the butler dataId. 

71 do_plot : `bool` 

72 Set to True for a comparison plot and some diagnostic numbers. 

73 log_level : `str` 

74 Set to the default log level you want jointcal to produce while the 

75 tests are running. See the developer docs about logging for valid 

76 levels: https://developer.lsst.io/coding/logging.html 

77 """ 

78 self.path = os.path.dirname(__file__) 

79 

80 self.center = center 

81 self.radius = radius 

82 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True) 

83 self.input_dir = input_dir 

84 self.all_visits = all_visits 

85 if other_args is None: 

86 other_args = [] 

87 self.other_args = other_args 

88 self.do_plot = do_plot 

89 self.log_level = log_level 

90 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match. 

91 # 100 is a balance between good centroids and enough sources. 

92 self.flux_limit = 100 

93 

94 # Individual tests may want to tweak the config that is passed to parseAndRun(). 

95 self.config = None 

96 self.configfiles = [] 

97 

98 # Append `msg` arguments to assert failures. 

99 self.longMessage = True 

100 

101 # Ensure that the filter list is reset for each test so that we avoid 

102 # confusion or contamination from other instruments. 

103 lsst.obs.base.FilterDefinitionCollection.reset() 

104 

105 self.set_output_dir() 

106 

107 def tearDown(self): 

108 shutil.rmtree(self.output_dir, ignore_errors=True) 

109 

110 if getattr(self, 'reference', None) is not None: 

111 del self.reference 

112 if getattr(self, 'oldWcsList', None) is not None: 

113 del self.oldWcsList 

114 if getattr(self, 'jointcalTask', None) is not None: 

115 del self.jointcalTask 

116 if getattr(self, 'jointcalStatistics', None) is not None: 

117 del self.jointcalStatistics 

118 if getattr(self, 'config', None) is not None: 

119 del self.config 

120 

121 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1, 

122 metrics=None): 

123 """ 

124 Test parseAndRun for jointcal on nCatalogs. 

125 

126 Checks relative and absolute astrometric error (arcsec) and photometric 

127 repeatability (PA1 from the SRD). 

128 

129 Parameters 

130 ---------- 

131 nCatalogs : `int` 

132 Number of catalogs to run jointcal on. Used to construct the "id" 

133 field for parseAndRun. 

134 dist_rms_relative : `astropy.Quantity` 

135 Minimum relative astrometric rms post-jointcal to pass the test. 

136 dist_rms_absolute : `astropy.Quantity` 

137 Minimum absolute astrometric rms post-jointcal to pass the test. 

138 pa1 : `float` 

139 Minimum PA1 (from Table 14 of the Science Requirements Document: 

140 https://ls.st/LPM-17) post-jointcal to pass the test. 

141 metrics : `dict`, optional 

142 Dictionary of 'metricName': value to test jointcal's result.metrics 

143 against. 

144 

145 Returns 

146 ------- 

147 dataRefs : `list` [`lsst.daf.persistence.ButlerDataRef`] 

148 The dataRefs that were processed. 

149 """ 

150 

151 resultFull = self._runJointcalTask(nCatalogs, metrics=metrics) 

152 result = resultFull.resultList[0].result # shorten this very long thing 

153 

154 def compute_statistics(refObjLoader): 

155 refCat = refObjLoader.loadSkyCircle(self.center, 

156 self.radius, 

157 result.defaultFilter.bandLabel, 

158 epoch=result.epoch).refCat 

159 rms_result = self.jointcalStatistics.compute_rms(result.dataRefs, refCat) 

160 # Make plots before testing, if requested, so we still get plots if tests fail. 

161 if self.do_plot: 

162 self._plotJointcalTask(result.dataRefs, result.oldWcsList) 

163 return rms_result 

164 

165 # we now have different astrometry/photometry refcats, so have to 

166 # do these calculations separately 

167 if self.jointcalStatistics.do_astrometry: 

168 refObjLoader = result.astrometryRefObjLoader 

169 # preserve do_photometry for the next `if` 

170 temp = copy.copy(self.jointcalStatistics.do_photometry) 

171 self.jointcalStatistics.do_photometry = False 

172 rms_result = compute_statistics(refObjLoader) 

173 self.jointcalStatistics.do_photometry = temp # restore do_photometry 

174 

175 if dist_rms_relative is not None and dist_rms_absolute is not None: 

176 self.assertLess(rms_result.dist_relative, dist_rms_relative) 

177 self.assertLess(rms_result.dist_absolute, dist_rms_absolute) 

178 

179 if self.jointcalStatistics.do_photometry: 

180 refObjLoader = result.photometryRefObjLoader 

181 self.jointcalStatistics.do_astrometry = False 

182 rms_result = compute_statistics(refObjLoader) 

183 

184 if pa1 is not None: 

185 self.assertLess(rms_result.pa1, pa1) 

186 

187 return result.dataRefs 

188 

189 def _runJointcalTask(self, nCatalogs, metrics=None): 

190 """ 

191 Run jointcalTask on nCatalogs, with the most basic tests. 

192 Tests for non-empty result list, and that the basic metrics are correct. 

193 

194 Parameters 

195 ---------- 

196 nCatalogs : `int` 

197 Number of catalogs to test on. 

198 metrics : `dict`, optional 

199 Dictionary of 'metricName': value to test jointcal's result.metrics 

200 against. 

201 

202 Returns 

203 ------- 

204 result : `pipe.base.Struct` 

205 The structure returned by jointcalTask.run() 

206 """ 

207 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs]) 

208 if self.log_level is not None: 

209 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level]) 

210 

211 # Place default configfile first so that specific subclass configfiles are applied after 

212 test_config = os.path.join(self.path, 'config/config.py') 

213 configfiles = [test_config] + self.configfiles 

214 

215 args = [self.input_dir, '--output', self.output_dir, 

216 '--clobber-versions', '--clobber-config', 

217 '--doraise', '--configfile', *configfiles, 

218 '--id', 'visit=%s'%visits] 

219 args.extend(self.other_args) 

220 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config) 

221 self.assertNotEqual(result.resultList, [], 'resultList should not be empty') 

222 self.assertEqual(result.resultList[0].exitStatus, 0) 

223 job = result.resultList[0].result.job 

224 self._test_metrics(job.measurements, metrics) 

225 

226 return result 

227 

228 def _plotJointcalTask(self, data_refs, oldWcsList): 

229 """ 

230 Plot the results of a jointcal run. 

231 

232 Parameters 

233 ---------- 

234 data_refs : `list` [`lsst.daf.persistence.ButlerDataRef`] 

235 The dataRefs that were processed. 

236 oldWcsList : `list` [`lsst.afw.image.SkyWcs`] 

237 The original WCS from each dataRef. 

238 """ 

239 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots') 

240 if not os.path.isdir(plot_dir): 

241 os.mkdir(plot_dir) 

242 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir) 

243 print("Plots saved to: {}".format(plot_dir)) 

244 

245 def _test_metrics(self, result, expect): 

246 """Test a dictionary of "metrics" against those returned by jointcal.py 

247 

248 Parameters 

249 ---------- 

250 result : `dict` 

251 Result metric dictionary from jointcal.py 

252 expect : `dict` 

253 Expected metric dictionary; set a value to None to not test it. 

254 """ 

255 for key in result: 

256 if expect[key.metric] is not None: 

257 value = result[key].quantity.value 

258 if isinstance(value, float): 

259 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

260 else: 

261 self.assertEqual(value, expect[key.metric], msg=key.metric) 

262 

263 def _importRepository(self, instrument, exportPath, exportFile): 

264 """Import a gen3 test repository into self.testDir 

265 

266 Parameters 

267 ---------- 

268 instrument : `str` 

269 Full string name for the instrument. 

270 exportPath : `str` 

271 Path to location of repository to export. 

272 This path must contain an `exports.yaml` file containing the 

273 description of the exported gen3 repo that will be imported. 

274 exportFile : `str` 

275 Filename of export data. 

276 """ 

277 self.repo = os.path.join(self.output_dir, 'testrepo') 

278 

279 # Make the repo and retrieve a writeable Butler 

280 _ = lsst.daf.butler.Butler.makeRepo(self.repo) 

281 butler = lsst.daf.butler.Butler(self.repo, writeable=True) 

282 # Register the instrument 

283 instrInstance = lsst.obs.base.utils.getInstrument(instrument) 

284 instrInstance.register(butler.registry) 

285 # Import the exportFile 

286 butler.import_(directory=exportPath, filename=exportFile, 

287 transfer='symlink', 

288 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

289 

290 def _runPipeline(self, repo, pipelineFile, queryString=None, 

291 inputCollections=None, outputCollection=None, 

292 configFiles=None, configOptions=None, 

293 registerDatasetTypes=False): 

294 """Run a pipeline via pipetask. 

295 

296 Parameters 

297 ---------- 

298 repo : `str` 

299 Gen3 Butler repository to read from/write to. 

300 pipelineFile : `str` 

301 The pipeline definition YAML file to execute. 

302 queryString : `str`, optional 

303 String to use for "-d" data query. For example, 

304 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'" 

305 inputCollections : `str`, optional 

306 String to use for "-i" input collections (comma delimited). 

307 For example, "refcats,HSC/runs/tests,HSC/calib" 

308 outputCollection : `str`, optional 

309 String to use for "-o" output collection. For example, 

310 "HSC/testdata/jointcal" 

311 configFiles : `list` [`str`], optional 

312 List of jointcal config files to use (with "-C"). 

313 configOptions : `list` [`str`], optional 

314 List of individual jointcal config options to use (with "-c"). 

315 registerDatasetTypes : bool, optional 

316 Set "--register-dataset-types" when running the pipeline. 

317 

318 Returns 

319 ------- 

320 exit_code: `int` 

321 The exit code of the executed `click.testing.CliRunner` 

322 

323 Raises 

324 ------ 

325 RuntimeError 

326 Raised if the CliRunner executed pipetask failed. 

327 

328 Notes 

329 ----- 

330 This approach uses the Click commandline interface, which is not ideal 

331 for tests like this. See DM-26239 for the replacement pipetask API, and 

332 DM-27940 for the ticket to replace the CliRunner once that API exists. 

333 """ 

334 pipelineArgs = ["run", 

335 "-b", repo, 

336 "-p", pipelineFile] 

337 

338 if queryString is not None: 

339 pipelineArgs.extend(["-d", queryString]) 

340 if inputCollections is not None: 

341 pipelineArgs.extend(["-i", inputCollections]) 

342 if outputCollection is not None: 

343 # Each test is in it's own butler, so we don't have to worry about collisions, 

344 # so we can use `--output-run` instead of just `-o`. 

345 pipelineArgs.extend(["--output-run", outputCollection]) 

346 if configFiles is not None: 

347 for configFile in configFiles: 

348 pipelineArgs.extend(["-C", f"jointcal:{configFile}"]) 

349 if configOptions is not None: 

350 for configOption in configOptions: 

351 pipelineArgs.extend(["-c", f"jointcal:{configOption}"]) 

352 if registerDatasetTypes: 

353 pipelineArgs.extend(["--register-dataset-types"]) 

354 

355 # TODO DM-27940: replace CliRunner with pipetask API. 

356 runner = click.testing.CliRunner() 

357 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

358 if results.exception: 

359 raise RuntimeError("Pipeline %s failed." % (' '.join(pipelineArgs))) from results.exception 

360 return results.exit_code 

361 

362 def _runGen3Jointcal(self, instrumentClass, instrumentName, queryString, 

363 configFiles=None, configOptions=None): 

364 """Create a Butler repo and run jointcal on it. 

365 

366 Parameters 

367 ---------- 

368 instrumentClass : `str` 

369 The full module name of the instrument to be registered in the 

370 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

371 instrumentName : `str` 

372 The name of the instrument as it appears in the repo collections. 

373 For example, "HSC". 

374 queryString : `str` 

375 The query string to be run for processing. For example, 

376 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'". 

377 configFiles : `list` [`str`], optional 

378 List of jointcal config files to use (with "-C"). 

379 configOptions : `list` [`str`], optional 

380 List of individual jointcal config options to use (with "-c"). 

381 """ 

382 self._importRepository(instrumentClass, 

383 self.input_dir, 

384 os.path.join(self.input_dir, "exports.yaml")) 

385 # TODO post-RFC-741: the names of these collections will have to change 

386 # once testdata_jointcal is updated to reflect the collection 

387 # conventions in RFC-741 (no ticket for that change yet). 

388 inputCollections = f"refcats/gen2,{instrumentName}/testdata,{instrumentName}/calib/unbounded" 

389 

390 pipelineFile = os.path.join(self.path, f"config/jointcalPipeline-{instrumentName}.yaml") 

391 configFiles = [os.path.join(self.path, "config/config-gen3.py")] + self.configfiles 

392 self._runPipeline(self.repo, pipelineFile, 

393 outputCollection=f"{instrumentName}/testdata/jointcal", 

394 inputCollections=inputCollections, 

395 queryString=queryString, 

396 configFiles=configFiles, 

397 configOptions=configOptions, 

398 registerDatasetTypes=True)