Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import copy 

23import os 

24import shutil 

25 

26import click.testing 

27 

28import lsst.afw.image.utils 

29import lsst.ctrl.mpexec.cli.pipetask 

30import lsst.daf.butler 

31import lsst.obs.base 

32import lsst.geom 

33 

34from lsst.jointcal import jointcal, utils 

35 

36 

37class JointcalTestBase: 

38 """ 

39 Base class for jointcal tests, to genericize some test running and setup. 

40 

41 Derive from this first, then from TestCase. 

42 """ 

43 

44 def set_output_dir(self): 

45 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

46 

47 def setUp_base(self, center, radius, 

48 match_radius=0.1*lsst.geom.arcseconds, 

49 input_dir="", 

50 all_visits=None, 

51 other_args=None, 

52 do_plot=False, 

53 log_level=None): 

54 """ 

55 Call from your child classes's setUp() to get the necessary variables built. 

56 

57 Parameters 

58 ---------- 

59 center : lsst.geom.SpherePoint 

60 Center of the reference catalog. 

61 radius : lsst.geom.Angle 

62 Radius from center to load reference catalog objects inside. 

63 match_radius : lsst.geom.Angle 

64 matching radius when calculating RMS of result. 

65 input_dir : str 

66 Directory of input butler repository. 

67 all_visits : list 

68 List of the available visits to generate the parseAndRun arguments. 

69 other_args : list 

70 Optional other arguments for the butler dataId. 

71 do_plot : bool 

72 Set to True for a comparison plot and some diagnostic numbers. 

73 log_level : str 

74 Set to the default log level you want jointcal to produce while the 

75 tests are running. See the developer docs about logging for valid 

76 levels: https://developer.lsst.io/coding/logging.html 

77 """ 

78 self.center = center 

79 self.radius = radius 

80 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True) 

81 self.input_dir = input_dir 

82 self.all_visits = all_visits 

83 if other_args is None: 

84 other_args = [] 

85 self.other_args = other_args 

86 self.do_plot = do_plot 

87 self.log_level = log_level 

88 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match. 

89 # 100 is a balance between good centroids and enough sources. 

90 self.flux_limit = 100 

91 

92 # Individual tests may want to tweak the config that is passed to parseAndRun(). 

93 self.config = None 

94 self.configfiles = [] 

95 

96 # Append `msg` arguments to assert failures. 

97 self.longMessage = True 

98 

99 # Ensure that the filter list is reset for each test so that we avoid 

100 # confusion or contamination from other instruments. 

101 lsst.obs.base.FilterDefinitionCollection.reset() 

102 

103 self.set_output_dir() 

104 

105 def tearDown(self): 

106 shutil.rmtree(self.output_dir, ignore_errors=True) 

107 

108 if getattr(self, 'reference', None) is not None: 

109 del self.reference 

110 if getattr(self, 'oldWcsList', None) is not None: 

111 del self.oldWcsList 

112 if getattr(self, 'jointcalTask', None) is not None: 

113 del self.jointcalTask 

114 if getattr(self, 'jointcalStatistics', None) is not None: 

115 del self.jointcalStatistics 

116 if getattr(self, 'config', None) is not None: 

117 del self.config 

118 

119 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1, 

120 metrics=None): 

121 """ 

122 Test parseAndRun for jointcal on nCatalogs. 

123 

124 Checks relative and absolute astrometric error (arcsec) and photometric 

125 repeatability (PA1 from the SRD). 

126 

127 Parameters 

128 ---------- 

129 nCatalogs : int 

130 Number of catalogs to run jointcal on. Used to construct the "id" 

131 field for parseAndRun. 

132 dist_rms_relative : astropy.Quantity 

133 Minimum relative astrometric rms post-jointcal to pass the test. 

134 dist_rms_absolute : astropy.Quantity 

135 Minimum absolute astrometric rms post-jointcal to pass the test. 

136 pa1 : float 

137 Minimum PA1 (from Table 14 of the Science Requirements Document: 

138 https://ls.st/LPM-17) post-jointcal to pass the test. 

139 metrics : dict, optional 

140 Dictionary of 'metricName': value to test jointcal's result.metrics 

141 against. 

142 

143 Returns 

144 ------- 

145 list of lsst.daf.persistence.ButlerDataRef 

146 The dataRefs that were processed. 

147 """ 

148 

149 result = self._runJointcalTask(nCatalogs, metrics=metrics) 

150 

151 data_refs = result.resultList[0].result.dataRefs 

152 oldWcsList = result.resultList[0].result.oldWcsList 

153 

154 defaultFilter = result.resultList[0].result.defaultFilter 

155 

156 def compute_statistics(refObjLoader): 

157 refCat = refObjLoader.loadSkyCircle(self.center, self.radius, defaultFilter).refCat 

158 rms_result = self.jointcalStatistics.compute_rms(data_refs, refCat) 

159 # Make plots before testing, if requested, so we still get plots if tests fail. 

160 if self.do_plot: 

161 self._plotJointcalTask(data_refs, oldWcsList) 

162 return rms_result 

163 

164 # we now have different astrometry/photometry refcats, so have to 

165 # do these calculations separately 

166 if self.jointcalStatistics.do_astrometry: 

167 refObjLoader = result.resultList[0].result.astrometryRefObjLoader 

168 # preserve do_photometry for the next `if` 

169 temp = copy.copy(self.jointcalStatistics.do_photometry) 

170 self.jointcalStatistics.do_photometry = False 

171 rms_result = compute_statistics(refObjLoader) 

172 self.jointcalStatistics.do_photometry = temp # restore do_photometry 

173 

174 if dist_rms_relative is not None and dist_rms_absolute is not None: 

175 self.assertLess(rms_result.dist_relative, dist_rms_relative) 

176 self.assertLess(rms_result.dist_absolute, dist_rms_absolute) 

177 

178 if self.jointcalStatistics.do_photometry: 

179 refObjLoader = result.resultList[0].result.photometryRefObjLoader 

180 self.jointcalStatistics.do_astrometry = False 

181 rms_result = compute_statistics(refObjLoader) 

182 

183 if pa1 is not None: 

184 self.assertLess(rms_result.pa1, pa1) 

185 

186 return data_refs 

187 

188 def _runJointcalTask(self, nCatalogs, metrics=None): 

189 """ 

190 Run jointcalTask on nCatalogs, with the most basic tests. 

191 Tests for non-empty result list, and that the basic metrics are correct. 

192 

193 Parameters 

194 ---------- 

195 nCatalogs : int 

196 Number of catalogs to test on. 

197 metrics : dict, optional 

198 Dictionary of 'metricName': value to test jointcal's result.metrics 

199 against. 

200 

201 Returns 

202 ------- 

203 pipe.base.Struct 

204 The structure returned by jointcalTask.run() 

205 """ 

206 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs]) 

207 if self.log_level is not None: 

208 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level]) 

209 

210 # Place default configfile first so that specific subclass configfiles are applied after 

211 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py') 

212 self.configfiles = [test_config] + self.configfiles 

213 

214 args = [self.input_dir, '--output', self.output_dir, 

215 '--clobber-versions', '--clobber-config', 

216 '--doraise', '--configfile', *self.configfiles, 

217 '--id', 'visit=%s'%visits] 

218 args.extend(self.other_args) 

219 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config) 

220 self.assertNotEqual(result.resultList, [], 'resultList should not be empty') 

221 self.assertEqual(result.resultList[0].exitStatus, 0) 

222 job = result.resultList[0].result.job 

223 self._test_metrics(job.measurements, metrics) 

224 

225 return result 

226 

227 def _plotJointcalTask(self, data_refs, oldWcsList): 

228 """ 

229 Plot the results of a jointcal run. 

230 

231 Parameters 

232 ---------- 

233 data_refs : list of lsst.daf.persistence.ButlerDataRef 

234 The dataRefs that were processed. 

235 oldWcsList : list of lsst.afw.image.Wcs 

236 The original WCS from each dataRef. 

237 """ 

238 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots') 

239 if not os.path.isdir(plot_dir): 

240 os.mkdir(plot_dir) 

241 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir) 

242 print("Plots saved to: {}".format(plot_dir)) 

243 

244 def _test_metrics(self, result, expect): 

245 """Test a dictionary of "metrics" against those returned by jointcal.py 

246 

247 Parameters 

248 ---------- 

249 result : dict 

250 Result metric dictionary from jointcal.py 

251 expect : dict 

252 Expected metric dictionary; set a value to None to not test it. 

253 """ 

254 for key in result: 

255 if expect[key.metric] is not None: 

256 value = result[key].quantity.value 

257 if isinstance(value, float): 

258 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

259 else: 

260 self.assertEqual(value, expect[key.metric], msg=key.metric) 

261 

262 def _importRepository(self, instrument, exportPath, exportFile): 

263 """Import a gen3 test repository into self.testDir 

264 

265 Parameters 

266 ---------- 

267 instrument : `str` 

268 Full string name for the instrument. 

269 exportPath : `str` 

270 Path to location of repository to export. 

271 This path must contain an `exports.yaml` file containing the 

272 description of the exported gen3 repo that will be imported. 

273 exportFile : `str` 

274 Filename of export data. 

275 """ 

276 self.repo = os.path.join(self.output_dir, 'testrepo') 

277 

278 # Make the repo and retrieve a writeable Butler 

279 _ = lsst.daf.butler.Butler.makeRepo(self.repo) 

280 butler = lsst.daf.butler.Butler(self.repo, writeable=True) 

281 # Register the instrument 

282 instrInstance = lsst.obs.base.utils.getInstrument(instrument) 

283 instrInstance.register(butler.registry) 

284 # Import the exportFile 

285 butler.import_(directory=exportPath, filename=exportFile, 

286 transfer='symlink', 

287 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

288 

289 def _runPipeline(self, repo, queryString=None, 

290 inputCollections=None, outputCollection=None, 

291 configFiles=None, configOptions=None, 

292 registerDatasetTypes=False): 

293 """Run a pipeline via pipetask. 

294 

295 Parameters 

296 ---------- 

297 repo : `str` 

298 Gen3 Butler repository to read from/write to. 

299 queryString : `str`, optional 

300 String to use for "-d" data query. For example, 

301 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'" 

302 inputCollections : `str`, optional 

303 String to use for "-i" input collections (comma delimited). 

304 For example, "refcats,HSC/runs/tests,HSC/calib" 

305 outputCollection : `str`, optional 

306 String to use for "-o" output collection. For example, 

307 "HSC/testdata/jointcal" 

308 configFiles : `list` [`str`], optional 

309 List of config files to use (with "-C"). 

310 registerDatasetTypes : bool, optional 

311 Set "--register-dataset-types" when running the pipeline. 

312 

313 Returns 

314 ------- 

315 exit_code: `int` 

316 The exit code of the executed `click.testing.CliRunner` 

317 

318 Raises 

319 ------ 

320 RuntimeError 

321 Raised if the CliRunner executed pipetask failed. 

322 

323 Notes 

324 ----- 

325 This approach uses the Click commandline interface, which is not ideal 

326 for tests like this. See DM-26239 for the replacement pipetask API, and 

327 DM-27940 for the ticket to replace the CliRunner once that API exists. 

328 """ 

329 pipelineArgs = ["run", 

330 "-b", repo, 

331 "-t lsst.jointcal.JointcalTask"] 

332 

333 if queryString is not None: 

334 pipelineArgs.extend(["-d", queryString]) 

335 if inputCollections is not None: 

336 pipelineArgs.extend(["-i", inputCollections]) 

337 if outputCollection is not None: 

338 pipelineArgs.extend(["-o", outputCollection]) 

339 if configFiles is not None: 

340 for configFile in configFiles: 

341 pipelineArgs.extend(["-C", configFile]) 

342 if registerDatasetTypes: 

343 pipelineArgs.extend(["--register-dataset-types"]) 

344 

345 # TODO DM-27940: replace CliRunner with pipetask API. 

346 runner = click.testing.CliRunner() 

347 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

348 if results.exception: 

349 raise RuntimeError("Pipeline %s failed." % (' '.join(pipelineArgs))) from results.exception 

350 return results.exit_code 

351 

352 def _runGen3Jointcal(self, instrumentClass, instrumentName, queryString): 

353 """Create a Butler repo and run jointcal on it. 

354 

355 

356 Parameters 

357 ---------- 

358 instrumentClass : `str` 

359 The full module name of the instrument to be registered in the 

360 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

361 instrumentName : `str` 

362 The name of the instrument as it appears in the repo collections. 

363 For example, "HSC". 

364 queryString : `str` 

365 The query string to be run for processing. For example, 

366 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'". 

367 """ 

368 self._importRepository(instrumentClass, 

369 self.input_dir, 

370 os.path.join(self.input_dir, "exports.yaml")) 

371 # TODO post-RFC-741: the names of these collections will have to change 

372 # once testdata_jointcal is updated to reflect the collection 

373 # conventions in RFC-741 (no ticket for that change yet). 

374 inputCollections = f"refcats,{instrumentName}/testdata,{instrumentName}/calib/unbounded" 

375 self._runPipeline(self.repo, 

376 outputCollection=f"{instrumentName}/testdata/jointcal", 

377 inputCollections=inputCollections, 

378 queryString=queryString, 

379 registerDatasetTypes=True)