Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import copy 

23import os 

24import shutil 

25 

26import click.testing 

27 

28import lsst.afw.image.utils 

29import lsst.ctrl.mpexec.cli.pipetask 

30import lsst.daf.butler 

31import lsst.obs.base 

32import lsst.geom 

33 

34from lsst.jointcal import jointcal, utils 

35 

36 

37class JointcalTestBase: 

38 """ 

39 Base class for jointcal tests, to genericize some test running and setup. 

40 

41 Derive from this first, then from TestCase. 

42 """ 

43 

44 def set_output_dir(self): 

45 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

46 

47 def setUp_base(self, center, radius, 

48 match_radius=0.1*lsst.geom.arcseconds, 

49 input_dir="", 

50 all_visits=None, 

51 other_args=None, 

52 do_plot=False, 

53 log_level=None): 

54 """ 

55 Call from your child classes's setUp() to get the necessary variables built. 

56 

57 Parameters 

58 ---------- 

59 center : lsst.geom.SpherePoint 

60 Center of the reference catalog. 

61 radius : lsst.geom.Angle 

62 Radius from center to load reference catalog objects inside. 

63 match_radius : lsst.geom.Angle 

64 matching radius when calculating RMS of result. 

65 input_dir : str 

66 Directory of input butler repository. 

67 all_visits : list 

68 List of the available visits to generate the parseAndRun arguments. 

69 other_args : list 

70 Optional other arguments for the butler dataId. 

71 do_plot : bool 

72 Set to True for a comparison plot and some diagnostic numbers. 

73 log_level : str 

74 Set to the default log level you want jointcal to produce while the 

75 tests are running. See the developer docs about logging for valid 

76 levels: https://developer.lsst.io/coding/logging.html 

77 """ 

78 self.center = center 

79 self.radius = radius 

80 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True) 

81 self.input_dir = input_dir 

82 self.all_visits = all_visits 

83 if other_args is None: 

84 other_args = [] 

85 self.other_args = other_args 

86 self.do_plot = do_plot 

87 self.log_level = log_level 

88 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match. 

89 # 100 is a balance between good centroids and enough sources. 

90 self.flux_limit = 100 

91 

92 # Individual tests may want to tweak the config that is passed to parseAndRun(). 

93 self.config = None 

94 self.configfiles = [] 

95 

96 # Append `msg` arguments to assert failures. 

97 self.longMessage = True 

98 

99 # Ensure that the filter list is reset for each test so that we avoid 

100 # confusion or contamination from other instruments. 

101 lsst.obs.base.FilterDefinitionCollection.reset() 

102 

103 self.set_output_dir() 

104 

105 def tearDown(self): 

106 shutil.rmtree(self.output_dir, ignore_errors=True) 

107 

108 if getattr(self, 'reference', None) is not None: 

109 del self.reference 

110 if getattr(self, 'oldWcsList', None) is not None: 

111 del self.oldWcsList 

112 if getattr(self, 'jointcalTask', None) is not None: 

113 del self.jointcalTask 

114 if getattr(self, 'jointcalStatistics', None) is not None: 

115 del self.jointcalStatistics 

116 if getattr(self, 'config', None) is not None: 

117 del self.config 

118 

119 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1, 

120 metrics=None): 

121 """ 

122 Test parseAndRun for jointcal on nCatalogs. 

123 

124 Checks relative and absolute astrometric error (arcsec) and photometric 

125 repeatability (PA1 from the SRD). 

126 

127 Parameters 

128 ---------- 

129 nCatalogs : int 

130 Number of catalogs to run jointcal on. Used to construct the "id" 

131 field for parseAndRun. 

132 dist_rms_relative : astropy.Quantity 

133 Minimum relative astrometric rms post-jointcal to pass the test. 

134 dist_rms_absolute : astropy.Quantity 

135 Minimum absolute astrometric rms post-jointcal to pass the test. 

136 pa1 : float 

137 Minimum PA1 (from Table 14 of the Science Requirements Document: 

138 https://ls.st/LPM-17) post-jointcal to pass the test. 

139 metrics : dict, optional 

140 Dictionary of 'metricName': value to test jointcal's result.metrics 

141 against. 

142 

143 Returns 

144 ------- 

145 list of lsst.daf.persistence.ButlerDataRef 

146 The dataRefs that were processed. 

147 """ 

148 

149 resultFull = self._runJointcalTask(nCatalogs, metrics=metrics) 

150 result = resultFull.resultList[0].result # shorten this very long thing 

151 

152 def compute_statistics(refObjLoader): 

153 refCat = refObjLoader.loadSkyCircle(self.center, 

154 self.radius, 

155 result.defaultBand, 

156 epoch=result.epoch).refCat 

157 rms_result = self.jointcalStatistics.compute_rms(result.dataRefs, refCat) 

158 # Make plots before testing, if requested, so we still get plots if tests fail. 

159 if self.do_plot: 

160 self._plotJointcalTask(result.dataRefs, result.oldWcsList) 

161 return rms_result 

162 

163 # we now have different astrometry/photometry refcats, so have to 

164 # do these calculations separately 

165 if self.jointcalStatistics.do_astrometry: 

166 refObjLoader = result.astrometryRefObjLoader 

167 # preserve do_photometry for the next `if` 

168 temp = copy.copy(self.jointcalStatistics.do_photometry) 

169 self.jointcalStatistics.do_photometry = False 

170 rms_result = compute_statistics(refObjLoader) 

171 self.jointcalStatistics.do_photometry = temp # restore do_photometry 

172 

173 if dist_rms_relative is not None and dist_rms_absolute is not None: 

174 self.assertLess(rms_result.dist_relative, dist_rms_relative) 

175 self.assertLess(rms_result.dist_absolute, dist_rms_absolute) 

176 

177 if self.jointcalStatistics.do_photometry: 

178 refObjLoader = result.photometryRefObjLoader 

179 self.jointcalStatistics.do_astrometry = False 

180 rms_result = compute_statistics(refObjLoader) 

181 

182 if pa1 is not None: 

183 self.assertLess(rms_result.pa1, pa1) 

184 

185 return result.dataRefs 

186 

187 def _runJointcalTask(self, nCatalogs, metrics=None): 

188 """ 

189 Run jointcalTask on nCatalogs, with the most basic tests. 

190 Tests for non-empty result list, and that the basic metrics are correct. 

191 

192 Parameters 

193 ---------- 

194 nCatalogs : int 

195 Number of catalogs to test on. 

196 metrics : dict, optional 

197 Dictionary of 'metricName': value to test jointcal's result.metrics 

198 against. 

199 

200 Returns 

201 ------- 

202 pipe.base.Struct 

203 The structure returned by jointcalTask.run() 

204 """ 

205 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs]) 

206 if self.log_level is not None: 

207 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level]) 

208 

209 # Place default configfile first so that specific subclass configfiles are applied after 

210 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py') 

211 self.configfiles = [test_config] + self.configfiles 

212 

213 args = [self.input_dir, '--output', self.output_dir, 

214 '--clobber-versions', '--clobber-config', 

215 '--doraise', '--configfile', *self.configfiles, 

216 '--id', 'visit=%s'%visits] 

217 args.extend(self.other_args) 

218 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config) 

219 self.assertNotEqual(result.resultList, [], 'resultList should not be empty') 

220 self.assertEqual(result.resultList[0].exitStatus, 0) 

221 job = result.resultList[0].result.job 

222 self._test_metrics(job.measurements, metrics) 

223 

224 return result 

225 

226 def _plotJointcalTask(self, data_refs, oldWcsList): 

227 """ 

228 Plot the results of a jointcal run. 

229 

230 Parameters 

231 ---------- 

232 data_refs : list of lsst.daf.persistence.ButlerDataRef 

233 The dataRefs that were processed. 

234 oldWcsList : list of lsst.afw.image.Wcs 

235 The original WCS from each dataRef. 

236 """ 

237 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots') 

238 if not os.path.isdir(plot_dir): 

239 os.mkdir(plot_dir) 

240 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir) 

241 print("Plots saved to: {}".format(plot_dir)) 

242 

243 def _test_metrics(self, result, expect): 

244 """Test a dictionary of "metrics" against those returned by jointcal.py 

245 

246 Parameters 

247 ---------- 

248 result : dict 

249 Result metric dictionary from jointcal.py 

250 expect : dict 

251 Expected metric dictionary; set a value to None to not test it. 

252 """ 

253 for key in result: 

254 if expect[key.metric] is not None: 

255 value = result[key].quantity.value 

256 if isinstance(value, float): 

257 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

258 else: 

259 self.assertEqual(value, expect[key.metric], msg=key.metric) 

260 

261 def _importRepository(self, instrument, exportPath, exportFile): 

262 """Import a gen3 test repository into self.testDir 

263 

264 Parameters 

265 ---------- 

266 instrument : `str` 

267 Full string name for the instrument. 

268 exportPath : `str` 

269 Path to location of repository to export. 

270 This path must contain an `exports.yaml` file containing the 

271 description of the exported gen3 repo that will be imported. 

272 exportFile : `str` 

273 Filename of export data. 

274 """ 

275 self.repo = os.path.join(self.output_dir, 'testrepo') 

276 

277 # Make the repo and retrieve a writeable Butler 

278 _ = lsst.daf.butler.Butler.makeRepo(self.repo) 

279 butler = lsst.daf.butler.Butler(self.repo, writeable=True) 

280 # Register the instrument 

281 instrInstance = lsst.obs.base.utils.getInstrument(instrument) 

282 instrInstance.register(butler.registry) 

283 # Import the exportFile 

284 butler.import_(directory=exportPath, filename=exportFile, 

285 transfer='symlink', 

286 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

287 

288 def _runPipeline(self, repo, queryString=None, 

289 inputCollections=None, outputCollection=None, 

290 configFiles=None, configOptions=None, 

291 registerDatasetTypes=False): 

292 """Run a pipeline via pipetask. 

293 

294 Parameters 

295 ---------- 

296 repo : `str` 

297 Gen3 Butler repository to read from/write to. 

298 queryString : `str`, optional 

299 String to use for "-d" data query. For example, 

300 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'" 

301 inputCollections : `str`, optional 

302 String to use for "-i" input collections (comma delimited). 

303 For example, "refcats,HSC/runs/tests,HSC/calib" 

304 outputCollection : `str`, optional 

305 String to use for "-o" output collection. For example, 

306 "HSC/testdata/jointcal" 

307 configFiles : `list` [`str`], optional 

308 List of config files to use (with "-C"). 

309 registerDatasetTypes : bool, optional 

310 Set "--register-dataset-types" when running the pipeline. 

311 

312 Returns 

313 ------- 

314 exit_code: `int` 

315 The exit code of the executed `click.testing.CliRunner` 

316 

317 Raises 

318 ------ 

319 RuntimeError 

320 Raised if the CliRunner executed pipetask failed. 

321 

322 Notes 

323 ----- 

324 This approach uses the Click commandline interface, which is not ideal 

325 for tests like this. See DM-26239 for the replacement pipetask API, and 

326 DM-27940 for the ticket to replace the CliRunner once that API exists. 

327 """ 

328 pipelineArgs = ["run", 

329 "-b", repo, 

330 "-t lsst.jointcal.JointcalTask"] 

331 

332 if queryString is not None: 

333 pipelineArgs.extend(["-d", queryString]) 

334 if inputCollections is not None: 

335 pipelineArgs.extend(["-i", inputCollections]) 

336 if outputCollection is not None: 

337 pipelineArgs.extend(["-o", outputCollection]) 

338 if configFiles is not None: 

339 for configFile in configFiles: 

340 pipelineArgs.extend(["-C", configFile]) 

341 if registerDatasetTypes: 

342 pipelineArgs.extend(["--register-dataset-types"]) 

343 

344 # TODO DM-27940: replace CliRunner with pipetask API. 

345 runner = click.testing.CliRunner() 

346 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

347 if results.exception: 

348 raise RuntimeError("Pipeline %s failed." % (' '.join(pipelineArgs))) from results.exception 

349 return results.exit_code 

350 

351 def _runGen3Jointcal(self, instrumentClass, instrumentName, queryString): 

352 """Create a Butler repo and run jointcal on it. 

353 

354 

355 Parameters 

356 ---------- 

357 instrumentClass : `str` 

358 The full module name of the instrument to be registered in the 

359 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

360 instrumentName : `str` 

361 The name of the instrument as it appears in the repo collections. 

362 For example, "HSC". 

363 queryString : `str` 

364 The query string to be run for processing. For example, 

365 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'". 

366 """ 

367 self._importRepository(instrumentClass, 

368 self.input_dir, 

369 os.path.join(self.input_dir, "exports.yaml")) 

370 # TODO post-RFC-741: the names of these collections will have to change 

371 # once testdata_jointcal is updated to reflect the collection 

372 # conventions in RFC-741 (no ticket for that change yet). 

373 inputCollections = f"refcats,{instrumentName}/testdata,{instrumentName}/calib/unbounded" 

374 self._runPipeline(self.repo, 

375 outputCollection=f"{instrumentName}/testdata/jointcal", 

376 inputCollections=inputCollections, 

377 queryString=queryString, 

378 registerDatasetTypes=True)