Coverage for tests/jointcalTestBase.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

109 statements  

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["importRepository", "JointcalTestBase"] 

23 

24import os 

25import shutil 

26import tempfile 

27import numpy as np 

28 

29import lsst.afw.image.utils 

30from lsst.ctrl.mpexec import SimplePipelineExecutor 

31import lsst.daf.butler 

32from lsst.daf.butler.script import ingest_files 

33import lsst.obs.base 

34import lsst.geom 

35from lsst.verify.bin.jobReporter import JobReporter 

36 

37import lsst.jointcal 

38 

39 

40def importRepository(instrument, exportPath, exportFile, outputDir=None, 

41 refcats=None, refcatPath=""): 

42 """Import a gen3 test repository into a test path. 

43 

44 Parameters 

45 ---------- 

46 instrument : `str` 

47 Full string name for the instrument. 

48 exportPath : `str` 

49 Path to location of repository to export. 

50 exportFile : `str` 

51 Filename of YAML butler export data, describing the data to import. 

52 outputDir : `str`, or `None` 

53 Root path to put the test repo in; appended with `testrepo/`. 

54 If not supplied, a temporary path is generated with mkdtemp; this path 

55 must be deleted by the calling code, it is not automatically removed. 

56 refcats : `dict` [`str`, `str`] 

57 Mapping of refcat name to relative path to .ecsv ingest file. 

58 refcatPath : `str` 

59 Path prefix for refcat ingest files and files linked therein. 

60 

61 Returns 

62 ------- 

63 repopath : `str` 

64 The path to the newly created butler repo. 

65 """ 

66 if outputDir is None: 

67 repopath = tempfile.mkdtemp() 

68 else: 

69 repopath = os.path.join(outputDir, "testrepo") 

70 

71 # Make the repo and retrieve a writeable Butler 

72 _ = lsst.daf.butler.Butler.makeRepo(repopath) 

73 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

74 

75 instrInstance = lsst.obs.base.utils.getInstrument(instrument) 

76 instrInstance.register(butler.registry) 

77 

78 # Register refcats first, so the `refcats` collection will exist. 

79 if refcats is not None: 

80 for name, file in refcats.items(): 

81 graph = butler.registry.dimensions.extract(["htm7"]) 

82 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog", 

83 universe=butler.registry.dimensions) 

84 butler.registry.registerDatasetType(datasetType) 

85 ingest_files(repopath, name, "refcats", file, prefix=refcatPath) 

86 # New butler to get the refcats collections to be visible 

87 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

88 

89 butler.import_(directory=exportPath, filename=exportFile, 

90 transfer='symlink', 

91 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

92 return repopath 

93 

94 

95class JointcalTestBase: 

96 """ 

97 Base class for jointcal tests, to genericize some test running and setup. 

98 

99 Derive from this first, then from TestCase. 

100 """ 

101 

102 def set_output_dir(self): 

103 """Set the output directory to the name of the test method, in .test/ 

104 """ 

105 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

106 

107 def setUp_base(self, 

108 instrumentClass, 

109 instrumentName, 

110 input_dir="", 

111 all_visits=None, 

112 log_level=None, 

113 where="", 

114 refcats=None, 

115 refcatPath="", 

116 inputCollections=None, 

117 outputDataId=None): 

118 """ 

119 Call from your child classes's setUp() to get the necessary variables built. 

120 

121 Parameters 

122 ---------- 

123 instrumentClass : `str` 

124 The full module name of the instrument to be registered in the 

125 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

126 instrumentName : `str` 

127 The name of the instrument as it appears in the repo collections. 

128 For example, "HSC". 

129 input_dir : `str` 

130 Directory of input butler repository. 

131 all_visits : `list` [`int`] 

132 List of the available visits to generate the dataset query from. 

133 log_level : `str` 

134 Set to the default log level you want jointcal to produce while the 

135 tests are running. See the developer docs about logging for valid 

136 levels: https://developer.lsst.io/coding/logging.html 

137 where : `str` 

138 Data ID query for pipetask specifying the data to run on. 

139 refcats : `dict` [`str`, `str`] 

140 Mapping of refcat name to relative path to .ecsv ingest file. 

141 refcatPath : `str` 

142 Path prefix for refcat ingest files and files linked therein. 

143 inputCollections : `list` [`str`] 

144 String to use for "-i" input collections (comma delimited). 

145 For example, "refcats,HSC/runs/tests,HSC/calib" 

146 outputDataId : `dict` 

147 Partial dataIds for testing whether the output files were written. 

148 """ 

149 self.path = os.path.dirname(__file__) 

150 

151 self.instrumentClass = instrumentClass 

152 self.instrumentName = instrumentName 

153 

154 self.input_dir = input_dir 

155 self.all_visits = all_visits 

156 self.log_level = log_level 

157 

158 self.configfiles = [] 

159 

160 # Make unittest output more verbose failure messages to assert failures. 

161 self.longMessage = True 

162 

163 self.where = where 

164 self.refcats = refcats 

165 self.refcatPath = refcatPath 

166 self.inputCollections = inputCollections 

167 

168 self.outputDataId = outputDataId 

169 

170 # Ensure that the filter list is reset for each test so that we avoid 

171 # confusion or contamination from other instruments. 

172 lsst.obs.base.FilterDefinitionCollection.reset() 

173 

174 self.set_output_dir() 

175 

176 def tearDown(self): 

177 shutil.rmtree(self.output_dir, ignore_errors=True) 

178 

179 def _test_metrics(self, result, expect): 

180 """Test a dictionary of "metrics" against those returned by jointcal.py 

181 

182 Parameters 

183 ---------- 

184 result : `dict` 

185 Result metric dictionary from jointcal.py 

186 expect : `dict` 

187 Expected metric dictionary; set a value to None to not test it. 

188 """ 

189 for key in result: 

190 with self.subTest(key.metric): 

191 if expect[key.metric] is not None: 

192 value = result[key].quantity.value 

193 if isinstance(value, float): 

194 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

195 else: 

196 self.assertEqual(value, expect[key.metric], msg=key.metric) 

197 

198 def _runJointcal(self, repo, 

199 inputCollections, outputCollection, 

200 configFiles=None, configOptions=None, 

201 registerDatasetTypes=False, whereSuffix=None, 

202 nJobs=1): 

203 """Run a pipeline via the SimplePipelineExecutor. 

204 

205 Parameters 

206 ---------- 

207 repo : `str` 

208 Gen3 Butler repository to read from/write to. 

209 inputCollections : `list` [`str`] 

210 String to use for "-i" input collections (comma delimited). 

211 For example, "refcats,HSC/runs/tests,HSC/calib" 

212 outputCollection : `str` 

213 Name of the output collection to write to. For example, 

214 "HSC/testdata/jointcal" 

215 configFiles : `list` [`str`], optional 

216 List of jointcal config files to use. 

217 configOptions : `dict` [`str`], optional 

218 Individual jointcal config options (field: value) to override. 

219 registerDatasetTypes : bool, optional 

220 Set "--register-dataset-types" when running the pipeline. 

221 whereSuffix : `str`, optional 

222 Additional parameters to the ``where`` pipetask statement. 

223 nJobs : `int`, optional 

224 Number of quanta expected to be run. 

225 

226 Returns 

227 ------- 

228 job : `lsst.verify.Job` 

229 Job containing the metric measurements from this test run. 

230 """ 

231 config = lsst.jointcal.JointcalConfig() 

232 if configFiles: 

233 for file in configFiles: 

234 config.load(file) 

235 if configOptions: 

236 for key, value in configOptions.items(): 

237 setattr(config, key, value) 

238 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where 

239 lsst.daf.butler.cli.cliLog.CliLog.initLog(False) 

240 butler = SimplePipelineExecutor.prep_butler(repo, 

241 inputs=inputCollections, 

242 output=outputCollection, 

243 output_run=outputCollection.replace("all", "jointcal")) 

244 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask, 

245 config=config, 

246 where=where, 

247 butler=butler) 

248 executor.run(register_dataset_types=registerDatasetTypes) 

249 # JobReporter bundles all metrics in the collection into one job. 

250 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run() 

251 # should only ever get one job output in tests, unless specified 

252 self.assertEqual(len(jobs), nJobs) 

253 # Sort the jobs, as QuantumGraph ordering is not guaranteed, and some 

254 # tests check metrics values for one job while producing two. 

255 sorted_jobs = {key: jobs[key] for key in sorted(list(jobs.keys()))} 

256 return list(sorted_jobs.values())[0] 

257 

258 def _runJointcalTest(self, astrometryOutputs=None, photometryOutputs=None, 

259 configFiles=None, configOptions=None, whereSuffix=None, 

260 metrics=None, nJobs=1): 

261 """Create a Butler repo and run jointcal on it. 

262 

263 Parameters 

264 ---------- 

265 atsrometryOutputs, photometryOutputs : `dict` [`int`, `list`] 

266 visit: [detectors] dictionary to test that output files were saved. 

267 Default None means that there should be no output for that 

268 respective dataset type. 

269 configFiles : `list` [`str`], optional 

270 List of jointcal config files to use. 

271 configOptions : `dict` [`str`], optional 

272 Individual jointcal config options (field: value) to override. 

273 whereSuffix : `str`, optional 

274 Additional parameters to the ``where`` pipetask statement. 

275 metrics : `dict`, optional 

276 Dictionary of 'metricName': value to test jointcal's result.metrics 

277 against. 

278 nJobs : `int`, optional 

279 Number of quanta expected to be run. 

280 

281 Returns 

282 ------- 

283 repopath : `str` 

284 The path to the newly created butler repo. 

285 """ 

286 repopath = importRepository(self.instrumentClass, 

287 os.path.join(self.input_dir, "repo"), 

288 os.path.join(self.input_dir, "exports.yaml"), 

289 self.output_dir, 

290 refcats=self.refcats, 

291 refcatPath=self.refcatPath) 

292 configs = [os.path.join(self.path, "config/config-gen3.py")] 

293 configs.extend(self.configfiles or []) 

294 configs.extend(configFiles or []) 

295 collection = f"{self.instrumentName}/tests/all" 

296 job = self._runJointcal(repopath, 

297 self.inputCollections, 

298 collection, 

299 configFiles=configs, 

300 configOptions=configOptions, 

301 registerDatasetTypes=True, 

302 whereSuffix=whereSuffix, 

303 nJobs=nJobs) 

304 

305 if metrics: 

306 self._test_metrics(job.measurements, metrics) 

307 

308 butler = lsst.daf.butler.Butler(repopath, collections=collection) 

309 if astrometryOutputs is not None: 

310 for visit, detectors in astrometryOutputs.items(): 

311 self._check_astrometry_output(butler, visit, detectors) 

312 else: 

313 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalSkyWcsCatalog"))), 0) 

314 if photometryOutputs is not None: 

315 for visit, detectors in photometryOutputs.items(): 

316 self._check_photometry_output(butler, visit, detectors) 

317 else: 

318 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalPhotoCalibCatalog"))), 0) 

319 

320 return repopath 

321 

322 def _check_astrometry_output(self, butler, visit, detectors): 

323 """Check that the WCSs were written for each detector, and only for the 

324 correct detectors. 

325 """ 

326 dataId = self.outputDataId.copy() 

327 dataId['visit'] = visit 

328 

329 catalog = butler.get('jointcalSkyWcsCatalog', dataId) 

330 for record in catalog: 

331 self.assertIsInstance(record.getWcs(), lsst.afw.geom.SkyWcs, 

332 msg=f"visit {visit}: {record}") 

333 np.testing.assert_array_equal(catalog['id'], detectors) 

334 

335 def _check_photometry_output(self, butler, visit, detectors): 

336 """Check that the PhotoCalibs were written for each detector, and only 

337 for the correct detectors. 

338 """ 

339 dataId = self.outputDataId.copy() 

340 dataId['visit'] = visit 

341 

342 catalog = butler.get('jointcalPhotoCalibCatalog', dataId) 

343 for record in catalog: 

344 self.assertIsInstance(record.getPhotoCalib(), lsst.afw.image.PhotoCalib, 

345 msg=f"visit {visit}: {record}") 

346 np.testing.assert_array_equal(catalog['id'], detectors)