Coverage for tests/jointcalTestBase.py: 16%

109 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-08 12:12 +0000

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["importRepository", "JointcalTestBase"] 

23 

24import os 

25import shutil 

26import tempfile 

27import numpy as np 

28 

29import lsst.afw.image.utils 

30from lsst.ctrl.mpexec import SimplePipelineExecutor 

31import lsst.daf.butler 

32from lsst.daf.butler.script import ingest_files 

33import lsst.pipe.base 

34import lsst.obs.base 

35import lsst.geom 

36from lsst.verify.bin.jobReporter import JobReporter 

37 

38import lsst.jointcal 

39 

40 

41def importRepository(instrument, exportPath, exportFile, outputDir=None, 

42 refcats=None, refcatPath=""): 

43 """Import a gen3 test repository into a test path. 

44 

45 Parameters 

46 ---------- 

47 instrument : `str` 

48 Full string name for the instrument. 

49 exportPath : `str` 

50 Path to location of repository to export. 

51 exportFile : `str` 

52 Filename of YAML butler export data, describing the data to import. 

53 outputDir : `str`, or `None` 

54 Root path to put the test repo in; appended with `testrepo/`. 

55 If not supplied, a temporary path is generated with mkdtemp; this path 

56 must be deleted by the calling code, it is not automatically removed. 

57 refcats : `dict` [`str`, `str`] 

58 Mapping of refcat name to relative path to .ecsv ingest file. 

59 refcatPath : `str` 

60 Path prefix for refcat ingest files and files linked therein. 

61 

62 Returns 

63 ------- 

64 repopath : `str` 

65 The path to the newly created butler repo. 

66 """ 

67 if outputDir is None: 

68 repopath = tempfile.mkdtemp() 

69 else: 

70 repopath = os.path.join(outputDir, "testrepo") 

71 

72 # Make the repo and retrieve a writeable Butler 

73 _ = lsst.daf.butler.Butler.makeRepo(repopath) 

74 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

75 

76 instrInstance = lsst.pipe.base.Instrument.from_string(instrument) 

77 instrInstance.register(butler.registry) 

78 

79 # Register refcats first, so the `refcats` collection will exist. 

80 if refcats is not None: 

81 for name, file in refcats.items(): 

82 graph = butler.registry.dimensions.extract(["htm7"]) 

83 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog", 

84 universe=butler.registry.dimensions) 

85 butler.registry.registerDatasetType(datasetType) 

86 ingest_files(repopath, name, "refcats", file, prefix=refcatPath) 

87 # New butler to get the refcats collections to be visible 

88 butler = lsst.daf.butler.Butler(repopath, writeable=True) 

89 

90 butler.import_(directory=exportPath, filename=exportFile, 

91 transfer='symlink', 

92 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

93 return repopath 

94 

95 

96class JointcalTestBase: 

97 """ 

98 Base class for jointcal tests, to genericize some test running and setup. 

99 

100 Derive from this first, then from TestCase. 

101 """ 

102 

103 def set_output_dir(self): 

104 """Set the output directory to the name of the test method, in .test/ 

105 """ 

106 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1]) 

107 

108 def setUp_base(self, 

109 instrumentClass, 

110 instrumentName, 

111 input_dir="", 

112 all_visits=None, 

113 log_level=None, 

114 where="", 

115 refcats=None, 

116 refcatPath="", 

117 inputCollections=None, 

118 outputDataId=None): 

119 """ 

120 Call from your child classes's setUp() to get the necessary variables built. 

121 

122 Parameters 

123 ---------- 

124 instrumentClass : `str` 

125 The full module name of the instrument to be registered in the 

126 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam" 

127 instrumentName : `str` 

128 The name of the instrument as it appears in the repo collections. 

129 For example, "HSC". 

130 input_dir : `str` 

131 Directory of input butler repository. 

132 all_visits : `list` [`int`] 

133 List of the available visits to generate the dataset query from. 

134 log_level : `str` 

135 Set to the default log level you want jointcal to produce while the 

136 tests are running. See the developer docs about logging for valid 

137 levels: https://developer.lsst.io/coding/logging.html 

138 where : `str` 

139 Data ID query for pipetask specifying the data to run on. 

140 refcats : `dict` [`str`, `str`] 

141 Mapping of refcat name to relative path to .ecsv ingest file. 

142 refcatPath : `str` 

143 Path prefix for refcat ingest files and files linked therein. 

144 inputCollections : `list` [`str`] 

145 String to use for "-i" input collections (comma delimited). 

146 For example, "refcats,HSC/runs/tests,HSC/calib" 

147 outputDataId : `dict` 

148 Partial dataIds for testing whether the output files were written. 

149 """ 

150 self.path = os.path.dirname(__file__) 

151 

152 self.instrumentClass = instrumentClass 

153 self.instrumentName = instrumentName 

154 

155 self.input_dir = input_dir 

156 self.all_visits = all_visits 

157 self.log_level = log_level 

158 

159 self.configfiles = [] 

160 

161 # Make unittest output more verbose failure messages to assert failures. 

162 self.longMessage = True 

163 

164 self.where = where 

165 self.refcats = refcats 

166 self.refcatPath = refcatPath 

167 self.inputCollections = inputCollections 

168 

169 self.outputDataId = outputDataId 

170 

171 self.set_output_dir() 

172 

173 def tearDown(self): 

174 shutil.rmtree(self.output_dir, ignore_errors=True) 

175 

176 def _test_metrics(self, result, expect): 

177 """Test a dictionary of "metrics" against those returned by jointcal.py 

178 

179 Parameters 

180 ---------- 

181 result : `dict` 

182 Result metric dictionary from jointcal.py 

183 expect : `dict` 

184 Expected metric dictionary; set a value to None to not test it. 

185 """ 

186 for key in result: 

187 with self.subTest(key.metric): 

188 if expect[key.metric] is not None: 

189 value = result[key].quantity.value 

190 if isinstance(value, float): 

191 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5) 

192 else: 

193 self.assertEqual(value, expect[key.metric], msg=key.metric) 

194 

195 def _runJointcal(self, repo, 

196 inputCollections, outputCollection, 

197 configFiles=None, configOptions=None, 

198 registerDatasetTypes=False, whereSuffix=None, 

199 nJobs=1): 

200 """Run a pipeline via the SimplePipelineExecutor. 

201 

202 Parameters 

203 ---------- 

204 repo : `str` 

205 Gen3 Butler repository to read from/write to. 

206 inputCollections : `list` [`str`] 

207 String to use for "-i" input collections (comma delimited). 

208 For example, "refcats,HSC/runs/tests,HSC/calib" 

209 outputCollection : `str` 

210 Name of the output collection to write to. For example, 

211 "HSC/testdata/jointcal" 

212 configFiles : `list` [`str`], optional 

213 List of jointcal config files to use. 

214 configOptions : `dict` [`str`], optional 

215 Individual jointcal config options (field: value) to override. 

216 registerDatasetTypes : bool, optional 

217 Set "--register-dataset-types" when running the pipeline. 

218 whereSuffix : `str`, optional 

219 Additional parameters to the ``where`` pipetask statement. 

220 nJobs : `int`, optional 

221 Number of quanta expected to be run. 

222 

223 Returns 

224 ------- 

225 job : `lsst.verify.Job` 

226 Job containing the metric measurements from this test run. 

227 """ 

228 config = lsst.jointcal.JointcalConfig() 

229 if configFiles: 

230 for file in configFiles: 

231 config.load(file) 

232 if configOptions: 

233 for key, value in configOptions.items(): 

234 setattr(config, key, value) 

235 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where 

236 lsst.daf.butler.cli.cliLog.CliLog.initLog(False) 

237 butler = SimplePipelineExecutor.prep_butler(repo, 

238 inputs=inputCollections, 

239 output=outputCollection, 

240 output_run=outputCollection.replace("all", "jointcal")) 

241 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask, 

242 config=config, 

243 where=where, 

244 butler=butler) 

245 executor.run(register_dataset_types=registerDatasetTypes) 

246 # JobReporter bundles all metrics in the collection into one job. 

247 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run() 

248 # should only ever get one job output in tests, unless specified 

249 self.assertEqual(len(jobs), nJobs) 

250 # Sort the jobs, as QuantumGraph ordering is not guaranteed, and some 

251 # tests check metrics values for one job while producing two. 

252 sorted_jobs = {key: jobs[key] for key in sorted(list(jobs.keys()))} 

253 return list(sorted_jobs.values())[0] 

254 

255 def _runJointcalTest(self, astrometryOutputs=None, photometryOutputs=None, 

256 configFiles=None, configOptions=None, whereSuffix=None, 

257 metrics=None, nJobs=1): 

258 """Create a Butler repo and run jointcal on it. 

259 

260 Parameters 

261 ---------- 

262 atsrometryOutputs, photometryOutputs : `dict` [`int`, `list`] 

263 visit: [detectors] dictionary to test that output files were saved. 

264 Default None means that there should be no output for that 

265 respective dataset type. 

266 configFiles : `list` [`str`], optional 

267 List of jointcal config files to use. 

268 configOptions : `dict` [`str`], optional 

269 Individual jointcal config options (field: value) to override. 

270 whereSuffix : `str`, optional 

271 Additional parameters to the ``where`` pipetask statement. 

272 metrics : `dict`, optional 

273 Dictionary of 'metricName': value to test jointcal's result.metrics 

274 against. 

275 nJobs : `int`, optional 

276 Number of quanta expected to be run. 

277 

278 Returns 

279 ------- 

280 repopath : `str` 

281 The path to the newly created butler repo. 

282 """ 

283 repopath = importRepository(self.instrumentClass, 

284 os.path.join(self.input_dir, "repo"), 

285 os.path.join(self.input_dir, "exports.yaml"), 

286 self.output_dir, 

287 refcats=self.refcats, 

288 refcatPath=self.refcatPath) 

289 configs = [os.path.join(self.path, "config/config.py")] 

290 configs.extend(self.configfiles or []) 

291 configs.extend(configFiles or []) 

292 collection = f"{self.instrumentName}/tests/all" 

293 job = self._runJointcal(repopath, 

294 self.inputCollections, 

295 collection, 

296 configFiles=configs, 

297 configOptions=configOptions, 

298 registerDatasetTypes=True, 

299 whereSuffix=whereSuffix, 

300 nJobs=nJobs) 

301 

302 if metrics: 

303 self._test_metrics(job.measurements, metrics) 

304 

305 butler = lsst.daf.butler.Butler(repopath, collections=collection) 

306 if astrometryOutputs is not None: 

307 for visit, detectors in astrometryOutputs.items(): 

308 self._check_astrometry_output(butler, visit, detectors) 

309 else: 

310 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalSkyWcsCatalog"))), 0) 

311 if photometryOutputs is not None: 

312 for visit, detectors in photometryOutputs.items(): 

313 self._check_photometry_output(butler, visit, detectors) 

314 else: 

315 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalPhotoCalibCatalog"))), 0) 

316 

317 return repopath 

318 

319 def _check_astrometry_output(self, butler, visit, detectors): 

320 """Check that the WCSs were written for each detector, and only for the 

321 correct detectors. 

322 """ 

323 dataId = self.outputDataId.copy() 

324 dataId['visit'] = visit 

325 

326 catalog = butler.get('jointcalSkyWcsCatalog', dataId) 

327 for record in catalog: 

328 self.assertIsInstance(record.getWcs(), lsst.afw.geom.SkyWcs, 

329 msg=f"visit {visit}: {record}") 

330 np.testing.assert_array_equal(catalog['id'], detectors) 

331 

332 def _check_photometry_output(self, butler, visit, detectors): 

333 """Check that the PhotoCalibs were written for each detector, and only 

334 for the correct detectors. 

335 """ 

336 dataId = self.outputDataId.copy() 

337 dataId['visit'] = visit 

338 

339 catalog = butler.get('jointcalPhotoCalibCatalog', dataId) 

340 for record in catalog: 

341 self.assertIsInstance(record.getPhotoCalib(), lsst.afw.image.PhotoCalib, 

342 msg=f"visit {visit}: {record}") 

343 np.testing.assert_array_equal(catalog['id'], detectors)