Coverage for tests/test_transform.py: 38%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

160 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Test the basic operation of measurement transformations. 

24 

25We test measurement transforms in two ways: 

26 

27First, we construct and run a simple TransformTask on the (mocked) results of 

28measurement tasks. The same test is carried out against both 

29SingleFrameMeasurementTask and ForcedMeasurementTask, on the basis that the 

30transformation system should be agnostic as to the origin of the source 

31catalog it is transforming. 

32 

33Secondly, we use data from the obs_test package to demonstrate that the 

34transformtion system and its interface package are capable of operating on 

35data processed by the rest of the stack. 

36 

37For the purposes of testing, we define a "TrivialMeasurement" plugin and 

38associated transformation. Rather than building a catalog by measuring 

39genuine SourceRecords, we directly populate a catalog following the 

40TrivialMeasurement schema, then check that it is transformed properly by the 

41TrivialMeasurementTransform. 

42""" 

43import contextlib 

44import os 

45import shutil 

46import tempfile 

47import unittest 

48 

49import lsst.utils 

50import lsst.afw.table as afwTable 

51import lsst.geom as geom 

52import lsst.daf.persistence as dafPersist 

53import lsst.meas.base as measBase 

54import lsst.utils.tests 

55from lsst.pipe.tasks.multiBand import MeasureMergedCoaddSourcesConfig 

56from lsst.pipe.tasks.processCcd import ProcessCcdTask, ProcessCcdConfig 

57from lsst.pipe.tasks.transformMeasurement import (TransformConfig, TransformTask, SrcTransformTask, 

58 RunTransformConfig, CoaddSrcTransformTask) 

59 

60PLUGIN_NAME = "base_TrivialMeasurement" 

61 

62# Rather than providing real WCS and calibration objects to the 

63# transformation, we use this simple placeholder to keep track of the number 

64# of times it is accessed. 

65 

66 

67class Placeholder: 

68 

69 def __init__(self): 

70 self.count = 0 

71 

72 def increment(self): 

73 self.count += 1 

74 

75 

76class TrivialMeasurementTransform(measBase.transforms.MeasurementTransform): 

77 

78 def __init__(self, config, name, mapper): 

79 """Pass through all input fields to the output, and add a new field 

80 named after the measurement with the suffix "_transform". 

81 """ 

82 measBase.transforms.MeasurementTransform.__init__(self, config, name, mapper) 

83 for key, field in mapper.getInputSchema().extract(name + "*").values(): 

84 mapper.addMapping(key) 

85 self.key = mapper.editOutputSchema().addField(name + "_transform", type="D", doc="transformed dummy") 

86 

87 def __call__(self, inputCatalog, outputCatalog, wcs, photoCalib): 

88 """Transform inputCatalog to outputCatalog. 

89 

90 We update the wcs and photoCalib placeholders to indicate that they have 

91 been seen in the transformation, but do not use their values. 

92 

93 @param[in] inputCatalog SourceCatalog of measurements for transformation. 

94 @param[out] outputCatalog BaseCatalog of transformed measurements. 

95 @param[in] wcs Dummy WCS information; an instance of Placeholder. 

96 @param[in] photoCalib Dummy calibration information; an instance of Placeholder. 

97 """ 

98 if hasattr(wcs, "increment"): 

99 wcs.increment() 

100 if hasattr(photoCalib, "increment"): 

101 photoCalib.increment() 

102 inColumns = inputCatalog.getColumnView() 

103 outColumns = outputCatalog.getColumnView() 

104 outColumns[self.key] = -1.0 * inColumns[self.name] 

105 

106 

107class TrivialMeasurementBase: 

108 

109 """Default values for a trivial measurement plugin, subclassed below""" 

110 @staticmethod 

111 def getExecutionOrder(): 

112 return 0 

113 

114 @staticmethod 

115 def getTransformClass(): 

116 return TrivialMeasurementTransform 

117 

118 def measure(self, measRecord, exposure): 

119 measRecord.set(self.key, 1.0) 

120 

121 

122@measBase.register(PLUGIN_NAME) 

123class SFTrivialMeasurement(TrivialMeasurementBase, measBase.sfm.SingleFramePlugin): 

124 

125 """Single frame version of the trivial measurement""" 

126 

127 def __init__(self, config, name, schema, metadata): 

128 measBase.sfm.SingleFramePlugin.__init__(self, config, name, schema, metadata) 

129 self.key = schema.addField(name, type="D", doc="dummy field") 

130 

131 

132@measBase.register(PLUGIN_NAME) 

133class ForcedTrivialMeasurement(TrivialMeasurementBase, measBase.forcedMeasurement.ForcedPlugin): 

134 

135 """Forced frame version of the trivial measurement""" 

136 

137 def __init__(self, config, name, schemaMapper, metadata): 

138 measBase.forcedMeasurement.ForcedPlugin.__init__(self, config, name, schemaMapper, metadata) 

139 self.key = schemaMapper.editOutputSchema().addField(name, type="D", doc="dummy field") 

140 

141 

142class TransformTestCase(lsst.utils.tests.TestCase): 

143 

144 def _transformAndCheck(self, measConf, schema, transformTask): 

145 """Check the results of applying transformTask to a SourceCatalog. 

146 

147 @param[in] measConf Measurement plugin configuration. 

148 @param[in] schema Input catalog schema. 

149 @param[in] transformTask Instance of TransformTask to be applied. 

150 

151 For internal use by this test case. 

152 """ 

153 # There should now be one transformation registered per measurement plugin. 

154 self.assertEqual(len(measConf.plugins.names), len(transformTask.transforms)) 

155 

156 # Rather than do a real measurement, we use a dummy source catalog 

157 # containing a source at an arbitrary position. 

158 inCat = afwTable.SourceCatalog(schema) 

159 r = inCat.addNew() 

160 r.setCoord(geom.SpherePoint(0.0, 11.19, geom.degrees)) 

161 r[PLUGIN_NAME] = 1.0 

162 

163 wcs, photoCalib = Placeholder(), Placeholder() 

164 outCat = transformTask.run(inCat, wcs, photoCalib) 

165 

166 # Check that all sources have been transformed appropriately. 

167 for inSrc, outSrc in zip(inCat, outCat): 

168 self.assertEqual(outSrc[PLUGIN_NAME], inSrc[PLUGIN_NAME]) 

169 self.assertEqual(outSrc[PLUGIN_NAME + "_transform"], inSrc[PLUGIN_NAME] * -1.0) 

170 for field in transformTask.config.toDict()['copyFields']: 

171 self.assertEqual(outSrc.get(field), inSrc.get(field)) 

172 

173 # Check that the wcs and photoCalib objects were accessed once per transform. 

174 self.assertEqual(wcs.count, len(transformTask.transforms)) 

175 self.assertEqual(photoCalib.count, len(transformTask.transforms)) 

176 

177 def testSingleFrameMeasurementTransform(self): 

178 """Test applying a transform task to the results of single frame measurement.""" 

179 schema = afwTable.SourceTable.makeMinimalSchema() 

180 sfmConfig = measBase.SingleFrameMeasurementConfig(plugins=[PLUGIN_NAME]) 

181 # We don't use slots in this test 

182 for key in sfmConfig.slots: 

183 setattr(sfmConfig.slots, key, None) 

184 sfmTask = measBase.SingleFrameMeasurementTask(schema, config=sfmConfig) 

185 transformTask = TransformTask(measConfig=sfmConfig, 

186 inputSchema=sfmTask.schema, outputDataset="src") 

187 self._transformAndCheck(sfmConfig, sfmTask.schema, transformTask) 

188 

189 def testForcedMeasurementTransform(self): 

190 """Test applying a transform task to the results of forced measurement.""" 

191 schema = afwTable.SourceTable.makeMinimalSchema() 

192 forcedConfig = measBase.ForcedMeasurementConfig(plugins=[PLUGIN_NAME]) 

193 # We don't use slots in this test 

194 for key in forcedConfig.slots: 

195 setattr(forcedConfig.slots, key, None) 

196 forcedConfig.copyColumns = {"id": "objectId", "parent": "parentObjectId"} 

197 forcedTask = measBase.ForcedMeasurementTask(schema, config=forcedConfig) 

198 transformConfig = TransformConfig(copyFields=("objectId", "coord_ra", "coord_dec")) 

199 transformTask = TransformTask(measConfig=forcedConfig, 

200 inputSchema=forcedTask.schema, outputDataset="forced_src", 

201 config=transformConfig) 

202 self._transformAndCheck(forcedConfig, forcedTask.schema, transformTask) 

203 

204 

205@contextlib.contextmanager 

206def tempDirectory(*args, **kwargs): 

207 """A context manager which provides a temporary directory and automatically cleans up when done.""" 

208 dirname = tempfile.mkdtemp(*args, **kwargs) 

209 try: 

210 yield dirname 

211 finally: 

212 shutil.rmtree(dirname, ignore_errors=True) 

213 

214 

215class RunTransformTestCase(lsst.utils.tests.TestCase): 

216 

217 def testInterface(self): 

218 obsTestDir = lsst.utils.getPackageDir('obs_test') 

219 inputDir = os.path.join(obsTestDir, "data", "input") 

220 

221 # Configure a ProcessCcd task such that it will return a minimal 

222 # number of measurements plus our test plugin. 

223 cfg = ProcessCcdConfig() 

224 cfg.calibrate.measurement.plugins.names = ["base_SdssCentroid", "base_SkyCoord", PLUGIN_NAME] 

225 cfg.calibrate.measurement.slots.shape = None 

226 cfg.calibrate.measurement.slots.psfFlux = None 

227 cfg.calibrate.measurement.slots.apFlux = None 

228 cfg.calibrate.measurement.slots.gaussianFlux = None 

229 cfg.calibrate.measurement.slots.modelFlux = None 

230 cfg.calibrate.measurement.slots.calibFlux = None 

231 # no reference catalog, so... 

232 cfg.calibrate.doAstrometry = False 

233 cfg.calibrate.doPhotoCal = False 

234 # disable aperture correction because we aren't measuring aperture flux 

235 cfg.calibrate.doApCorr = False 

236 # Extendedness requires modelFlux, disabled above. 

237 cfg.calibrate.catalogCalculation.plugins.names.discard("base_ClassificationExtendedness") 

238 

239 # Process the test data with ProcessCcd then perform a transform. 

240 with tempDirectory() as tempDir: 

241 measResult = ProcessCcdTask.parseAndRun(args=[inputDir, "--output", tempDir, "--id", "visit=1"], 

242 config=cfg, doReturnResults=True) 

243 trArgs = [tempDir, "--output", tempDir, "--id", "visit=1", 

244 "-c", "inputConfigType=processCcd_config"] 

245 trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True) 

246 

247 # It should be possible to reprocess the data through a new transform task with exactly 

248 # the same configuration without throwing. This check is useful since we are 

249 # constructing the task on the fly, which could conceivably cause problems with 

250 # configuration/metadata persistence. 

251 trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True) 

252 

253 measSrcs = measResult.resultList[0].result.calibRes.sourceCat 

254 trSrcs = trResult.resultList[0].result 

255 

256 # The length of the measured and transformed catalogs should be the same. 

257 self.assertEqual(len(measSrcs), len(trSrcs)) 

258 

259 # Each source should have been measured & transformed appropriately. 

260 for measSrc, trSrc in zip(measSrcs, trSrcs): 

261 # The TrivialMeasurement should be transformed as defined above. 

262 self.assertEqual(trSrc[PLUGIN_NAME], measSrc[PLUGIN_NAME]) 

263 self.assertEqual(trSrc[PLUGIN_NAME + "_transform"], -1.0 * measSrc[PLUGIN_NAME]) 

264 

265 # The SdssCentroid should be transformed to celestial coordinates. 

266 # Checking that the full transformation has been done correctly is 

267 # out of scope for this test case; we just ensure that there's 

268 # plausible position in the transformed record. 

269 trCoord = afwTable.CoordKey(trSrcs.schema["base_SdssCentroid"]).get(trSrc) 

270 self.assertAlmostEqual(measSrc.getCoord().getLongitude(), trCoord.getLongitude()) 

271 self.assertAlmostEqual(measSrc.getCoord().getLatitude(), trCoord.getLatitude()) 

272 

273 

274class CoaddTransformTestCase(lsst.utils.tests.TestCase): 

275 """Check that CoaddSrcTransformTask is set up properly. 

276 

277 RunTransformTestCase, above, has tested the basic RunTransformTask mechanism. 

278 Here, we just check that it is appropriately adapted for coadds. 

279 """ 

280 MEASUREMENT_CONFIG_DATASET = "measureCoaddSources_config" 

281 

282 # The following are hard-coded in lsst.pipe.tasks.multiBand: 

283 SCHEMA_SUFFIX = "Coadd_meas_schema" 

284 SOURCE_SUFFIX = "Coadd_meas" 

285 CALEXP_SUFFIX = "Coadd_calexp" 

286 

287 def setUp(self): 

288 # We need a temporary repository in which we can store test configs. 

289 self.repo = tempfile.mkdtemp() 

290 with open(os.path.join(self.repo, "_mapper"), "w") as f: 

291 f.write("lsst.obs.test.TestMapper") 

292 self.butler = dafPersist.Butler(self.repo) 

293 

294 # Persist a coadd measurement config. 

295 # We disable all measurement plugins so that there's no actual work 

296 # for the TransformTask to do. 

297 measCfg = MeasureMergedCoaddSourcesConfig() 

298 measCfg.measurement.plugins.names = [] 

299 self.butler.put(measCfg, self.MEASUREMENT_CONFIG_DATASET) 

300 

301 # Record the type of coadd on which our supposed measurements have 

302 # been carried out: we need to check this was propagated to the 

303 # transformation task. 

304 self.coaddName = measCfg.coaddName 

305 

306 # Since we disabled all measurement plugins, our catalog can be 

307 # simple. 

308 c = afwTable.SourceCatalog(afwTable.SourceTable.makeMinimalSchema()) 

309 self.butler.put(c, self.coaddName + self.SCHEMA_SUFFIX) 

310 

311 # Our transformation config needs to know the type of the measurement 

312 # configuration. 

313 trCfg = RunTransformConfig() 

314 trCfg.inputConfigType = self.MEASUREMENT_CONFIG_DATASET 

315 

316 self.transformTask = CoaddSrcTransformTask(config=trCfg, log=None, butler=self.butler) 

317 

318 def tearDown(self): 

319 del self.butler 

320 del self.transformTask 

321 shutil.rmtree(self.repo) 

322 

323 def testCoaddName(self): 

324 """Check that we have correctly derived the coadd name.""" 

325 self.assertEqual(self.transformTask.coaddName, self.coaddName) 

326 

327 def testSourceType(self): 

328 """Check that we have correctly derived the type of the measured sources.""" 

329 self.assertEqual(self.transformTask.sourceType, self.coaddName + self.SOURCE_SUFFIX) 

330 

331 def testCalexpType(self): 

332 """Check that we have correctly derived the type of the measurement images.""" 

333 self.assertEqual(self.transformTask.calexpType, self.coaddName + self.CALEXP_SUFFIX) 

334 

335 

336class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

337 pass 

338 

339 

340def setup_module(module): 

341 lsst.utils.tests.init() 

342 

343 

344if __name__ == "__main__": 344 ↛ 345line 344 didn't jump to line 345, because the condition on line 344 was never true

345 lsst.utils.tests.init() 

346 unittest.main()