Coverage for tests/test_transform.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2015 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Test the basic operation of measurement transformations.
25We test measurement transforms in two ways:
27First, we construct and run a simple TransformTask on the (mocked) results of
28measurement tasks. The same test is carried out against both
29SingleFrameMeasurementTask and ForcedMeasurementTask, on the basis that the
30transformation system should be agnostic as to the origin of the source
31catalog it is transforming.
33Secondly, we use data from the obs_test package to demonstrate that the
34transformtion system and its interface package are capable of operating on
35data processed by the rest of the stack.
37For the purposes of testing, we define a "TrivialMeasurement" plugin and
38associated transformation. Rather than building a catalog by measuring
39genuine SourceRecords, we directly populate a catalog following the
40TrivialMeasurement schema, then check that it is transformed properly by the
41TrivialMeasurementTransform.
42"""
43import contextlib
44import os
45import shutil
46import tempfile
47import unittest
49import lsst.utils
50import lsst.afw.table as afwTable
51import lsst.geom as geom
52import lsst.daf.persistence as dafPersist
53import lsst.meas.base as measBase
54import lsst.utils.tests
55from lsst.pipe.tasks.multiBand import MeasureMergedCoaddSourcesConfig
56from lsst.pipe.tasks.processCcd import ProcessCcdTask, ProcessCcdConfig
57from lsst.pipe.tasks.transformMeasurement import (TransformConfig, TransformTask, SrcTransformTask,
58 RunTransformConfig, CoaddSrcTransformTask)
60PLUGIN_NAME = "base_TrivialMeasurement"
62# Rather than providing real WCS and calibration objects to the
63# transformation, we use this simple placeholder to keep track of the number
64# of times it is accessed.
67class Placeholder:
69 def __init__(self):
70 self.count = 0
72 def increment(self):
73 self.count += 1
76class TrivialMeasurementTransform(measBase.transforms.MeasurementTransform):
78 def __init__(self, config, name, mapper):
79 """Pass through all input fields to the output, and add a new field
80 named after the measurement with the suffix "_transform".
81 """
82 measBase.transforms.MeasurementTransform.__init__(self, config, name, mapper)
83 for key, field in mapper.getInputSchema().extract(name + "*").values():
84 mapper.addMapping(key)
85 self.key = mapper.editOutputSchema().addField(name + "_transform", type="D", doc="transformed dummy")
87 def __call__(self, inputCatalog, outputCatalog, wcs, photoCalib):
88 """Transform inputCatalog to outputCatalog.
90 We update the wcs and photoCalib placeholders to indicate that they have
91 been seen in the transformation, but do not use their values.
93 @param[in] inputCatalog SourceCatalog of measurements for transformation.
94 @param[out] outputCatalog BaseCatalog of transformed measurements.
95 @param[in] wcs Dummy WCS information; an instance of Placeholder.
96 @param[in] photoCalib Dummy calibration information; an instance of Placeholder.
97 """
98 if hasattr(wcs, "increment"):
99 wcs.increment()
100 if hasattr(photoCalib, "increment"):
101 photoCalib.increment()
102 inColumns = inputCatalog.getColumnView()
103 outColumns = outputCatalog.getColumnView()
104 outColumns[self.key] = -1.0 * inColumns[self.name]
107class TrivialMeasurementBase:
109 """Default values for a trivial measurement plugin, subclassed below"""
110 @staticmethod
111 def getExecutionOrder():
112 return 0
114 @staticmethod
115 def getTransformClass():
116 return TrivialMeasurementTransform
118 def measure(self, measRecord, exposure):
119 measRecord.set(self.key, 1.0)
122@measBase.register(PLUGIN_NAME)
123class SFTrivialMeasurement(TrivialMeasurementBase, measBase.sfm.SingleFramePlugin):
125 """Single frame version of the trivial measurement"""
127 def __init__(self, config, name, schema, metadata):
128 measBase.sfm.SingleFramePlugin.__init__(self, config, name, schema, metadata)
129 self.key = schema.addField(name, type="D", doc="dummy field")
132@measBase.register(PLUGIN_NAME)
133class ForcedTrivialMeasurement(TrivialMeasurementBase, measBase.forcedMeasurement.ForcedPlugin):
135 """Forced frame version of the trivial measurement"""
137 def __init__(self, config, name, schemaMapper, metadata):
138 measBase.forcedMeasurement.ForcedPlugin.__init__(self, config, name, schemaMapper, metadata)
139 self.key = schemaMapper.editOutputSchema().addField(name, type="D", doc="dummy field")
142class TransformTestCase(lsst.utils.tests.TestCase):
144 def _transformAndCheck(self, measConf, schema, transformTask):
145 """Check the results of applying transformTask to a SourceCatalog.
147 @param[in] measConf Measurement plugin configuration.
148 @param[in] schema Input catalog schema.
149 @param[in] transformTask Instance of TransformTask to be applied.
151 For internal use by this test case.
152 """
153 # There should now be one transformation registered per measurement plugin.
154 self.assertEqual(len(measConf.plugins.names), len(transformTask.transforms))
156 # Rather than do a real measurement, we use a dummy source catalog
157 # containing a source at an arbitrary position.
158 inCat = afwTable.SourceCatalog(schema)
159 r = inCat.addNew()
160 r.setCoord(geom.SpherePoint(0.0, 11.19, geom.degrees))
161 r[PLUGIN_NAME] = 1.0
163 wcs, photoCalib = Placeholder(), Placeholder()
164 outCat = transformTask.run(inCat, wcs, photoCalib)
166 # Check that all sources have been transformed appropriately.
167 for inSrc, outSrc in zip(inCat, outCat):
168 self.assertEqual(outSrc[PLUGIN_NAME], inSrc[PLUGIN_NAME])
169 self.assertEqual(outSrc[PLUGIN_NAME + "_transform"], inSrc[PLUGIN_NAME] * -1.0)
170 for field in transformTask.config.toDict()['copyFields']:
171 self.assertEqual(outSrc.get(field), inSrc.get(field))
173 # Check that the wcs and photoCalib objects were accessed once per transform.
174 self.assertEqual(wcs.count, len(transformTask.transforms))
175 self.assertEqual(photoCalib.count, len(transformTask.transforms))
177 def testSingleFrameMeasurementTransform(self):
178 """Test applying a transform task to the results of single frame measurement."""
179 schema = afwTable.SourceTable.makeMinimalSchema()
180 sfmConfig = measBase.SingleFrameMeasurementConfig(plugins=[PLUGIN_NAME])
181 # We don't use slots in this test
182 for key in sfmConfig.slots:
183 setattr(sfmConfig.slots, key, None)
184 sfmTask = measBase.SingleFrameMeasurementTask(schema, config=sfmConfig)
185 transformTask = TransformTask(measConfig=sfmConfig,
186 inputSchema=sfmTask.schema, outputDataset="src")
187 self._transformAndCheck(sfmConfig, sfmTask.schema, transformTask)
189 def testForcedMeasurementTransform(self):
190 """Test applying a transform task to the results of forced measurement."""
191 schema = afwTable.SourceTable.makeMinimalSchema()
192 forcedConfig = measBase.ForcedMeasurementConfig(plugins=[PLUGIN_NAME])
193 # We don't use slots in this test
194 for key in forcedConfig.slots:
195 setattr(forcedConfig.slots, key, None)
196 forcedConfig.copyColumns = {"id": "objectId", "parent": "parentObjectId"}
197 forcedTask = measBase.ForcedMeasurementTask(schema, config=forcedConfig)
198 transformConfig = TransformConfig(copyFields=("objectId", "coord_ra", "coord_dec"))
199 transformTask = TransformTask(measConfig=forcedConfig,
200 inputSchema=forcedTask.schema, outputDataset="forced_src",
201 config=transformConfig)
202 self._transformAndCheck(forcedConfig, forcedTask.schema, transformTask)
205@contextlib.contextmanager
206def tempDirectory(*args, **kwargs):
207 """A context manager which provides a temporary directory and automatically cleans up when done."""
208 dirname = tempfile.mkdtemp(*args, **kwargs)
209 try:
210 yield dirname
211 finally:
212 shutil.rmtree(dirname, ignore_errors=True)
215class RunTransformTestCase(lsst.utils.tests.TestCase):
217 def testInterface(self):
218 obsTestDir = lsst.utils.getPackageDir('obs_test')
219 inputDir = os.path.join(obsTestDir, "data", "input")
221 # Configure a ProcessCcd task such that it will return a minimal
222 # number of measurements plus our test plugin.
223 cfg = ProcessCcdConfig()
224 cfg.calibrate.measurement.plugins.names = ["base_SdssCentroid", "base_SkyCoord", PLUGIN_NAME]
225 cfg.calibrate.measurement.slots.shape = None
226 cfg.calibrate.measurement.slots.psfFlux = None
227 cfg.calibrate.measurement.slots.apFlux = None
228 cfg.calibrate.measurement.slots.gaussianFlux = None
229 cfg.calibrate.measurement.slots.modelFlux = None
230 cfg.calibrate.measurement.slots.calibFlux = None
231 # no reference catalog, so...
232 cfg.calibrate.doAstrometry = False
233 cfg.calibrate.doPhotoCal = False
234 # disable aperture correction because we aren't measuring aperture flux
235 cfg.calibrate.doApCorr = False
236 # Extendedness requires modelFlux, disabled above.
237 cfg.calibrate.catalogCalculation.plugins.names.discard("base_ClassificationExtendedness")
239 # Process the test data with ProcessCcd then perform a transform.
240 with tempDirectory() as tempDir:
241 measResult = ProcessCcdTask.parseAndRun(args=[inputDir, "--output", tempDir, "--id", "visit=1"],
242 config=cfg, doReturnResults=True)
243 trArgs = [tempDir, "--output", tempDir, "--id", "visit=1",
244 "-c", "inputConfigType=processCcd_config"]
245 trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True)
247 # It should be possible to reprocess the data through a new transform task with exactly
248 # the same configuration without throwing. This check is useful since we are
249 # constructing the task on the fly, which could conceivably cause problems with
250 # configuration/metadata persistence.
251 trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True)
253 measSrcs = measResult.resultList[0].result.calibRes.sourceCat
254 trSrcs = trResult.resultList[0].result
256 # The length of the measured and transformed catalogs should be the same.
257 self.assertEqual(len(measSrcs), len(trSrcs))
259 # Each source should have been measured & transformed appropriately.
260 for measSrc, trSrc in zip(measSrcs, trSrcs):
261 # The TrivialMeasurement should be transformed as defined above.
262 self.assertEqual(trSrc[PLUGIN_NAME], measSrc[PLUGIN_NAME])
263 self.assertEqual(trSrc[PLUGIN_NAME + "_transform"], -1.0 * measSrc[PLUGIN_NAME])
265 # The SdssCentroid should be transformed to celestial coordinates.
266 # Checking that the full transformation has been done correctly is
267 # out of scope for this test case; we just ensure that there's
268 # plausible position in the transformed record.
269 trCoord = afwTable.CoordKey(trSrcs.schema["base_SdssCentroid"]).get(trSrc)
270 self.assertAlmostEqual(measSrc.getCoord().getLongitude(), trCoord.getLongitude())
271 self.assertAlmostEqual(measSrc.getCoord().getLatitude(), trCoord.getLatitude())
274class CoaddTransformTestCase(lsst.utils.tests.TestCase):
275 """Check that CoaddSrcTransformTask is set up properly.
277 RunTransformTestCase, above, has tested the basic RunTransformTask mechanism.
278 Here, we just check that it is appropriately adapted for coadds.
279 """
280 MEASUREMENT_CONFIG_DATASET = "measureCoaddSources_config"
282 # The following are hard-coded in lsst.pipe.tasks.multiBand:
283 SCHEMA_SUFFIX = "Coadd_meas_schema"
284 SOURCE_SUFFIX = "Coadd_meas"
285 CALEXP_SUFFIX = "Coadd_calexp"
287 def setUp(self):
288 # We need a temporary repository in which we can store test configs.
289 self.repo = tempfile.mkdtemp()
290 with open(os.path.join(self.repo, "_mapper"), "w") as f:
291 f.write("lsst.obs.test.TestMapper")
292 self.butler = dafPersist.Butler(self.repo)
294 # Persist a coadd measurement config.
295 # We disable all measurement plugins so that there's no actual work
296 # for the TransformTask to do.
297 measCfg = MeasureMergedCoaddSourcesConfig()
298 measCfg.measurement.plugins.names = []
299 self.butler.put(measCfg, self.MEASUREMENT_CONFIG_DATASET)
301 # Record the type of coadd on which our supposed measurements have
302 # been carried out: we need to check this was propagated to the
303 # transformation task.
304 self.coaddName = measCfg.coaddName
306 # Since we disabled all measurement plugins, our catalog can be
307 # simple.
308 c = afwTable.SourceCatalog(afwTable.SourceTable.makeMinimalSchema())
309 self.butler.put(c, self.coaddName + self.SCHEMA_SUFFIX)
311 # Our transformation config needs to know the type of the measurement
312 # configuration.
313 trCfg = RunTransformConfig()
314 trCfg.inputConfigType = self.MEASUREMENT_CONFIG_DATASET
316 self.transformTask = CoaddSrcTransformTask(config=trCfg, log=None, butler=self.butler)
318 def tearDown(self):
319 del self.butler
320 del self.transformTask
321 shutil.rmtree(self.repo)
323 def testCoaddName(self):
324 """Check that we have correctly derived the coadd name."""
325 self.assertEqual(self.transformTask.coaddName, self.coaddName)
327 def testSourceType(self):
328 """Check that we have correctly derived the type of the measured sources."""
329 self.assertEqual(self.transformTask.sourceType, self.coaddName + self.SOURCE_SUFFIX)
331 def testCalexpType(self):
332 """Check that we have correctly derived the type of the measurement images."""
333 self.assertEqual(self.transformTask.calexpType, self.coaddName + self.CALEXP_SUFFIX)
336class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
337 pass
340def setup_module(module):
341 lsst.utils.tests.init()
344if __name__ == "__main__": 344 ↛ 345line 344 didn't jump to line 345, because the condition on line 344 was never true
345 lsst.utils.tests.init()
346 unittest.main()