Coverage for tests/testUtil.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of ctrl_mpexec. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (http://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
DatasetType, Registry)
dimensions=("instrument", "detector")): dimensions=["instrument", "detector"], storageClass="NumpyArray", doc="Input dataset type for this task") dimensions=["instrument", "detector"], storageClass="NumpyArray", doc="Output dataset type for this task") storageClass="NumpyArray", doc="Init Output dataset type for this task")
pipelineConnections=AddTaskConnections):
# example task which overrides run() method
"""InitOutputs for this task"""
"""Number of times run() method was called for this class"""
"""Raises exception at this call to run()"""
if AddTask.stopAt == AddTask.countExec: raise RuntimeError("pretend something bad happened") AddTask.countExec += 1 self.metadata.add("add", self.config.addend) output = [val + self.config.addend for val in input] _LOG.info("input = %s, output = %s", input, output) return pipeBase.Struct(output=output)
if taskName == "AddTask": return AddTask, "AddTask"
if config is None: config = taskClass.ConfigClass() if overrides: overrides.applyTo(config) return taskClass(config=config, initInputs=None)
"""Mock version of butler, only usable for testing
Parameters ---------- fullRegistry : `boolean`, optional If True then instantiate SQLite registry with default configuration. If False then registry is just a namespace with `dimensions` attribute containing DimensionUniverse from default configuration. """ self.datasets = {} self.fullRegistry = fullRegistry if self.fullRegistry: testDir = os.path.dirname(__file__) configFile = os.path.join(testDir, "config/butler.yaml") butlerConfig = ButlerConfig(configFile) self.registry = Registry.fromConfig(butlerConfig, create=True) self.registry.registerRun(collection) self.run = collection else: self.registry = SimpleNamespace(dimensions=DimensionUniverse.fromConfig()) self.run = collection
"""Copied from real Butler """ if isinstance(datasetRefOrType, DatasetRef): if dataId is not None or kwds: raise ValueError("DatasetRef given, cannot use dataId as well") datasetType = datasetRefOrType.datasetType dataId = datasetRefOrType.dataId else: # Don't check whether DataId is provided, because Registry APIs # can usually construct a better error message when it wasn't. if isinstance(datasetRefOrType, DatasetType): datasetType = datasetRefOrType else: datasetType = self.registry.getDatasetType(datasetRefOrType) return datasetType, dataId
def key(dataId): """Make a dict key out of dataId. """ return frozenset(dataId.items())
datasetType, dataId = self._standardizeArgs(datasetRefOrType, dataId, **kwds) _LOG.info("butler.get: datasetType=%s dataId=%s", datasetType.name, dataId) dsTypeName = datasetType.name key = self.key(dataId) dsdata = self.datasets.get(dsTypeName) if dsdata: return dsdata.get(key) return None
datasetType, dataId = self._standardizeArgs(datasetRefOrType, dataId, **kwds) _LOG.info("butler.put: datasetType=%s dataId=%s obj=%r", datasetType.name, dataId, obj) dsTypeName = datasetType.name key = self.key(dataId) dsdata = self.datasets.setdefault(dsTypeName, {}) dsdata[key] = obj if self.fullRegistry: ref = self.registry.addDataset(datasetType, dataId, run=self.run, producer=producer, recursive=False, **kwds) else: # we should return DatasetRef with reasonable ID, ID is supposed to be unique refId = sum(len(val) for val in self.datasets.values()) ref = DatasetRef(datasetType, dataId, id=refId) return ref
datasetType, dataId = self._standardizeArgs(datasetRefOrType, dataId, **kwds) _LOG.info("butler.remove: datasetType=%s dataId=%s", datasetType.name, dataId) dsTypeName = datasetType.name key = self.key(dataId) dsdata = self.datasets.get(dsTypeName) del dsdata[key] ref = self.registry.find(self.run, datasetType, dataId, **kwds) if remember: self.registry.disassociate(self.run, [ref]) else: self.registry.removeDataset(ref)
"""Register all dataset types used by tasks in a registry.
Copied and modified from `PreExecInit.initializeDatasetTypes`.
Parameters ---------- registry : `~lsst.daf.butler.Registry` Registry instance. pipeline : `~lsst.pipe.base.Pipeline` Iterable of TaskDef instances. """ for taskDef in pipeline: datasetTypes = pipeBase.TaskDatasetTypes.fromTaskDef(taskDef, registry=registry) for datasetType in itertools.chain(datasetTypes.initInputs, datasetTypes.initOutputs, datasetTypes.inputs, datasetTypes.outputs, datasetTypes.prerequisites): _LOG.info("Registering %s with registry", datasetType) # this is a no-op if it already exists and is consistent, # and it raises if it is inconsistent. registry.registerDatasetType(datasetType)
"""Make simple QuantumGraph for tests.
Makes simple one-task pipeline with AddTask, sets up in-memory registry and butler, fills them with minimal data, and generates QuantumGraph with all of that.
Parameters ---------- nQuanta : `int` Number of quanta in a graph. pipeline : `~lsst.pipe.base.Pipeline` If `None` then one-task pipeline is made with `AddTask` and default `AddTaskConfig`. butler : `~lsst.daf.butler.Butler`, optional Data butler instance, this should be an instance returned from a previous call to this method. skipExisting : `bool`, optional If `True` (default), a Quantum is not created if all its outputs already exist. clobberExisting : `bool`, optional If `True`, overwrite any outputs that already exist. Cannot be `True` if ``skipExisting`` is.
Returns ------- butler : `~lsst.daf.butler.Butler` Butler instance qgraph : `~lsst.pipe.base.QuantumGraph` Quantum graph instance """
if pipeline is None: pipeline = pipeBase.Pipeline("test pipeline") pipeline.addTask(AddTask, "task1") pipeline = list(pipeline.toExpandedPipeline())
if butler is None:
butler = ButlerMock(fullRegistry=True)
# Add dataset types to registry registerDatasetTypes(butler.registry, pipeline)
# Small set of DataIds included in QGraph records = [dict(instrument="INSTR", id=i, full_name=str(i)) for i in range(nQuanta)] dataIds = [dict(instrument="INSTR", detector=detector) for detector in range(nQuanta)]
# Add all needed dimensions to registry butler.registry.insertDimensionData("instrument", dict(name="INSTR")) butler.registry.insertDimensionData("detector", *records)
# Add inputs to butler for i, dataId in enumerate(dataIds): data = numpy.array([i, 10*i]) butler.put(data, "add_input", dataId)
# Make the graph, task factory is not needed here builder = pipeBase.GraphBuilder(registry=butler.registry, skipExisting=skipExisting, clobberExisting=clobberExisting) qgraph = builder.makeGraph( pipeline, inputCollections=defaultdict(functools.partial(list, [butler.run])), outputCollection=butler.run, userQuery="" )
return butler, qgraph |