27 """This module defines the Butler class.""" 28 from builtins
import str, super
29 from past.builtins
import basestring
30 from builtins
import object
38 from .
import ReadProxy, ButlerSubset, ButlerDataRef, \
39 Storage, Policy, NoResults, Repository, DataId, RepositoryCfg, \
40 RepositoryArgs, listify, setify, sequencify, doImport, ButlerComposite, genericAssembler, \
41 genericDisassembler, PosixStorage, ParentsMismatch
43 preinitedMapperWarning = (
"Passing an instantiated mapper into " +
44 "Butler.__init__ will prevent Butler from passing " +
45 "parentRegistry or repositoryCfg information to " +
46 "the mapper, which is done only at init time. " +
47 "It is better to pass a importable string or " +
52 """Represents a Butler configuration. 56 cfg is 'wet paint' and very likely to change. Use of it in production 57 code other than via the 'old butler' API is strongly discouraged. 59 yaml_tag =
u"!ButlerCfg" 62 super().
__init__({
'repoCfg': repoCfg,
'cls': cls})
66 """Container object for repository data used by Butler 71 The arguments that are used to find or create the RepositoryCfg. 73 "input", "output", or "parent", indicating why Butler loaded this repository. 74 * input: the Repository was passed as a Butler input. 75 * output: the Repository was passed as a Butler output. 76 * parent: the Repository was specified in the RepositoryCfg parents list of a readable repository. 81 The configuration for the Repository. 84 "new", "existing", or "nested". Indicates the origin of the repository and its RepositoryCfg: 85 * new: it was created by this instance of Butler, and this instance of Butler will generate the 87 * existing: it was found (via the root or cfgRoot argument) 88 * nested: the full RepositoryCfg was nested in another RepositoryCfg's parents list (this can happen 89 if parameters of an input specified by RepositoryArgs or dict does not entirely match an existing 93 Path or URI to the location of the RepositoryCfg file. 95 repo : lsst.daf.persistence.Repository 96 The Repository class instance. 98 parentRepoDatas : list of RepoData 99 The parents of this Repository, as indicated this Repository's RepositoryCfg. If this is a new 100 Repository then these are the inputs to this Butler (and will be saved in the RepositoryCfg). These 101 RepoData objects are not owned by this RepoData, these are references to peer RepoData objects in the 102 Butler's RepoDataContainer. 104 isV1Repository : bool 105 True if this is an Old Butler repository. In this case the repository does not have a RepositoryCfg 106 file. It may have a _mapper file and may have a _parent symlink. It will never be treated as a "new" 107 repository, i.e. even though there is not a RepositoryCfg file, one will not be generated. 108 If False, this is a New Butler repository and is specified by RepositoryCfg file. 111 These are values that may be used to restrict the search of input repositories. Details are available 112 in the RepositoryArgs and DataId classes. 115 "input", "output", or "parent", indicating why Butler loaded this repository. 116 * input: the Repository was passed as a Butler input. 117 * output: the Repository was passed as a Butler output. 118 * parent: the Repository was specified in the RepositoryCfg parents list of a readable repository. 120 _repoArgs : RepositoryArgs 121 Contains the arguments that were used to specify this Repository. 151 "parentRepoDatas={}," +
154 "parentRegistry={})").format(
155 self.__class__.__name__,
167 def setCfg(self, cfg, origin, root, isV1Repository):
168 """Set information about the cfg into the RepoData 173 The RepositoryCfg for the repo. 175 'new', 'existing', or 'nested' 177 URI or absolute path to the location of the RepositoryCfg.yaml file. 183 if origin
not in (
'new',
'existing',
'nested'):
184 raise RuntimeError(
"Invalid value for origin:{}".format(origin))
204 if val
not in (
'input',
'output',
'parent'):
205 raise RuntimeError(
"Invalid value for role: {}".format(val))
209 """Get the parents & grandparents etc of this repo data, in depth-first search order. 211 Duplicate entries will be removed in cases where the same parent appears more than once in the parent 216 context : set, optional 217 Users should typically omit context and accept the default argument. Context is used to keep a set 218 of known RepoDatas when calling this function recursively, for duplicate elimination. 223 A list of the parents & grandparents etc of a given repo data, in depth-first search order. 228 if id(self)
in context:
230 context.add(id(self))
232 parents.append(parent)
233 parents += parent.getParentRepoDatas(context)
244 """Container object for RepoData instances owned by a Butler instance. 248 repoDataList : list of RepoData 249 repoData - RepoData instance to add 255 self.
_all = repoDataList
259 """Get a list of RepoData that are used to as inputs to the Butler. 260 The list is created lazily as needed, and cached. 264 A list of RepoData with readable repositories, in the order to be used when searching. 267 raise RuntimeError(
"Inputs not yet initialized.")
271 """Get a list of RepoData that are used to as outputs to the Butler. 272 The list is created lazily as needed, and cached. 276 A list of RepoData with writable repositories, in the order to be use when searching. 279 raise RuntimeError(
"Outputs not yet initialized.")
283 """Get a list of all RepoData that are used to as by the Butler. 284 The list is created lazily as needed, and cached. 288 A list of RepoData with writable repositories, in the order to be use when searching. 293 return "%s(_inputs=%r, \n_outputs=%s, \n_all=%s)" % (
294 self.__class__.__name__,
299 def _buildLookupLists(self):
300 """Build the inputs and outputs lists based on the order of self.all().""" 302 def addToList(repoData, lst):
303 """Add a repoData and each of its parents (depth first) to a list""" 304 if id(repoData)
in alreadyAdded:
307 alreadyAdded.add(id(repoData))
308 for parent
in repoData.parentRepoDatas:
309 addToList(parent, lst)
312 raise RuntimeError(
"Lookup lists are already built.")
313 inputs = [repoData
for repoData
in self.
all()
if repoData.role ==
'input']
314 outputs = [repoData
for repoData
in self.
all()
if repoData.role ==
'output']
317 for repoData
in outputs:
318 if 'r' in repoData.repoArgs.mode: 319 addToList(repoData.repoData, self._inputs) 320 for repoData
in inputs:
321 addToList(repoData.repoData, self.
_inputs)
322 self.
_outputs = [repoData.repoData
for repoData
in outputs]
326 """Butler provides a generic mechanism for persisting and retrieving data using mappers. 328 A Butler manages a collection of datasets known as a repository. Each dataset has a type representing its 329 intended usage and a location. Note that the dataset type is not the same as the C++ or Python type of the 330 object containing the data. For example, an ExposureF object might be used to hold the data for a raw 331 image, a post-ISR image, a calibrated science image, or a difference image. These would all be different 334 A Butler can produce a collection of possible values for a key (or tuples of values for multiple keys) if 335 given a partial data identifier. It can check for the existence of a file containing a dataset given its 336 type and data identifier. The Butler can then retrieve the dataset. Similarly, it can persist an object to 337 an appropriate location when given its associated data identifier. 339 Note that the Butler has two more advanced features when retrieving a data set. First, the retrieval is 340 lazy. Input does not occur until the data set is actually accessed. This allows datasets to be retrieved 341 and placed on a clipboard prospectively with little cost, even if the algorithm of a stage ends up not 342 using them. Second, the Butler will call a standardization hook upon retrieval of the dataset. This 343 function, contained in the input mapper object, must perform any necessary manipulations to force the 344 retrieved object to conform to standards, including translating metadata. 348 __init__(self, root, mapper=None, **mapperArgs) 350 defineAlias(self, alias, datasetType) 352 getKeys(self, datasetType=None, level=None) 354 queryMetadata(self, datasetType, format=None, dataId={}, **rest) 356 datasetExists(self, datasetType, dataId={}, **rest) 358 get(self, datasetType, dataId={}, immediate=False, **rest) 360 put(self, obj, datasetType, dataId={}, **rest) 362 subset(self, datasetType, level=None, dataId={}, **rest) 364 dataRef(self, datasetType, level=None, dataId={}, **rest) 368 The preferred method of initialization is to use the `inputs` and `outputs` __init__ parameters. These 369 are described in the parameters section, below. 371 For backward compatibility: this initialization method signature can take a posix root path, and 372 optionally a mapper class instance or class type that will be instantiated using the mapperArgs input 373 argument. However, for this to work in a backward compatible way it creates a single repository that is 374 used as both an input and an output repository. This is NOT preferred, and will likely break any 375 provenance system we have in place. 380 .. note:: Deprecated in 12_0 381 `root` will be removed in TBD, it is replaced by `inputs` and `outputs` for 382 multiple-repository support. 383 A file system path. Will only work with a PosixRepository. 384 mapper : string or instance 385 .. note:: Deprecated in 12_0 386 `mapper` will be removed in TBD, it is replaced by `inputs` and `outputs` for 387 multiple-repository support. 388 Provides a mapper to be used with Butler. 390 .. note:: Deprecated in 12_0 391 `mapperArgs` will be removed in TBD, it is replaced by `inputs` and `outputs` for 392 multiple-repository support. 393 Provides arguments to be passed to the mapper if the mapper input argument is a class type to be 394 instantiated by Butler. 395 inputs : RepositoryArgs, dict, or string 396 Can be a single item or a list. Provides arguments to load an existing repository (or repositories). 397 String is assumed to be a URI and is used as the cfgRoot (URI to the location of the cfg file). (Local 398 file system URI does not have to start with 'file://' and in this way can be a relative path). The 399 `RepositoryArgs` class can be used to provide more parameters with which to initialize a repository 400 (such as `mapper`, `mapperArgs`, `tags`, etc. See the `RepositoryArgs` documentation for more 401 details). A dict may be used as shorthand for a `RepositoryArgs` class instance. The dict keys must 402 match parameters to the `RepositoryArgs.__init__` function. 403 outputs : RepositoryArgs, dict, or string 404 Provides arguments to load one or more existing repositories or create new ones. The different types 405 are handled the same as for `inputs`. 407 The Butler init sequence loads all of the input and output repositories. 408 This creates the object hierarchy to read from and write to them. Each 409 repository can have 0 or more parents, which also get loaded as inputs. 410 This becomes a DAG of repositories. Ultimately, Butler creates a list of 411 these Repositories in the order that they are used. 413 Initialization Sequence 414 ======================= 416 During initialization Butler creates a Repository class instance & support structure for each object 417 passed to `inputs` and `outputs` as well as the parent repositories recorded in the `RepositoryCfg` of 418 each existing readable repository. 420 This process is complex. It is explained below to shed some light on the intent of each step. 422 1. Input Argument Standardization 423 --------------------------------- 425 In `Butler._processInputArguments` the input arguments are verified to be legal (and a RuntimeError is 426 raised if not), and they are converted into an expected format that is used for the rest of the Butler 427 init sequence. See the docstring for `_processInputArguments`. 429 2. Create RepoData Objects 430 -------------------------- 432 Butler uses an object, called `RepoData`, to keep track of information about each repository; each 433 repository is contained in a single `RepoData`. The attributes are explained in its docstring. 435 After `_processInputArguments`, a RepoData is instantiated and put in a list for each repository in 436 `outputs` and `inputs`. This list of RepoData, the `repoDataList`, now represents all the output and input 437 repositories (but not parent repositories) that this Butler instance will use. 439 3. Get `RepositoryCfg`s 440 ----------------------- 442 `Butler._getCfgs` gets the `RepositoryCfg` for each repository the `repoDataList`. The behavior is 443 described in the docstring. 448 `Butler._addParents` then considers the parents list in the `RepositoryCfg` of each `RepoData` in the 449 `repoDataList` and inserts new `RepoData` objects for each parent not represented in the proper location 450 in the `repoDataList`. Ultimately a flat list is built to represent the DAG of readable repositories 451 represented in depth-first order. 453 5. Set and Verify Parents of Outputs 454 ------------------------------------ 456 To be able to load parent repositories when output repositories are used as inputs, the input repositories 457 are recorded as parents in the `RepositoryCfg` file of new output repositories. When an output repository 458 already exists, for consistency the Butler's inputs must match the list of parents specified the already- 459 existing output repository's `RepositoryCfg` file. 461 In `Butler._setAndVerifyParentsLists`, the list of parents is recorded in the `RepositoryCfg` of new 462 repositories. For existing repositories the list of parents is compared with the `RepositoryCfg`'s parents 463 list, and if they do not match a `RuntimeError` is raised. 465 6. Set the Default Mapper 466 ------------------------- 468 If all the input repositories use the same mapper then we can assume that mapper to be the 469 "default mapper". If there are new output repositories whose `RepositoryArgs` do not specify a mapper and 470 there is a default mapper then the new output repository will be set to use that default mapper. 472 This is handled in `Butler._setDefaultMapper`. 474 7. Cache References to Parent RepoDatas 475 --------------------------------------- 477 In `Butler._connectParentRepoDatas`, in each `RepoData` in `repoDataList`, a list of `RepoData` object 478 references is built that matches the parents specified in that `RepoData`'s `RepositoryCfg`. 480 This list is used later to find things in that repository's parents, without considering peer repository's 481 parents. (e.g. finding the registry of a parent) 486 Tags are described at https://ldm-463.lsst.io/v/draft/#tagging 488 In `Butler._setRepoDataTags`, for each `RepoData`, the tags specified by its `RepositoryArgs` are recorded 489 in a set, and added to the tags set in each of its parents, for ease of lookup when mapping. 491 9. Find Parent Registry and Instantiate RepoData 492 ------------------------------------------------ 494 At this point there is enough information to instantiate the `Repository` instances. There is one final 495 step before instantiating the Repository, which is to try to get a parent registry that can be used by the 496 child repository. The criteria for "can be used" is spelled out in `Butler._setParentRegistry`. However, 497 to get the registry from the parent, the parent must be instantiated. The `repoDataList`, in depth-first 498 search order, is built so that the most-dependent repositories are first, and the least dependent 499 repositories are last. So the `repoDataList` is reversed and the Repositories are instantiated in that 500 order; for each RepoData a parent registry is searched for, and then the Repository is instantiated with 501 whatever registry could be found.""" 503 def __init__(self, root=None, mapper=None, inputs=None, outputs=None, **mapperArgs):
504 self.
_initArgs = {
'root': root,
'mapper': mapper,
'inputs': inputs,
'outputs': outputs,
505 'mapperArgs': mapperArgs}
507 self.
log = Log.getLogger(
"daf.persistence.butler")
510 root=root, mapper=mapper, inputs=inputs, outputs=outputs, **mapperArgs)
513 inputs = [
RepoData(args,
'input')
for args
in inputs]
514 outputs = [
RepoData(args,
'output')
for args
in outputs]
515 repoDataList = outputs + inputs
531 for repoData
in repoDataList:
534 def _initRepo(self, repoData):
535 if repoData.repo
is not None:
539 for parentRepoData
in repoData.parentRepoDatas:
540 if parentRepoData.cfg.mapper != repoData.cfg.mapper:
542 if parentRepoData.repo
is None:
544 parentRegistry = parentRepoData.repo.getRegistry()
545 repoData.parentRegistry = parentRegistry
if parentRegistry
else parentRepoData.parentRegistry
546 if repoData.parentRegistry:
550 def _processInputArguments(self, root=None, mapper=None, inputs=None, outputs=None, **mapperArgs):
551 """Process, verify, and standardize the input arguments. 552 * Inputs can not be for Old Butler (root, mapper, mapperArgs) AND New Butler (inputs, outputs) 553 `root`, `mapper`, and `mapperArgs` are Old Butler init API. 554 `inputs` and `outputs` are New Butler init API. 555 Old Butler and New Butler init API may not be mixed, Butler may be initialized with only the Old 556 arguments or the New arguments. 557 * Verify that if there is a readable output that there is exactly one output. (This restriction is in 558 place because all readable repositories must be parents of writable repositories, and for 559 consistency the DAG of readable repositories must always be the same. Keeping the list of parents 560 becomes very complicated in the presence of multiple readable output repositories. It is better to 561 only write to output repositories, and then create a new Butler instance and use the outputs as 562 inputs, and write to new output repositories.) 563 * Make a copy of inputs & outputs so they may be modified without changing the passed-in arguments. 564 * Convert any input/output values that are URI strings to RepositoryArgs. 565 * Listify inputs & outputs. 566 * Set default RW mode on inputs & outputs as needed. 570 Same as Butler.__init__ 574 (list of RepositoryArgs, list of RepositoryArgs) 575 First item is a list to use as inputs. 576 Second item is a list to use as outputs. 581 If Old Butler and New Butler arguments are both used this will raise. 582 If an output is readable there is more than one output this will raise. 585 inputs = copy.deepcopy(inputs)
586 outputs = copy.deepcopy(outputs)
588 isV1Args = inputs
is None and outputs
is None 592 mapperArgs=mapperArgs
or None)
593 elif root
or mapper
or mapperArgs:
595 'Butler version 1 API (root, mapper, **mapperArgs) may ' +
596 'not be used with version 2 API (inputs, outputs)')
605 if not isinstance(args, RepositoryArgs)
else args
for args
in inputs]
607 if not isinstance(args, RepositoryArgs)
else args
for args
in outputs]
611 if args.mode
is None:
613 elif 'rw' == args.mode:
615 elif 'r' != args.mode: 616 raise RuntimeError(
"The mode of an input should be readable.")
618 if args.mode
is None:
620 elif 'w' not in args.mode:
621 raise RuntimeError(
"The mode of an output should be writable.")
623 for args
in inputs + outputs:
624 if (args.mapper
and not isinstance(args.mapper, basestring)
and 625 not inspect.isclass(args.mapper)):
626 self.
log.warn(preinitedMapperWarning)
631 raise RuntimeError(
"Butler does not support multiple output repositories if any of the " 632 "outputs are readable.")
637 def inputIsInOutputs(inputArgs, outputArgsList):
638 for o
in outputArgsList:
639 if (
'r' in o.mode and 640 o.root == inputArgs.root and 641 o.mapper == inputArgs.mapper
and 642 o.mapperArgs == inputArgs.mapperArgs
and 643 o.tags == inputArgs.tags
and 644 o.policy == inputArgs.policy):
645 self.
log.debug((
"Input repositoryArgs {} is also listed in outputs as readable; " +
646 "throwing away the input.").format(inputArgs))
650 inputs = [args
for args
in inputs
if not inputIsInOutputs(args, outputs)]
651 return inputs, outputs
654 def _getParentVal(repoData):
655 """Get the value of this repoData as it should appear in the parents 656 list of other repositories""" 657 if repoData.isV1Repository:
659 if repoData.cfgOrigin ==
'nested':
662 return repoData.cfg.root
665 def _getParents(ofRepoData, repoInfo):
666 """Create a parents list of repoData from inputs and (readable) outputs.""" 669 for repoData
in repoInfo:
670 if repoData
is ofRepoData:
672 if 'r' not in repoData.repoArgs.mode: 674 parents.append(Butler._getParentVal(repoData))
678 def _getOldButlerRepositoryCfg(repositoryArgs):
679 if not Storage.isPosix(repositoryArgs.cfgRoot):
681 if not PosixStorage.v1RepoExists(repositoryArgs.cfgRoot):
683 if not repositoryArgs.mapper:
684 repositoryArgs.mapper = PosixStorage.getMapperClass(repositoryArgs.cfgRoot)
685 cfg = RepositoryCfg.makeFromArgs(repositoryArgs)
686 parent = PosixStorage.getParentSymlinkPath(repositoryArgs.cfgRoot)
688 parent = Butler._getOldButlerRepositoryCfg(
RepositoryArgs(cfgRoot=parent, mode=
'r')) 689 if parent
is not None:
690 cfg.addParents([parent])
693 def _getRepositoryCfg(self, repositoryArgs):
694 """Try to get a repository from the location described by cfgRoot. 698 repositoryArgs : RepositoryArgs or string 699 Provides arguments to load an existing repository (or repositories). String is assumed to be a URI 700 and is used as the cfgRoot (URI to the location of the cfg file). 704 (RepositoryCfg or None, bool) 705 The RepositoryCfg, or None if one cannot be found, and True if the RepositoryCfg was created by 706 reading an Old Butler repository, or False if it is a New Butler Repository. 708 if not isinstance(repositoryArgs, RepositoryArgs):
711 cfg = self.storage.getRepositoryCfg(repositoryArgs.cfgRoot) 712 isOldButlerRepository = False 714 cfg = Butler._getOldButlerRepositoryCfg(repositoryArgs)
716 isOldButlerRepository =
True 717 return cfg, isOldButlerRepository
719 def _getCfgs(self, repoDataList):
720 """Get or make a RepositoryCfg for each RepoData, and add the cfg to the RepoData. 721 If the cfg exists, compare values. If values match then use the cfg as an "existing" cfg. If the 722 values do not match, use the cfg as a "nested" cfg. 723 If the cfg does not exist, the RepositoryArgs must be for a writable repository. 727 repoDataList : list of RepoData 728 The RepoData that are output and inputs of this Butler 733 If the passed-in RepositoryArgs indicate an existing repository but other cfg parameters in those 735 match the existing repository's cfg a RuntimeError will be raised. 737 def cfgMatchesArgs(args, cfg):
738 """Test if there are any values in an RepositoryArgs that conflict with the values in a cfg""" 739 if args.mapper
is not None and cfg.mapper != args.mapper:
741 if args.mapperArgs
is not None and cfg.mapperArgs != args.mapperArgs:
743 if args.policy
is not None and cfg.policy != args.policy:
747 for repoData
in repoDataList:
750 if 'w' not in repoData.repoArgs.mode:
752 "No cfg found for read-only input repository at {}".format(repoData.repoArgs.cfgRoot))
753 repoData.setCfg(cfg=RepositoryCfg.makeFromArgs(repoData.repoArgs),
755 root=repoData.repoArgs.cfgRoot,
756 isV1Repository=isOldButlerRepository)
766 for i, parent
in enumerate(cfg.parents):
767 if isinstance(parent, RepositoryCfg):
770 if parentIsOldButlerRepository:
771 parentCfg.mapperArgs = cfg.mapperArgs
772 self.
log.info((
"Butler is replacing an Old Butler parent repository path '{}' " 773 "found in the parents list of a New Butler repositoryCfg: {} " 774 "with a repositoryCfg that includes the child repository's " 775 "mapperArgs: {}. This affects the instantiated RepositoryCfg " 776 "but does not change the persisted child repositoryCfg.yaml file." 777 ).format(parent, cfg, parentCfg))
778 cfg._parents[i] = cfg._normalizeParents(cfg.root, [parentCfg])[0]
780 if 'w' in repoData.repoArgs.mode:
782 if not cfgMatchesArgs(repoData.repoArgs, cfg):
783 raise RuntimeError((
"The RepositoryArgs and RepositoryCfg must match for writable " +
784 "repositories, RepositoryCfg:{}, RepositoryArgs:{}").format(
785 cfg, repoData.repoArgs))
786 repoData.setCfg(cfg=cfg, origin=
'existing', root=repoData.repoArgs.cfgRoot,
787 isV1Repository=isOldButlerRepository)
790 if cfgMatchesArgs(repoData.repoArgs, cfg):
791 repoData.setCfg(cfg=cfg, origin=
'existing', root=repoData.repoArgs.cfgRoot,
792 isV1Repository=isOldButlerRepository)
794 repoData.setCfg(cfg=cfg, origin=
'nested', root=
None,
795 isV1Repository=isOldButlerRepository)
797 def _addParents(self, repoDataList):
798 """For each repoData in the input list, see if its parents are the next items in the list, and if not 799 add the parent, so that the repoDataList includes parents and is in order to operate depth-first 0..n. 803 repoDataList : list of RepoData 804 The RepoData for the Butler outputs + inputs. 809 Raised if a RepositoryCfg can not be found at a location where a parent repository should be. 813 if repoDataIdx == len(repoDataList):
815 repoData = repoDataList[repoDataIdx]
816 if 'r' not in repoData.repoArgs.mode: 819 if repoData.isNewRepository:
822 if repoData.cfg.parents
is None:
825 for repoParentIdx, repoParent
in enumerate(repoData.cfg.parents):
826 parentIdxInRepoDataList = repoDataIdx + repoParentIdx + 1
827 if not isinstance(repoParent, RepositoryCfg):
829 if repoParentCfg
is not None:
830 cfgOrigin =
'existing' 832 isOldButlerRepository =
False 833 repoParentCfg = repoParent
835 if (parentIdxInRepoDataList < len(repoDataList)
and 836 repoDataList[parentIdxInRepoDataList].cfg == repoParentCfg):
839 role = 'input' if repoData.role ==
'output' else 'parent' 841 newRepoInfo.repoData.setCfg(cfg=repoParentCfg, origin=cfgOrigin, root=args.cfgRoot,
842 isV1Repository=isOldButlerRepository)
843 repoDataList.insert(parentIdxInRepoDataList, newRepoInfo)
846 def _setAndVerifyParentsLists(self, repoDataList):
847 """Make a list of all the input repositories of this Butler, these are the parents of the outputs. 848 For new output repositories, set the parents in the RepositoryCfg. For existing output repositories 849 verify that the RepositoryCfg's parents match the parents list. 853 repoDataList : list of RepoData 854 All the RepoDatas loaded by this butler, in search order. 859 If an existing output repository is loaded and its parents do not match the parents of this Butler 860 an error will be raised. 862 def getIOParents(ofRepoData, repoDataList):
863 """make a parents list for repo in `ofRepoData` that is comprised of inputs and readable 864 outputs (not parents-of-parents) of this butler""" 866 for repoData
in repoDataList:
867 if repoData.role ==
'parent':
869 if repoData
is ofRepoData:
871 if repoData.role ==
'output':
872 if 'r' in repoData.repoArgs.mode: 873 raise RuntimeError(
"If an output is readable it must be the only output.")
880 for repoData
in repoDataList:
881 if repoData.role !=
'output':
883 parents = getIOParents(repoData, repoDataList)
885 if repoData.cfgOrigin ==
'new':
886 repoData.cfg.addParents(parents)
887 elif repoData.cfgOrigin
in (
'existing',
'nested'):
888 if repoData.cfg.parents != parents:
890 repoData.cfg.extendParents(parents)
891 except ParentsMismatch
as e:
892 raise RuntimeError((
"Inputs of this Butler:{} do not match parents of existing " +
893 "writable cfg:{} (ParentMismatch exception: {}").format(
894 parents, repoData.cfg.parents, e))
896 def _setDefaultMapper(self, repoDataList):
897 """Establish a default mapper if there is one and assign it to outputs that do not have a mapper 900 If all inputs have the same mapper it will be used as the default mapper. 904 repoDataList : list of RepoData 905 All the RepoDatas loaded by this butler, in search order. 910 If a default mapper can not be established and there is an output that does not have a mapper. 912 needyOutputs = [rd
for rd
in repoDataList
if rd.role ==
'output' and rd.cfg.mapper
is None]
913 if len(needyOutputs)
is 0:
915 mappers = set([rd.cfg.mapper
for rd
in repoDataList
if rd.role ==
'input'])
916 if len(mappers) != 1:
917 inputs = [rd
for rd
in repoDataList
if rd.role ==
'input']
919 (
"No default mapper could be established from inputs:{} and no mapper specified " +
920 "for outputs:{}").format(inputs, needyOutputs))
921 defaultMapper = mappers.pop()
922 for repoData
in needyOutputs:
923 repoData.cfg.mapper = defaultMapper
925 def _connectParentRepoDatas(self, repoDataList):
926 """For each RepoData in repoDataList, find its parent in the repoDataList and cache a reference to it. 930 repoDataList : list of RepoData 931 All the RepoDatas loaded by this butler, in search order. 936 When a parent is listed in the parents list but not found in the repoDataList. This is not 937 expected to ever happen and would indicate an internal Butler error. 939 for repoData
in repoDataList:
940 for parent
in repoData.cfg.parents:
942 for otherRepoData
in repoDataList:
943 if isinstance(parent, RepositoryCfg):
944 if otherRepoData.repoData.repoData.cfg == parent:
945 parentToAdd = otherRepoData.repoData
947 elif otherRepoData.repoData.cfg.root == parent:
948 parentToAdd = otherRepoData.repoData
950 if parentToAdd
is None:
952 "Could not find a parent matching {} to add to {}".format(parent, repoData))
953 repoData.addParentRepoData(parentToAdd)
956 def _getParentRepoData(parent, repoDataList):
957 """get a parent RepoData from a cfg from a list of RepoData 961 parent : string or RepositoryCfg 962 cfgRoot of a repo or a cfg that describes the repo 963 repoDataList : list of RepoData 969 A RepoData if one can be found, else None 972 for otherRepoData
in repoDataList:
973 if isinstance(parent, RepositoryCfg):
974 if otherRepoData.cfg == parent:
975 repoData = otherRepoData
977 elif otherRepoData.cfg.root == parent:
978 repoData = otherRepoData
982 def _setRepoDataTags(self):
983 """Set the tags from each repoArgs into all its parent repoArgs so that they can be included in tagged 985 def setTags(repoData, tags, context):
986 if id(repoData)
in context:
988 repoData.addTags(tags)
989 context.add(id(repoData))
990 for parentRepoData
in repoData.parentRepoDatas:
991 setTags(parentRepoData, tags, context)
992 for repoData
in self.
_repos.outputs() + self.
_repos.inputs():
993 setTags(repoData.repoData, repoData.repoArgs.tags, set())
995 def _convertV1Args(self, root, mapper, mapperArgs):
996 """Convert Old Butler RepositoryArgs (root, mapper, mapperArgs) to New Butler RepositoryArgs 1002 Posix path to repository root 1003 mapper : class, class instance, or string 1004 Instantiated class, a class object to be instantiated, or a string that refers to a class that 1005 can be imported & used as the mapper. 1007 RepositoryArgs & their values used when instantiating the mapper. 1012 (inputs, outputs) - values to be used for inputs and outputs in Butler.__init__ 1014 if (mapper
and not isinstance(mapper, basestring)
and 1015 not inspect.isclass(mapper)):
1016 self.
log.warn(preinitedMapperWarning)
1019 if hasattr(mapper,
'root'):
1028 mapperArgs=mapperArgs)
1029 return inputs, outputs
1032 return 'Butler(datasetTypeAliasDict=%s, repos=%s)' % (
1035 def _getDefaultMapper(self):
1037 """Get the default mapper. Currently this means if all the repositories use exactly the same mapper, 1038 that mapper may be considered the default. 1040 This definition may be changing; mappers may be able to exclude themselves as candidates for default, 1041 and they may nominate a different mapper instead. Also, we may not want to look at *all* the 1042 repositories, but only a depth-first search on each of the input & output repositories, and use the 1043 first-found mapper for each of those. TBD. 1052 Mapper class or None 1053 Returns the class type of the default mapper, or None if a default 1054 mapper can not be determined. 1056 defaultMapper =
None 1058 for inputRepoData
in self.
_repos.inputs():
1060 if inputRepoData.cfg.mapper
is not None:
1061 mapper = inputRepoData.cfg.mapper
1066 if isinstance(mapper, basestring):
1067 mapper = doImport(mapper)
1068 elif not inspect.isclass(mapper):
1069 mapper = mapper.__class__
1075 if defaultMapper
is None:
1076 defaultMapper = mapper
1077 elif mapper == defaultMapper:
1079 elif mapper
is not None:
1081 return defaultMapper
1083 def _assignDefaultMapper(self, defaultMapper):
1084 for repoData
in self.
_repos.all().values():
1085 if repoData.cfg.mapper
is None and (repoData.isNewRepository
or repoData.isV1Repository):
1086 if defaultMapper
is None:
1088 "No mapper specified for %s and no default mapper could be determined." %
1090 repoData.cfg.mapper = defaultMapper
1094 """posix-only; gets the mapper class at the path specified by root (if a file _mapper can be found at 1095 that location or in a parent location. 1097 As we abstract the storage and support different types of storage locations this method will be 1098 moved entirely into Butler Access, or made more dynamic, and the API will very likely change.""" 1099 return Storage.getMapperClass(root)
1102 """Register an alias that will be substituted in datasetTypes. 1107 The alias keyword. It may start with @ or not. It may not contain @ except as the first character. 1108 datasetType - string 1109 The string that will be substituted when @alias is passed into datasetType. It may not contain '@' 1113 atLoc = alias.rfind(
'@')
1115 alias =
"@" + str(alias)
1117 raise RuntimeError(
"Badly formatted alias string: %s" % (alias,))
1120 if datasetType.count(
'@') != 0:
1121 raise RuntimeError(
"Badly formatted type string: %s" % (datasetType))
1126 if key.startswith(alias)
or alias.startswith(key):
1127 raise RuntimeError(
"Alias: %s overlaps with existing alias: %s" % (alias, key))
1131 def getKeys(self, datasetType=None, level=None, tag=None):
1132 """Get the valid data id keys at or above the given level of hierarchy for the dataset type or the 1133 entire collection if None. The dict values are the basic Python types corresponding to the keys (int, 1138 datasetType - string 1139 The type of dataset to get keys for, entire collection if None. 1141 The hierarchy level to descend to. None if it should not be restricted. Use an empty string if the 1142 mapper should lookup the default level. 1143 tags - any, or list of any 1144 Any object that can be tested to be the same as the tag in a dataId passed into butler input 1145 functions. Applies only to input repositories: If tag is specified by the dataId then the repo 1146 will only be read from used if the tag in the dataId matches a tag used for that repository. 1150 Returns a dict. The dict keys are the valid data id keys at or above the given level of hierarchy for 1151 the dataset type or the entire collection if None. The dict values are the basic Python types 1152 corresponding to the keys (int, float, string). 1158 for repoData
in self.
_repos.inputs():
1159 if not tag
or len(tag.intersection(repoData.tags)) > 0:
1160 keys = repoData.repo.getKeys(datasetType, level)
1163 if keys
is not None:
1168 """Returns the valid values for one or more keys when given a partial 1169 input collection data id. 1173 datasetType - string 1174 The type of dataset to inquire about. 1176 Key or tuple of keys to be returned. 1177 dataId - DataId, dict 1178 The partial data id. 1180 Keyword arguments for the partial data id. 1184 A list of valid values or tuples of valid values as specified by the 1190 dataId.update(**rest)
1194 for repoData
in self.
_repos.inputs():
1195 if not dataId.tag
or len(dataId.tag.intersection(repoData.tags)) > 0:
1196 tuples = repoData.repo.queryMetadata(datasetType, format, dataId)
1203 if len(format) == 1:
1215 """Determines if a dataset file exists. 1219 datasetType - string 1220 The type of dataset to inquire about. 1221 dataId - DataId, dict 1222 The data id of the dataset. 1224 If True, look only in locations where the dataset could be written, 1225 and return True only if it is present in all of them. 1226 **rest keyword arguments for the data id. 1231 True if the dataset exists or is non-file-based. 1235 dataId.update(**rest)
1236 locations = self.
_locate(datasetType, dataId, write=write)
1238 if locations
is None:
1240 locations = [locations]
1245 for location
in locations:
1248 if isinstance(location, ButlerComposite):
1249 for name, componentInfo
in location.componentInfo.items():
1250 if componentInfo.subset:
1251 subset = self.
subset(datasetType=componentInfo.datasetType, dataId=location.dataId)
1252 exists = all([obj.datasetExists()
for obj
in subset])
1254 exists = self.
datasetExists(componentInfo.datasetType, location.dataId)
1258 if not location.repository.exists(location):
1262 def _locate(self, datasetType, dataId, write):
1263 """Get one or more ButlerLocations and/or ButlercComposites. 1267 datasetType : string 1268 The datasetType that is being searched for. The datasetType may be followed by a dot and 1269 a component name (component names are specified in the policy). IE datasetType.componentName 1271 dataId : dict or DataId class instance 1275 True if this is a search to write an object. False if it is a search to read an object. This 1276 affects what type (an object or a container) is returned. 1280 If write is False, will return either a single object or None. If write is True, will return a list 1281 (which may be empty) 1283 repos = self.
_repos.outputs()
if write
else self.
_repos.inputs()
1285 for repoData
in repos:
1287 if not write
and dataId.tag
and len(dataId.tag.intersection(repoData.tags)) == 0:
1289 components = datasetType.split(
'.')
1290 datasetType = components[0]
1291 components = components[1:]
1293 location = repoData.repo.map(datasetType, dataId, write=write)
1296 if location
is None:
1298 location.datasetType = datasetType
1299 if len(components) > 0:
1300 if not isinstance(location, ButlerComposite):
1301 raise RuntimeError(
"The location for a dotted datasetType must be a composite.")
1303 components[0] = location.componentInfo[components[0]].datasetType
1305 datasetType =
'.'.join(components)
1306 location = self.
_locate(datasetType, dataId, write)
1308 if location
is None:
1319 if hasattr(location.mapper,
"bypass_" + location.datasetType):
1323 location.bypass = bypass
1324 except (NoResults, IOError):
1325 self.
log.debug(
"Continuing dataset search while evaluating " 1326 "bypass function for Dataset type:{} Data ID:{} at " 1327 "location {}".format(datasetType, dataId, location))
1331 if (isinstance(location, ButlerComposite)
or hasattr(location,
'bypass')
or 1332 location.repository.exists(location)):
1336 locations.extend(location)
1338 locations.append(location)
1344 def _getBypassFunc(location, dataId):
1345 pythonType = location.getPythonType()
1346 if pythonType
is not None:
1347 if isinstance(pythonType, basestring):
1348 pythonType = doImport(pythonType)
1349 bypassFunc = getattr(location.mapper,
"bypass_" + location.datasetType)
1350 return lambda: bypassFunc(location.datasetType, pythonType, location, dataId)
1352 def get(self, datasetType, dataId=None, immediate=True, **rest):
1353 """Retrieves a dataset given an input collection data id. 1357 datasetType - string 1358 The type of dataset to retrieve. 1362 If False use a proxy for delayed loading. 1364 keyword arguments for the data id. 1368 An object retrieved from the dataset (or a proxy for one). 1372 dataId.update(**rest)
1374 location = self.
_locate(datasetType, dataId, write=
False)
1375 if location
is None:
1376 raise NoResults(
"No locations for get:", datasetType, dataId)
1377 self.
log.debug(
"Get type=%s keys=%s from %s", datasetType, dataId, str(location))
1379 if hasattr(location,
'bypass'):
1382 return location.bypass
1385 return self.
_read(location)
1386 if location.mapper.canStandardize(location.datasetType):
1387 innerCallback = callback
1390 return location.mapper.standardize(location.datasetType, innerCallback(), dataId)
1393 return ReadProxy(callback)
1395 def put(self, obj, datasetType, dataId={}, doBackup=False, **rest):
1396 """Persists a dataset given an output collection data id. 1401 The object to persist. 1402 datasetType - string 1403 The type of dataset to persist. 1407 If True, rename existing instead of overwriting. 1408 WARNING: Setting doBackup=True is not safe for parallel processing, as it may be subject to race 1411 Keyword arguments for the data id. 1415 dataId.update(**rest)
1417 for location
in self.
_locate(datasetType, dataId, write=
True):
1418 if isinstance(location, ButlerComposite):
1419 disassembler = location.disassembler
if location.disassembler
else genericDisassembler
1420 disassembler(obj=obj, dataId=location.dataId, componentInfo=location.componentInfo)
1421 for name, info
in location.componentInfo.items():
1422 if not info.inputOnly:
1423 self.
put(info.obj, info.datasetType, location.dataId, doBackup=doBackup)
1426 location.getRepository().backup(location.datasetType, dataId)
1427 location.getRepository().write(location, obj)
1429 def subset(self, datasetType, level=None, dataId={}, **rest):
1430 """Return complete dataIds for a dataset type that match a partial (or empty) dataId. 1432 Given a partial (or empty) dataId specified in dataId and **rest, find all datasets that match the 1433 dataId. Optionally restrict the results to a given level specified by a dataId key (e.g. visit or 1434 sensor or amp for a camera). Return an iterable collection of complete dataIds as ButlerDataRefs. 1435 Datasets with the resulting dataIds may not exist; that needs to be tested with datasetExists(). 1439 datasetType - string 1440 The type of dataset collection to subset 1442 The level of dataId at which to subset. Use an empty string if the mapper should look up the 1447 Keyword arguments for the data id. 1451 subset - ButlerSubset 1452 Collection of ButlerDataRefs for datasets matching the data id. 1456 To print the full dataIds for all r-band measurements in a source catalog 1457 (note that the subset call is equivalent to: `butler.subset('src', dataId={'filter':'r'})`): 1459 >>> subset = butler.subset('src', filter=
'r') 1460 >>> for data_ref
in subset: print(data_ref.dataId)
1462 datasetType = self._resolveDatasetTypeAlias(datasetType) 1464 # Currently expected behavior of subset is that if specified level is None then the mapper's default 1465 # level should be used. Convention for level within Butler is that an empty string is used to indicate 1470 dataId = DataId(dataId) 1471 dataId.update(**rest) 1472 return ButlerSubset(self, datasetType, level, dataId) 1474 def dataRef(self, datasetType, level=None, dataId={}, **rest): 1475 """Returns a single ButlerDataRef.
1477 Given a complete dataId specified
in dataId
and **rest, find the unique dataset at the given level
1478 specified by a dataId key (e.g. visit
or sensor
or amp
for a camera)
and return a ButlerDataRef.
1482 datasetType - string
1483 The type of dataset collection to reference
1485 The level of dataId at which to reference
1489 Keyword arguments
for the data id.
1493 dataRef - ButlerDataRef
1494 ButlerDataRef
for dataset matching the data id
1497 datasetType = self._resolveDatasetTypeAlias(datasetType) 1498 dataId = DataId(dataId) 1499 subset = self.subset(datasetType, level, dataId, **rest) 1500 if len(subset) != 1: 1501 raise RuntimeError("No unique dataset for: Dataset type:%s Level:%s Data ID:%s Keywords:%s" % 1502 (str(datasetType), str(level), str(dataId), str(rest))) 1503 return ButlerDataRef(subset, subset.cache[0]) 1505 def getUri(self, datasetType, dataId=None, write=False, **rest): 1506 """Return the URI
for a dataset
1508 .. warning:: This
is intended only
for debugging. The URI should
1509 never be used
for anything other than printing.
1511 .. note:: In the event there are multiple URIs
for read, we
return only
1514 .. note::
getUri() does
not currently support composite datasets.
1519 The dataset type of interest.
1520 dataId : `dict`, optional
1521 The data identifier.
1522 write : `bool`, optional
1523 Return the URI
for writing?
1524 rest : `dict`, optional
1525 Keyword arguments
for the data id.
1532 datasetType = self._resolveDatasetTypeAlias(datasetType) 1533 dataId = DataId(dataId) 1534 dataId.update(**rest) 1535 locations = self._locate(datasetType, dataId, write=write) 1536 if locations is None: 1537 raise NoResults("No locations for getUri: ", datasetType, dataId) 1540 # Follow the write path 1541 # Return the first valid write location. 1542 for location in locations: 1543 if isinstance(location, ButlerComposite): 1544 for name, info in location.componentInfo.items(): 1545 if not info.inputOnly: 1546 return self.getUri(info.datasetType, location.dataId, write=True) 1548 return location.getLocationsWithRoot()[0] 1549 # fall back to raise 1550 raise NoResults("No locations for getUri(write=True): ", datasetType, dataId) 1552 # Follow the read path, only return the first valid read 1553 return locations.getLocationsWithRoot()[0] 1555 def _read(self, location): 1556 """Unpersist an object using data inside a ButlerLocation
or ButlerComposite object.
1560 location : ButlerLocation
or ButlerComposite
1561 A ButlerLocation
or ButlerComposite instance populated with data needed to read the object.
1566 An instance of the object specified by the location.
1568 self.log.debug("Starting read from %s", location) 1570 if isinstance(location, ButlerComposite): 1571 for name, componentInfo in location.componentInfo.items(): 1572 if componentInfo.subset: 1573 subset = self.subset(datasetType=componentInfo.datasetType, dataId=location.dataId) 1574 componentInfo.obj = [obj.get() for obj in subset] 1576 obj = self.get(componentInfo.datasetType, location.dataId, immediate=True) 1577 componentInfo.obj = obj 1578 assembler = location.assembler or genericAssembler 1579 results = assembler(dataId=location.dataId, componentInfo=location.componentInfo, 1580 cls=location.python) 1583 results = location.repository.read(location) 1584 if len(results) == 1: 1585 results = results[0] 1586 self.log.debug("Ending read from %s", location) 1589 def __reduce__(self): 1590 ret = (_unreduce, (self._initArgs, self.datasetTypeAliasDict)) 1593 def _resolveDatasetTypeAlias(self, datasetType): 1594 """Replaces all the known alias keywords
in the given string with the alias value.
1598 datasetType - string
1599 A datasetType string to search & replace on
1603 datasetType - string
1604 The de-aliased string
1606 for key in self.datasetTypeAliasDict: 1607 # if all aliases have been replaced, bail out 1608 if datasetType.find('@') == -1: 1610 datasetType = datasetType.replace(key, self.datasetTypeAliasDict[key]) 1612 # If an alias specifier can not be resolved then throw. 1613 if datasetType.find('@') != -1: 1614 raise RuntimeError("Unresolvable alias specifier in datasetType: %s" % (datasetType)) 1619 def _unreduce(initArgs, datasetTypeAliasDict): 1620 mapperArgs = initArgs.pop('mapperArgs') 1621 initArgs.update(mapperArgs) 1622 butler = Butler(**initArgs) 1623 butler.datasetTypeAliasDict = datasetTypeAliasDict
def _buildLookupLists(self)
def _resolveDatasetTypeAlias(self, datasetType)
def datasetExists(self, datasetType, dataId={}, write=False, rest)
def _convertV1Args(self, root, mapper, mapperArgs)
def _setRepoDataTags(self)
def __init__(self, root=None, mapper=None, inputs=None, outputs=None, mapperArgs)
def setCfg(self, cfg, origin, root, isV1Repository)
def _getRepositoryCfg(self, repositoryArgs)
def getParentRepoDatas(self, context=None)
def _getCfgs(self, repoDataList)
def subset(self, datasetType, level=None, dataId={}, rest)
def __init__(self, cls, repoCfg)
def isNewRepository(self)
def _read(self, location)
def _initRepo(self, repoData)
def _setDefaultMapper(self, repoDataList)
def getUri(self, datasetType, dataId=None, write=False, rest)
def defineAlias(self, alias, datasetType)
def _connectParentRepoDatas(self, repoDataList)
def __init__(self, repoDataList)
def _addParents(self, repoDataList)
def getKeys(self, datasetType=None, level=None, tag=None)
def _getBypassFunc(location, dataId)
def put(self, obj, datasetType, dataId={}, doBackup=False, rest)
def queryMetadata(self, datasetType, format, dataId={}, rest)
def _processInputArguments(self, root=None, mapper=None, inputs=None, outputs=None, mapperArgs)
def addParentRepoData(self, parentRepoData)
def _locate(self, datasetType, dataId, write)
def _getParentVal(repoData)
def _setAndVerifyParentsLists(self, repoDataList)
def get(self, datasetType, dataId=None, immediate=True, rest)
def __init__(self, args, role)