7 """Parallel version of IngestTask""" 10 return float(time)*len(parsedCmd.files)/numCores
13 def _makeArgumentParser(cls, *args, **kwargs):
14 """Build an ArgumentParser 16 Removes the batch-specific parts. 18 kwargs.pop(
"doBatch",
False)
19 kwargs.pop(
"add_help",
False)
24 """Run with a MPI process pool""" 28 args = parser.parse_args(config)
29 task = cls(config=args.config)
37 We read and ingest the files in parallel, and then 38 stuff the registry database in serial. 43 infoList = pool.map(self.
runFile, filenameList,
None, args)
47 context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
48 with context
as registry:
49 for hduInfoList
in infoList:
50 if hduInfoList
is None:
52 for info
in hduInfoList:
53 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
def expandFiles(self, fileNameList)
def runFile(self, infile, registry, args, pos)
def parseAndRun(cls, args, kwargs)
def writeMetadata(self, args, kwargs)
def writeConfig(self, args, kwargs)
def batchWallTime(cls, time, parsedCmd, numCores)