3from lsst.pipe.base
import Struct
4from lsst.pipe.tasks.ingest
import IngestTask, IngestError
8 """Parallel version of IngestTask"""
11 return float(time)*len(parsedCmd.files)/numCores
14 def _makeArgumentParser(cls, *args, **kwargs):
15 """Build an ArgumentParser
17 Removes the batch-specific parts.
19 kwargs.pop("doBatch",
False)
20 kwargs.pop(
"add_help",
False)
21 return cls.ArgumentParser(*args, name=
"ingest", **kwargs)
25 """Run with a MPI process pool"""
27 config = cls.ConfigClass()
28 parser = cls.ArgumentParser(name=cls._DefaultName)
29 args = parser.parse_args(config)
30 task = cls(config=args.config)
35 """Run ingest on one file
37 This is a wrapper method
for calling ``runFile``.
41 struct : `lsst.pipe.base.Struct`
42 Structure containing ``filename`` (`str`)
and ``position`` (`int`).
43 args : `argparse.Namespace`
44 Parsed command-line arguments.
48 hduInfoList : `list` of `dict`
49 Parsed information
from FITS HDUs,
or ``
None``.
51 filename = struct.filename
52 position = struct.position
54 return self.runFile(filename,
None, args, position)
55 except IngestError
as exc:
56 self.log.warn(f
"Unable to ingest {filename}: {exc}")
63 We read and ingest the files
in parallel,
and then
64 stuff the registry database
in serial.
68 filenameList = self.expandFiles(args.files)
69 dataList = [Struct(filename=filename, position=ii)
for ii, filename
in enumerate(filenameList)]
74 context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
75 with context
as registry:
76 for hduInfoList
in infoList:
77 if hduInfoList
is None:
79 for info
in hduInfoList:
80 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
def runFileWrapper(self, struct, args)
def writeConfig(self, *args, **kwargs)
def batchWallTime(cls, time, parsedCmd, numCores)
def parseAndRun(cls, *args, **kwargs)
def writeMetadata(self, *args, **kwargs)