lsst.pipe.drivers gef6abdcebb+207b46c02f
ingestDriver.py
Go to the documentation of this file.
1from lsst.ctrl.pool.pool import Pool, startPool, abortOnError
2from lsst.ctrl.pool.parallel import BatchCmdLineTask
3from lsst.pipe.base import Struct
4from lsst.pipe.tasks.ingest import IngestTask, IngestError
5
6
8 """Parallel version of IngestTask"""
9 @classmethod
10 def batchWallTime(cls, time, parsedCmd, numCores):
11 return float(time)*len(parsedCmd.files)/numCores
12
13 @classmethod
14 def _makeArgumentParser(cls, *args, **kwargs):
15 """Build an ArgumentParser
16
17 Removes the batch-specific parts.
18 """
19 kwargs.pop("doBatch", False)
20 kwargs.pop("add_help", False)
21 return cls.ArgumentParserArgumentParser(*args, name="ingest", **kwargs)
22
23 @classmethod
24 def parseAndRun(cls, *args, **kwargs):
25 """Run with a MPI process pool"""
26 pool = startPool()
27 config = cls.ConfigClassConfigClass()
28 parser = cls.ArgumentParserArgumentParser(name=cls._DefaultName)
29 args = parser.parse_args(config)
30 task = cls(config=args.config)
31 task.run(args)
32 pool.exit()
33
34 def runFileWrapper(self, struct, args):
35 """Run ingest on one file
36
37 This is a wrapper method for calling ``runFile``.
38
39 Parameters
40 ----------
41 struct : `lsst.pipe.base.Struct`
42 Structure containing ``filename`` (`str`) and ``position`` (`int`).
43 args : `argparse.Namespace`
44 Parsed command-line arguments.
45
46 Returns
47 -------
48 hduInfoList : `list` of `dict`
49 Parsed information from FITS HDUs, or ``None``.
50 """
51 filename = struct.filename
52 position = struct.position
53 try:
54 return self.runFilerunFile(filename, None, args, position)
55 except IngestError as exc:
56 self.log.warn(f"Unable to ingest {filename}: {exc}")
57 return None
58
59 @abortOnError
60 def run(self, args):
61 """Run ingest
62
63 We read and ingest the files in parallel, and then
64 stuff the registry database in serial.
65 """
66 # Parallel
67 pool = Pool(None)
68 filenameList = self.expandFilesexpandFiles(args.files)
69 dataList = [Struct(filename=filename, position=ii) for ii, filename in enumerate(filenameList)]
70 infoList = pool.map(self.runFileWrapperrunFileWrapper, dataList, args)
71
72 # Serial
73 root = args.input
74 context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
75 with context as registry:
76 for hduInfoList in infoList:
77 if hduInfoList is None:
78 continue
79 for info in hduInfoList:
80 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
81
82 def writeConfig(self, *args, **kwargs):
83 pass
84
85 def writeMetadata(self, *args, **kwargs):
86 pass
def writeConfig(self, *args, **kwargs)
Definition: ingestDriver.py:82
def batchWallTime(cls, time, parsedCmd, numCores)
Definition: ingestDriver.py:10
def writeMetadata(self, *args, **kwargs)
Definition: ingestDriver.py:85
def expandFiles(self, fileNameList)
def runFile(self, infile, registry, args, pos)