lsst.pipe.drivers  20.0.0-1-g5b95a8c+42727149f1
ingestDriver.py
Go to the documentation of this file.
1 from lsst.ctrl.pool.pool import Pool, startPool, abortOnError
2 from lsst.ctrl.pool.parallel import BatchCmdLineTask
3 from lsst.pipe.base import Struct
4 from lsst.pipe.tasks.ingest import IngestTask, IngestError
5 
6 
8  """Parallel version of IngestTask"""
9  @classmethod
10  def batchWallTime(cls, time, parsedCmd, numCores):
11  return float(time)*len(parsedCmd.files)/numCores
12 
13  @classmethod
14  def _makeArgumentParser(cls, *args, **kwargs):
15  """Build an ArgumentParser
16 
17  Removes the batch-specific parts.
18  """
19  kwargs.pop("doBatch", False)
20  kwargs.pop("add_help", False)
21  return cls.ArgumentParser(*args, name="ingest", **kwargs)
22 
23  @classmethod
24  def parseAndRun(cls, *args, **kwargs):
25  """Run with a MPI process pool"""
26  pool = startPool()
27  config = cls.ConfigClass()
28  parser = cls.ArgumentParser(name=cls._DefaultName)
29  args = parser.parse_args(config)
30  task = cls(config=args.config)
31  task.run(args)
32  pool.exit()
33 
34  def runFileWrapper(self, struct, args):
35  """Run ingest on one file
36 
37  This is a wrapper method for calling ``runFile``.
38 
39  Parameters
40  ----------
41  struct : `lsst.pipe.base.Struct`
42  Structure containing ``filename`` (`str`) and ``position`` (`int`).
43  args : `argparse.Namespace`
44  Parsed command-line arguments.
45 
46  Returns
47  -------
48  hduInfoList : `list` of `dict`
49  Parsed information from FITS HDUs, or ``None``.
50  """
51  filename = struct.filename
52  position = struct.position
53  try:
54  return self.runFile(filename, None, args, position)
55  except IngestError as exc:
56  self.log.warn(f"Unable to ingest {filename}: {exc}")
57  return None
58 
59  @abortOnError
60  def run(self, args):
61  """Run ingest
62 
63  We read and ingest the files in parallel, and then
64  stuff the registry database in serial.
65  """
66  # Parallel
67  pool = Pool(None)
68  filenameList = self.expandFiles(args.files)
69  dataList = [Struct(filename=filename, position=ii) for ii, filename in enumerate(filenameList)]
70  infoList = pool.map(self.runFileWrapper, dataList, args)
71 
72  # Serial
73  root = args.input
74  context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
75  with context as registry:
76  for hduInfoList in infoList:
77  if hduInfoList is None:
78  continue
79  for info in hduInfoList:
80  self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
81 
82  def writeConfig(self, *args, **kwargs):
83  pass
84 
85  def writeMetadata(self, *args, **kwargs):
86  pass
lsst.pipe.drivers.ingestDriver.PoolIngestTask
Definition: ingestDriver.py:7
lsst.pipe::tasks::ingest::IngestTask::expandFiles
def expandFiles(self, fileNameList)
lsst.pipe::tasks::ingest::IngestTask
lsst::ctrl::pool::parallel
lsst.pipe.drivers.ingestDriver.PoolIngestTask.writeConfig
def writeConfig(self, *args, **kwargs)
Definition: ingestDriver.py:82
lsst.pipe.drivers.ingestDriver.PoolIngestTask.batchWallTime
def batchWallTime(cls, time, parsedCmd, numCores)
Definition: ingestDriver.py:10
lsst.pipe::tasks::ingest::IngestTask::runFile
def runFile(self, infile, registry, args, pos)
lsst.pipe.drivers.ingestDriver.PoolIngestTask.writeMetadata
def writeMetadata(self, *args, **kwargs)
Definition: ingestDriver.py:85
lsst.pipe::tasks::ingest
lsst.pipe::tasks::ingest::IngestTask::ArgumentParser
ArgumentParser
lsst::ctrl::pool::parallel::BatchCmdLineTask
lsst.pipe.drivers.ingestDriver.PoolIngestTask.parseAndRun
def parseAndRun(cls, *args, **kwargs)
Definition: ingestDriver.py:24
lsst::ctrl::pool::pool
lsst::ctrl::pool::pool::Pool
lsst.pipe.drivers.ingestDriver.PoolIngestTask.run
def run(self, args)
Definition: ingestDriver.py:60
lsst.pipe.drivers.ingestDriver.PoolIngestTask.runFileWrapper
def runFileWrapper(self, struct, args)
Definition: ingestDriver.py:34
lsst.pipe::base
lsst.pipe::tasks::ingest::IngestTask::ConfigClass
ConfigClass