lsst.obs.base  20.0.0-74-g0218c7a+2b74d6f6a8
ingestRaws.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 from lsst.daf.butler import Butler
23 from lsst.daf.butler.core.utils import findFileResources
24 from lsst.pipe.base.configOverrides import ConfigOverrides
25 from lsst.utils import doImport
26 
27 
28 def ingestRaws(repo, locations, regex, output_run, config=None, config_file=None, transfer="auto",
29  processes=1, ingest_task="lsst.obs.base.RawIngestTask"):
30  """Ingests raw frames into the butler registry
31 
32  Parameters
33  ----------
34  repo : `str`
35  URI to the repository.
36  locations : `list` [`str`]
37  Files to ingest and directories to search for files that match
38  ``regex`` to ingest.
39  regex : `str`
40  Regex string used to find files in directories listed in locations.
41  output_run : `str`
42  The path to the location, the run, where datasets should be put.
43  config : `dict` [`str`, `str`] or `None`
44  Key-vaule pairs to apply as overrides to the ingest config.
45  config_file : `str` or `None`
46  Path to a config file that contains overrides to the ingest config.
47  transfer : `str` or None
48  The external data transfer type, by default "auto".
49  processess : `int`
50  Number of processes to use for ingest.
51  ingest_task : `str`
52  The fully qualified class name of the ingest task to use by default
53  lsst.obs.base.RawIngestTask.
54 
55  Raises
56  ------
57  Exception
58  Raised if operations on configuration object fail.
59  """
60  butler = Butler(repo, writeable=True)
61  TaskClass = doImport(ingest_task)
62  ingestConfig = TaskClass.ConfigClass()
63  ingestConfig.transfer = transfer
64  configOverrides = ConfigOverrides()
65  if config_file is not None:
66  configOverrides.addFileOverride(config_file)
67  if config is not None:
68  for name, value in config:
69  configOverrides.addValueOverride(name, value)
70  configOverrides.applyTo(ingestConfig)
71  ingester = TaskClass(config=ingestConfig, butler=butler)
72  files = findFileResources(locations, regex)
73  ingester.run(files, run=output_run, processes=processes)
lsst.obs.base.script.ingestRaws.ingestRaws
def ingestRaws(repo, locations, regex, output_run, config=None, config_file=None, transfer="auto", processes=1, ingest_task="lsst.obs.base.RawIngestTask")
Definition: ingestRaws.py:28
lsst::utils