lsst.obs.base  20.0.0-45-g887e66a+a9aa579b69
ingestRaws.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 from lsst.daf.butler import Butler
23 from lsst.daf.butler.core.utils import findFileResources
24 from lsst.pipe.base.configOverrides import ConfigOverrides
25 from lsst.utils import doImport
26 
27 
28 def ingestRaws(repo, locations, regex, output_run, config=None, config_file=None, transfer="auto",
29  ingest_task="lsst.obs.base.RawIngestTask"):
30  """Ingests raw frames into the butler registry
31 
32  Parameters
33  ----------
34  repo : `str`
35  URI to the repository.
36  locations : `list` [`str`]
37  Files to ingest and directories to search for files that match
38  ``regex`` to ingest.
39  regex : `str`
40  Regex string used to find files in directories listed in locations.
41  output_run : `str`
42  The path to the location, the run, where datasets should be put.
43  config : `dict` [`str`, `str`] or `None`
44  Key-vaule pairs to apply as overrides to the ingest config.
45  config_file : `str` or `None`
46  Path to a config file that contains overrides to the ingest config.
47  transfer : `str` or None
48  The external data transfer type, by default "auto".
49  ingest_task : `str`
50  The fully qualified class name of the ingest task to use by default
51  lsst.obs.base.RawIngestTask.
52 
53  Raises
54  ------
55  Exception
56  Raised if operations on configuration object fail.
57  """
58  butler = Butler(repo, writeable=True)
59  TaskClass = doImport(ingest_task)
60  ingestConfig = TaskClass.ConfigClass()
61  ingestConfig.transfer = transfer
62  configOverrides = ConfigOverrides()
63  if config_file is not None:
64  configOverrides.addFileOverride(config_file)
65  if config is not None:
66  for name, value in config:
67  configOverrides.addValueOverride(name, value)
68  configOverrides.applyTo(ingestConfig)
69  ingester = TaskClass(config=ingestConfig, butler=butler)
70  files = findFileResources(locations, regex)
71  ingester.run(files, run=output_run)
lsst::utils
lsst.obs.base.script.ingestRaws.ingestRaws
def ingestRaws(repo, locations, regex, output_run, config=None, config_file=None, transfer="auto", ingest_task="lsst.obs.base.RawIngestTask")
Definition: ingestRaws.py:28