Coverage for python/lsst/obs/base/script/ingestRaws.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from lsst.daf.butler import Butler
23from lsst.pipe.base.configOverrides import ConfigOverrides
24from lsst.utils import doImport
27def ingestRaws(
28 repo,
29 locations,
30 regex,
31 output_run,
32 config=None,
33 config_file=None,
34 transfer="auto",
35 processes=1,
36 ingest_task="lsst.obs.base.RawIngestTask",
37 track_file_attrs=True,
38):
39 """Ingests raw frames into the butler registry
41 Parameters
42 ----------
43 repo : `str`
44 URI to the repository.
45 locations : `list` [`str`]
46 Files to ingest and directories to search for files that match
47 ``regex`` to ingest.
48 regex : `str`
49 Regex string used to find files in directories listed in locations.
50 output_run : `str`
51 The path to the location, the run, where datasets should be put.
52 config : `dict` [`str`, `str`] or `None`
53 Key-value pairs to apply as overrides to the ingest config.
54 config_file : `str` or `None`
55 Path to a config file that contains overrides to the ingest config.
56 transfer : `str` or None
57 The external data transfer type, by default "auto".
58 processess : `int`
59 Number of processes to use for ingest.
60 ingest_task : `str`
61 The fully qualified class name of the ingest task to use by default
62 lsst.obs.base.RawIngestTask.
63 track_file_attrs : `bool`, optional
64 Control whether file attributes such as the size or checksum should
65 be tracked by the datastore. Whether this parameter is honored
66 depends on the specific datastore implentation.
68 Raises
69 ------
70 Exception
71 Raised if operations on configuration object fail.
72 """
73 butler = Butler(repo, writeable=True)
74 TaskClass = doImport(ingest_task)
75 ingestConfig = TaskClass.ConfigClass()
76 ingestConfig.transfer = transfer
77 configOverrides = ConfigOverrides()
78 if config_file is not None:
79 configOverrides.addFileOverride(config_file)
80 if config is not None:
81 for name, value in config.items():
82 configOverrides.addValueOverride(name, value)
83 configOverrides.applyTo(ingestConfig)
84 ingester = TaskClass(config=ingestConfig, butler=butler)
85 ingester.run(
86 locations, run=output_run, processes=processes, file_filter=regex, track_file_attrs=track_file_attrs
87 )