Coverage for python / lsst / obs / base / script / ingestRaws.py: 17%

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from lsst.daf.butler import Butler 

23from lsst.obs.base.ingest import RawIngestTask 

24from lsst.pipe.base.configOverrides import ConfigOverrides 

25from lsst.utils import doImportType 

26 

27 

28def ingestRaws( 

29 repo: str, 

30 locations: list[str], 

31 regex: str, 

32 output_run: str, 

33 fail_fast: bool = False, 

34 config: dict[str, str] | None = None, 

35 config_file: str | None = None, 

36 transfer: str | None = "auto", 

37 processes: int = 1, 

38 ingest_task: str = "lsst.obs.base.RawIngestTask", 

39 track_file_attrs: bool = True, 

40 update_records: bool = False, 

41 skip_existing: bool = False, 

42 search_indexes: bool = True, 

43) -> None: 

44 """Ingest raw frames into the butler registry. 

45 

46 Parameters 

47 ---------- 

48 repo : `str` 

49 URI to the repository. 

50 locations : `list` [`str`] 

51 Files to ingest and directories to search for files that match 

52 ``regex`` to ingest. 

53 regex : `str` 

54 Regex string used to find files in directories listed in locations. 

55 output_run : `str` 

56 The path to the location, the run, where datasets should be put. 

57 fail_fast : `bool` 

58 If True, stop ingest as soon as any problem is encountered with any 

59 file. Otherwise problem files will be skipped and logged and a report 

60 issued at completion. 

61 config : `dict` [`str`, `str`] or `None` 

62 Key-value pairs to apply as overrides to the ingest config. 

63 config_file : `str` or `None` 

64 Path to a config file that contains overrides to the ingest config. 

65 transfer : `str` or None 

66 The external data transfer type, by default "auto". 

67 processes : `int` 

68 Number of workers to use for ingest. 

69 ingest_task : `str` 

70 The fully qualified class name of the ingest task to use by default 

71 lsst.obs.base.RawIngestTask. 

72 track_file_attrs : `bool`, optional 

73 Control whether file attributes such as the size or checksum should 

74 be tracked by the datastore. Whether this parameter is honored 

75 depends on the specific datastore implementation. 

76 update_records : `bool`, optional 

77 Control whether recalculated exposure definitions will be accepted or 

78 not. 

79 skip_existing : `bool`, optional 

80 Control whether raws that are already in the Butler repo will be 

81 skipped without error. 

82 search_indexes : `bool`, optional 

83 Control whether raw ingest will search for per-directory index files 

84 or not. Disabling this can improve performance if you know that there 

85 are no indexes. 

86 

87 Raises 

88 ------ 

89 Exception 

90 Raised if operations on configuration object fail. 

91 """ 

92 TaskClass = doImportType(ingest_task) 

93 assert issubclass(TaskClass, RawIngestTask) 

94 ingestConfig = TaskClass.ConfigClass() 

95 ingestConfig.transfer = transfer 

96 configOverrides = ConfigOverrides() 

97 if config_file is not None: 

98 configOverrides.addFileOverride(config_file) 

99 if config is not None: 

100 for name, value in config.items(): 

101 configOverrides.addValueOverride(name, value) 

102 if fail_fast: 

103 configOverrides.addValueOverride("failFast", True) 

104 configOverrides.applyTo(ingestConfig) 

105 with Butler.from_config(repo, writeable=True) as butler: 

106 ingester = TaskClass(config=ingestConfig, butler=butler) # type: ignore[arg-type] 

107 ingester.run( 

108 locations, 

109 run=output_run, 

110 num_workers=processes, 

111 file_filter=regex, 

112 track_file_attrs=track_file_attrs, 

113 update_exposure_records=update_records, 

114 skip_existing_exposures=skip_existing, 

115 search_indexes=search_indexes, 

116 )