Coverage for python / lsst / pipe / base / quantum_graph / aggregator / _config.py: 100%

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("AggregatorConfig",) 

31 

32 

33import pydantic 

34 

35 

36class AggregatorConfig(pydantic.BaseModel): 

37 """Configuration for the provenance aggregator.""" 

38 

39 output_path: str | None = None 

40 """Path for the output provenance quantum graph file. 

41 

42 At present this option is intended only for debugging. 

43 """ 

44 

45 worker_log_dir: str | None = None 

46 """Path to a directory (POSIX only) for parallel worker logs.""" 

47 

48 worker_log_level: str = "VERBOSE" 

49 """Log level for worker processes/threads. 

50 

51 Per-quantum messages only appear at ``DEBUG`` level. 

52 """ 

53 

54 worker_profile_dir: str | None = None 

55 """Path to a directory (POSIX only) for parallel worker profiling dumps. 

56 

57 This option is ignored when `n_processes` is `1`. 

58 """ 

59 

60 n_processes: int = 1 

61 """Number of processes the scanner should use.""" 

62 

63 assume_complete: bool = True 

64 """If `True`, the aggregator can assume all quanta have run to completion 

65 (including any automatic retries). If `False`, only successes can be 

66 considered final, and quanta that appear to have failed or to have not been 

67 executed are ignored. 

68 """ 

69 

70 defensive_ingest: bool = False 

71 """If `True`, guard against datasets having already been ingested into the 

72 central butler repository. 

73 

74 Defensive ingest mode is automatically turned on (with a warning emitted) 

75 if an ingest attempt fails due to a database constraint violation. Enabling 

76 defensive mode up-front avoids this warning and is slightly more efficient 

77 when it is already known that some datasets have already been ingested. 

78 

79 Defensive mode does not guard against race conditions from multiple ingest 

80 processes running simultaneously, as it relies on a one-time query to 

81 determine what is already present in the central repository. 

82 """ 

83 

84 ingest_batch_size: int = 10000 

85 """Number of butler datasets that must accumulate to trigger an ingest.""" 

86 

87 register_dataset_types: bool = True 

88 """Whether to register output dataset types in the central butler 

89 repository before starting ingest. 

90 """ 

91 

92 update_output_chain: bool = True 

93 """Whether to prepend the output `~lsst.daf.butler.CollectionType.RUN` to 

94 the output `~lsst.daf.butler.CollectionType.CHAINED` collection. 

95 """ 

96 

97 dry_run: bool = False 

98 """If `True`, do not actually perform any deletions or central butler 

99 ingests. 

100 

101 Most log messages concerning deletions and ingests will still be emitted in 

102 order to provide a better emulation of a real run. 

103 """ 

104 

105 interactive_status: bool = False 

106 """Whether to use an interactive status display with progress bars. 

107 

108 If this is `True`, the `tqdm` module must be available. If this is 

109 `False`, a periodic logger will be used to display status at a fixed 

110 interval instead (see `log_status_interval`). 

111 """ 

112 

113 log_status_interval: float | None = None 

114 """Interval (in seconds) between periodic logger status updates.""" 

115 

116 worker_sleep: float = 0.01 

117 """Time (in seconds) a worker should wait when there are no requests from 

118 the main aggregator process. 

119 """ 

120 

121 zstd_level: int = 10 

122 """ZStandard compression level to use for all compressed-JSON blocks.""" 

123 

124 zstd_dict_size: int = 32768 

125 """Size (in bytes) of the ZStandard compression dictionary.""" 

126 

127 zstd_dict_n_inputs: int = 512 

128 """Number of samples of each type (see below) to include in ZStandard 

129 compression dictionary training. 

130 

131 Training is run on a random subset of the `PredictedQuantumDatasetsModel` 

132 objects in the predicted graph, as well as the first provenance quanta, 

133 logs, and metadata blocks encountered. 

134 """ 

135 

136 mock_storage_classes: bool = False 

137 """Enable support for storage classes by created by the 

138 lsst.pipe.base.tests.mocks package. 

139 """