Coverage for python / lsst / pipe / base / quantum_graph / aggregator / _config.py: 100%
40 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-20 01:37 -0800
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("AggregatorConfig",)
33import pydantic
36class AggregatorConfig(pydantic.BaseModel):
37 """Configuration for the provenance aggregator."""
39 output_path: str | None = None
40 """Path for the output provenance quantum graph file.
42 At present this option is intended only for debugging.
43 """
45 worker_log_dir: str | None = None
46 """Path to a directory (POSIX only) for parallel worker logs."""
48 worker_log_level: str = "VERBOSE"
49 """Log level for worker processes/threads.
51 Per-quantum messages only appear at ``DEBUG`` level.
52 """
54 worker_profile_dir: str | None = None
55 """Path to a directory (POSIX only) for parallel worker profiling dumps.
57 This option is ignored when `n_processes` is `1`.
58 """
60 n_processes: int = 1
61 """Number of processes the scanner should use."""
63 assume_complete: bool = True
64 """If `True`, the aggregator can assume all quanta have run to completion
65 (including any automatic retries). If `False`, only successes can be
66 considered final, and quanta that appear to have failed or to have not been
67 executed are ignored.
68 """
70 defensive_ingest: bool = False
71 """If `True`, guard against datasets having already been ingested into the
72 central butler repository.
74 Defensive ingest mode is automatically turned on (with a warning emitted)
75 if an ingest attempt fails due to a database constraint violation. Enabling
76 defensive mode up-front avoids this warning and is slightly more efficient
77 when it is already known that some datasets have already been ingested.
79 Defensive mode does not guard against race conditions from multiple ingest
80 processes running simultaneously, as it relies on a one-time query to
81 determine what is already present in the central repository.
82 """
84 ingest_batch_size: int = 10000
85 """Number of butler datasets that must accumulate to trigger an ingest."""
87 register_dataset_types: bool = True
88 """Whether to register output dataset types in the central butler
89 repository before starting ingest.
90 """
92 update_output_chain: bool = True
93 """Whether to prepend the output `~lsst.daf.butler.CollectionType.RUN` to
94 the output `~lsst.daf.butler.CollectionType.CHAINED` collection.
95 """
97 dry_run: bool = False
98 """If `True`, do not actually perform any deletions or central butler
99 ingests.
101 Most log messages concerning deletions and ingests will still be emitted in
102 order to provide a better emulation of a real run.
103 """
105 interactive_status: bool = False
106 """Whether to use an interactive status display with progress bars.
108 If this is `True`, the `tqdm` module must be available. If this is
109 `False`, a periodic logger will be used to display status at a fixed
110 interval instead (see `log_status_interval`).
111 """
113 log_status_interval: float | None = None
114 """Interval (in seconds) between periodic logger status updates."""
116 worker_sleep: float = 0.01
117 """Time (in seconds) a worker should wait when there are no requests from
118 the main aggregator process.
119 """
121 zstd_level: int = 10
122 """ZStandard compression level to use for all compressed-JSON blocks."""
124 zstd_dict_size: int = 32768
125 """Size (in bytes) of the ZStandard compression dictionary."""
127 zstd_dict_n_inputs: int = 512
128 """Number of samples of each type (see below) to include in ZStandard
129 compression dictionary training.
131 Training is run on a random subset of the `PredictedQuantumDatasetsModel`
132 objects in the predicted graph, as well as the first provenance quanta,
133 logs, and metadata blocks encountered.
134 """
136 mock_storage_classes: bool = False
137 """Enable support for storage classes by created by the
138 lsst.pipe.base.tests.mocks package.
139 """