Coverage for python / lsst / daf / butler / script / _pruneDatasets.py: 33%
86 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-12-19 01:11 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29import itertools
30from collections.abc import Callable, Iterable
31from enum import Enum, auto
32from typing import TYPE_CHECKING, Any
34from .._butler import Butler
35from .._collection_type import CollectionType
36from .queryDatasets import QueryDatasets
38if TYPE_CHECKING:
39 from astropy.table import Table
42class PruneDatasetsResult:
43 """Contains the results of a prune-datasets action.
45 The action may not be complete if the caller requested a confirmation, in
46 which case calling ``onConfirmation`` will perform the action.
48 Parameters
49 ----------
50 tables : `list` [`astropy.table.Table`], optional
51 The astropy tables that will be or were deleted, by default None.
52 state : `PruneDatasetsResult.State`, optional
53 The initial state of execution of the action, if `None` the result
54 state is ``INIT``, by default `None`.
55 errDict : `dict` [`str`, `str`] or `None`
56 Place to store error messages. Will be created if not given.
58 Attributes
59 ----------
60 tables
61 Same as in Parameters.
62 state : ``PruneDatasetsResult.State``
63 The current state of the action.
64 onConfirmation : `Callable[None, None]`
65 The function to call to perform the action if the caller wants to
66 confirm the tables before performing the action.
67 """
69 action: dict[str, Any] | None
70 onConfirmation: Callable | None
72 class State(Enum):
73 """State associated with dataset pruning request."""
75 INIT = auto()
76 DRY_RUN_COMPLETE = auto()
77 AWAITING_CONFIRMATION = auto()
78 FINISHED = auto()
79 ERR_PURGE_AND_DISASSOCIATE = auto()
80 ERR_NO_COLLECTION_RESTRICTION = auto()
81 ERR_PRUNE_ON_NOT_RUN = auto()
82 ERR_NO_OP = auto()
84 def __init__(
85 self,
86 tables: list[Table] | None = None,
87 state: State | None = None,
88 errDict: dict[str, str] | None = None,
89 ):
90 self.state = state or self.State.INIT
91 if tables is None:
92 tables = []
93 self.tables = tables
94 self.onConfirmation = None
95 # Action describes the removal action for dry-run, will be a dict with
96 # keys disassociate, unstore, purge, and collections.
97 self.action = None
98 # errDict is a container for variables related to the error that may be
99 # substituted into a user-visible string.
100 self.errDict = errDict or {}
102 @property
103 def dryRun(self) -> bool:
104 return self.state is self.State.DRY_RUN_COMPLETE
106 @property
107 def confirm(self) -> bool:
108 return self.state is self.State.AWAITING_CONFIRMATION
110 @property
111 def finished(self) -> bool:
112 return self.state is self.State.FINISHED
114 @property
115 def errPurgeAndDisassociate(self) -> bool:
116 return self.state is self.State.ERR_PURGE_AND_DISASSOCIATE
118 @property
119 def errNoCollectionRestriction(self) -> bool:
120 return self.state is self.State.ERR_NO_COLLECTION_RESTRICTION
122 @property
123 def errPruneOnNotRun(self) -> bool:
124 return self.state is self.State.ERR_PRUNE_ON_NOT_RUN
126 @property
127 def errNoOp(self) -> bool:
128 return self.state is self.State.ERR_NO_OP
131def pruneDatasets(
132 repo: str,
133 collections: Iterable[str],
134 datasets: Iterable[str],
135 where: str,
136 disassociate_tags: Iterable[str],
137 unstore: bool,
138 purge_run: str,
139 dry_run: bool,
140 confirm: bool,
141 find_all: bool,
142) -> PruneDatasetsResult:
143 """Prune datasets from a repository.
145 Parameters
146 ----------
147 repo : `str`
148 URI to the location of the repo or URI to a config file describing the
149 repo and its location.
150 collections : iterable [`str`]
151 A list of glob-style search string that identify the collections to
152 search for.
153 datasets : iterable [`str`]
154 A list of glob-style search string that identify the dataset type names
155 to search for.
156 where : `str`
157 A string expression similar to a SQL WHERE clause. May involve any
158 column of a dimension table or (as a shortcut for the primary key
159 column of a dimension table) dimension name.
160 disassociate_tags : `list` [`str`]
161 TAGGED collections to disassociate the datasets from. If not `None`
162 then ``purge_run`` must be `None`.
163 unstore : `bool`
164 Same as the unstore argument to ``Butler.pruneDatasets``.
165 purge_run : `str`
166 Completely remove datasets from the ``Registry``. Note that current
167 implementation accepts any RUN-type collection, but will remove
168 datasets from all collections in ``collections`` if it is non-empty.
169 dry_run : `bool`
170 Get results for what would be removed but do not remove.
171 confirm : `bool`
172 Get results for what would be removed and return the results for
173 display & confirmation, with a completion function to run after
174 confirmation.
175 find_all : `bool`
176 If False, for each result data ID, will only delete the dataset from
177 the first collection in which a dataset of that dataset type appears
178 (according to the order of ``collections`` passed in). If used,
179 ``collections`` must specify at least one expression and must not
180 contain wildcards. This is the inverse of ``QueryDataset``'s find_first
181 option.
183 Notes
184 -----
185 The matrix of legal & illegal combinations of purge, unstore, and
186 disassociate is this:
187 - none of (purge, unstore, disassociate): error, nothing to do
188 - purge only: ok
189 - unstore only: ok
190 - disassociate only: ok
191 - purge+unstore: ok, just ignore unstore (purge effectively implies
192 unstore)
193 - purge+disassociate: this is an error (instead of ignoring disassociate),
194 because that comes with a collection argument that we can't respect, and
195 that might be confusing (purge will disassociate from all TAGGED
196 collections, not just the one given)
197 - purge+unstore+disassociate: an error, for the same reason as just
198 purge+disassociate
199 - unstore+disassociate: ok; these operations are unrelated to each other
201 Returns
202 -------
203 results : `PruneDatasetsResult`
204 A data structure that contains information about datasets for removal,
205 removal status, and options to continue in some cases.
206 """
207 if not disassociate_tags and not unstore and not purge_run:
208 return PruneDatasetsResult(state=PruneDatasetsResult.State.ERR_NO_OP)
210 if disassociate_tags and purge_run:
211 return PruneDatasetsResult(state=PruneDatasetsResult.State.ERR_PURGE_AND_DISASSOCIATE)
213 # If collections is not specified and a purge_run is, use the purge_run for
214 # collections, or if disassociate_tags is then use that.
215 if not collections:
216 if purge_run:
217 collections = (purge_run,)
218 elif disassociate_tags:
219 collections = disassociate_tags
221 if not collections:
222 return PruneDatasetsResult(state=PruneDatasetsResult.State.ERR_NO_COLLECTION_RESTRICTION)
224 # If purging, verify that the collection to purge is RUN type collection.
225 with Butler.from_config(repo, without_datastore=True) as butler:
226 if purge_run:
227 collection_info = butler.collections.get_info(purge_run)
228 if collection_info.type is not CollectionType.RUN:
229 return PruneDatasetsResult(
230 state=PruneDatasetsResult.State.ERR_PRUNE_ON_NOT_RUN, errDict=dict(collection=purge_run)
231 )
233 datasets_found = QueryDatasets(
234 butler=butler,
235 glob=datasets,
236 collections=collections,
237 where=where,
238 # By default we want find_first to be True if collections are
239 # provided
240 # (else False) (find_first requires collections to be provided).
241 # But the user may specify that they want to find all (thus forcing
242 # find_first to be False)
243 find_first=not find_all,
244 show_uri=False,
245 )
246 dataset_refs = list(itertools.chain.from_iterable(datasets_found.getDatasets()))
248 result = PruneDatasetsResult(list(datasets_found.getTables()))
250 disassociate = bool(disassociate_tags) or bool(purge_run)
251 purge = bool(purge_run)
252 unstore = unstore or bool(purge_run)
254 if dry_run:
255 result.state = PruneDatasetsResult.State.DRY_RUN_COMPLETE
256 result.action = dict(disassociate=disassociate, purge=purge, unstore=unstore, collections=collections)
257 return result
259 def doPruneDatasets() -> PruneDatasetsResult:
260 with Butler.from_config(repo, writeable=True) as butler:
261 butler.pruneDatasets(
262 refs=dataset_refs,
263 disassociate=disassociate,
264 tags=disassociate_tags or (),
265 purge=purge,
266 unstore=unstore,
267 )
268 result.state = PruneDatasetsResult.State.FINISHED
269 return result
271 if confirm:
272 result.state = PruneDatasetsResult.State.AWAITING_CONFIRMATION
273 result.onConfirmation = doPruneDatasets
274 return result
276 return doPruneDatasets()