Coverage for python / lsst / daf / butler / cli / opt / options.py: 84%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "CollectionTypeCallback", 

31 "collection_type_option", 

32 "collections_option", 

33 "config_file_option", 

34 "config_option", 

35 "confirm_option", 

36 "dataset_type_option", 

37 "datasets_option", 

38 "limit_option", 

39 "log_file_option", 

40 "log_label_option", 

41 "log_level_option", 

42 "log_tty_option", 

43 "long_log_option", 

44 "offset_option", 

45 "options_file_option", 

46 "order_by_option", 

47 "processes_option", 

48 "regex_option", 

49 "register_dataset_types_option", 

50 "run_option", 

51 "track_file_attrs_option", 

52 "transfer_dimensions_option", 

53 "transfer_option", 

54 "transfer_option_no_short", 

55 "verbose_option", 

56 "where_option", 

57) 

58 

59from functools import partial 

60from typing import Any 

61 

62import click 

63 

64from lsst.daf.butler import CollectionType 

65 

66from ..cliLog import CliLog 

67from ..utils import MWOptionDecorator, MWPath, split_commas, split_kv, unwrap, yaml_presets 

68 

69 

70class CollectionTypeCallback: 

71 """Helper class for handling different collection types.""" 

72 

73 collectionTypes = tuple(collectionType.name for collectionType in CollectionType.all()) 

74 

75 @staticmethod 

76 def makeCollectionTypes( 

77 context: click.Context, param: click.Option, value: tuple[str, ...] | str 

78 ) -> tuple[CollectionType, ...]: 

79 if not value: 

80 # Click seems to demand that the default be an empty tuple, rather 

81 # than a sentinal like None. The behavior that we want is that 

82 # not passing this option at all passes all collection types, while 

83 # passing it uses only the passed collection types. That works 

84 # fine for now, since there's no command-line option to subtract 

85 # collection types, and hence the only way to get an empty tuple 

86 # is as the default. 

87 return tuple(CollectionType.all()) 

88 

89 return tuple(CollectionType.from_name(item) for item in split_commas(context, param, value)) 

90 

91 

92collection_type_option = MWOptionDecorator( 

93 "--collection-type", 

94 callback=CollectionTypeCallback.makeCollectionTypes, 

95 multiple=True, 

96 help="If provided, only list collections of this type.", 

97 type=click.Choice(choices=CollectionTypeCallback.collectionTypes, case_sensitive=False), 

98) 

99 

100 

101collections_option = MWOptionDecorator( 

102 "--collections", 

103 help=unwrap( 

104 """One or more expressions that fully or partially identify 

105 the collections to search for datasets. If not provided all 

106 datasets are returned.""" 

107 ), 

108 multiple=True, 

109 callback=split_commas, 

110) 

111 

112 

113def _config_split(*args: Any) -> dict[str | None, str]: 

114 # Config values might include commas so disable comma-splitting. 

115 result = split_kv(*args, multiple=False) 

116 assert isinstance(result, dict), "For mypy check that we get the expected result" 

117 return result 

118 

119 

120config_option = MWOptionDecorator( 

121 "-c", 

122 "--config", 

123 callback=_config_split, 

124 help="Config override, as a key-value pair.", 

125 metavar="TEXT=TEXT", 

126 multiple=True, 

127) 

128 

129 

130config_file_option = MWOptionDecorator( 

131 "-C", 

132 "--config-file", 

133 help=unwrap( 

134 """Path to a pex config override to be included after the 

135 Instrument config overrides are applied.""" 

136 ), 

137) 

138 

139 

140confirm_option = MWOptionDecorator( 

141 "--confirm/--no-confirm", 

142 default=True, 

143 help="Print expected action and a confirmation prompt before executing. Default is --confirm.", 

144) 

145 

146 

147dataset_type_option = MWOptionDecorator( 

148 "-d", "--dataset-type", callback=split_commas, help="Specific DatasetType(s) to validate.", multiple=True 

149) 

150 

151 

152datasets_option = MWOptionDecorator("--datasets") 

153 

154 

155logLevelChoices = ["CRITICAL", "ERROR", "WARNING", "INFO", "VERBOSE", "DEBUG", "TRACE"] 

156log_level_option = MWOptionDecorator( 

157 "--log-level", 

158 callback=partial( 

159 split_kv, 

160 choice=click.Choice(choices=logLevelChoices, case_sensitive=False), 

161 normalize=True, 

162 unseparated_okay=True, 

163 add_to_default=True, 

164 default_key=None, # No separator 

165 ), 

166 help=f"The logging level. Without an explicit logger name, will only affect the default root loggers " 

167 f"({', '.join(CliLog.root_loggers())}). To modify the root logger use '.=LEVEL'. " 

168 f"Supported levels are [{'|'.join(logLevelChoices)}]", 

169 is_eager=True, 

170 metavar="LEVEL|COMPONENT=LEVEL", 

171 multiple=True, 

172) 

173 

174 

175long_log_option = MWOptionDecorator( 

176 "--long-log", help="Make log messages appear in long format.", is_flag=True 

177) 

178 

179log_file_option = MWOptionDecorator( 

180 "--log-file", 

181 default=None, 

182 multiple=True, 

183 callback=split_commas, 

184 type=MWPath(file_okay=True, dir_okay=False, writable=True), 

185 help="File(s) to write log messages. If the path ends with '.json' then" 

186 " JSON log records will be written, else formatted text log records" 

187 " will be written. This file can exist and records will be appended.", 

188) 

189 

190log_label_option = MWOptionDecorator( 

191 "--log-label", 

192 default=None, 

193 multiple=True, 

194 callback=split_kv, 

195 type=str, 

196 help="Keyword=value pairs to add to MDC of log records.", 

197) 

198 

199log_tty_option = MWOptionDecorator( 

200 "--log-tty/--no-log-tty", 

201 default=True, 

202 help="Log to terminal (default). If false logging to terminal is disabled.", 

203) 

204 

205options_file_option = MWOptionDecorator( 

206 "--options-file", 

207 "-@", 

208 expose_value=False, # This option should not be forwarded 

209 help=unwrap( 

210 """URI to YAML file containing overrides 

211 of command line options. The YAML should be organized 

212 as a hierarchy with subcommand names at the top 

213 level options for that subcommand below.""" 

214 ), 

215 callback=yaml_presets, 

216) 

217 

218 

219processes_option = MWOptionDecorator( 

220 "-j", "--processes", default=1, help="Number of processes to use.", type=click.IntRange(min=1) 

221) 

222 

223 

224regex_option = MWOptionDecorator("--regex") 

225 

226 

227register_dataset_types_option = MWOptionDecorator( 

228 "--register-dataset-types", 

229 help="Register DatasetTypes that do not already exist in the Registry.", 

230 is_flag=True, 

231) 

232 

233run_option = MWOptionDecorator("--output-run", help="The name of the run datasets should be output to.") 

234 

235_transfer_params = dict( 

236 default="auto", # set to `None` if using `required=True` 

237 help="The external data transfer mode.", 

238 type=click.Choice( 

239 choices=["auto", "link", "symlink", "hardlink", "copy", "move", "relsymlink", "direct"], 

240 case_sensitive=False, 

241 ), 

242) 

243 

244transfer_option_no_short = MWOptionDecorator( 

245 "--transfer", 

246 **_transfer_params, 

247) 

248 

249transfer_option = MWOptionDecorator( 

250 "-t", 

251 "--transfer", 

252 **_transfer_params, 

253) 

254 

255 

256transfer_dimensions_option = MWOptionDecorator( 

257 "--transfer-dimensions/--no-transfer-dimensions", 

258 is_flag=True, 

259 default=True, 

260 help=unwrap( 

261 """If true, also copy dimension records along with datasets. 

262 If the dmensions are already present in the destination butler it 

263 can be more efficient to disable this. The default is to transfer 

264 dimensions.""" 

265 ), 

266) 

267 

268 

269verbose_option = MWOptionDecorator("-v", "--verbose", help="Increase verbosity.", is_flag=True) 

270 

271 

272where_option = MWOptionDecorator( 

273 "--where", default="", help="A string expression similar to a SQL WHERE clause." 

274) 

275 

276 

277order_by_option = MWOptionDecorator( 

278 "--order-by", 

279 help=unwrap( 

280 """One or more comma-separated names used to order records. Names can be dimension names, 

281 metadata field names, or "timespan.begin" / "timespan.end" for temporal dimensions. 

282 In some cases the dimension for a metadata field or timespan bound can be inferred, but usually 

283 qualifying these with "<dimension>.<field>" is necessary. 

284 To reverse ordering for a name, prefix it with a minus sign. 

285 """ 

286 ), 

287 multiple=True, 

288 callback=split_commas, 

289) 

290 

291 

292_default_limit = -20_000 

293limit_option = MWOptionDecorator( 

294 "--limit", 

295 help=unwrap( 

296 f"""Limit the number of results that are processed. 0 means no limit. A negative 

297 value specifies a cap where a warning will be issued if the cap is hit. 

298 Default value is {_default_limit}.""" 

299 ), 

300 type=int, 

301 default=_default_limit, 

302) 

303 

304offset_option = MWOptionDecorator( 

305 "--offset", 

306 help=unwrap("Skip initial number of records, only used when --limit is specified."), 

307 type=int, 

308 default=0, 

309) 

310 

311track_file_attrs_option = MWOptionDecorator( 

312 "--track-file-attrs/--no-track-file-attrs", 

313 default=True, 

314 help="Indicate to the datastore whether file attributes such as file size" 

315 " or checksum should be tracked or not. Whether this parameter is honored" 

316 " depends on the specific datastore implementation.", 

317)