Coverage for python / lsst / obs / base / cli / cmd / commands.py: 90%

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from collections.abc import Sequence 

23 

24import click 

25 

26from lsst.daf.butler import Butler 

27from lsst.daf.butler.cli.opt import ( 

28 collections_argument, 

29 config_file_option, 

30 config_option, 

31 locations_argument, 

32 options_file_option, 

33 processes_option, 

34 regex_option, 

35 repo_argument, 

36 run_option, 

37 transfer_option, 

38 where_option, 

39) 

40from lsst.daf.butler.cli.utils import ButlerCommand, split_commas, typeStrAcceptsMultiple 

41from lsst.pipe.base.cli.opt import instrument_argument 

42 

43from ... import script 

44from ..opt import failfast_option, labels_argument 

45 

46# regular expression that can be used to find supported fits file extensions. 

47fits_re = r"\.fit[s]?\b" 

48 

49 

50@click.command(short_help="Define visits from exposures.", cls=ButlerCommand) 

51@repo_argument(required=True) 

52@instrument_argument(required=True) 

53@config_file_option( 

54 help="Path to a pex_config override to be included after the Instrument config overrides are applied." 

55) 

56@click.option( 

57 "--collections", 

58 help="The collections to be searched (in order) for camera geometry.", 

59 multiple=True, 

60 callback=split_commas, 

61 metavar=typeStrAcceptsMultiple, 

62) 

63@where_option() 

64@click.option( 

65 "--update-records/--no-update-records", 

66 default=False, 

67 help="Use this option to force updates to the visit definition record. " 

68 "Should only be used if you know that there has been a change to the " 

69 "exposure records, such as a change to the metadata translator.", 

70) 

71@click.option( 

72 "--incremental/--no-incremental", 

73 default=False, 

74 help="Use this option to force updates to the visit definition record " 

75 "when multi-snap visits are being ingested incrementally and so you " 

76 "might encounter partial visits. Implies --update-records.", 

77) 

78@options_file_option() 

79def define_visits(*args, **kwargs): 

80 """Define visits from exposures in the butler registry. 

81 

82 The calibration collection containing the camera geometry can not 

83 be specified. 

84 """ 

85 script.defineVisits(*args, **kwargs) 

86 

87 

88@click.command(short_help="Ingest raw frames.", cls=ButlerCommand) 

89@repo_argument(required=True) 

90@locations_argument( 

91 help="LOCATIONS specifies files to ingest and/or locations to search for files.", required=True 

92) 

93@regex_option( 

94 default=fits_re, 

95 help="Regex string used to find files in directories listed in LOCATIONS. " 

96 "Searches for fits files by default.", 

97) 

98@config_option(metavar="TEXT=TEXT", multiple=True) 

99@config_file_option(type=click.Path(exists=True, writable=False, file_okay=True, dir_okay=False)) 

100@run_option(required=False) 

101@transfer_option() 

102@processes_option() 

103@click.option( 

104 "--ingest-task", 

105 default="lsst.obs.base.RawIngestTask", 

106 help="The fully qualified class name of the ingest task to use.", 

107) 

108@click.option( 

109 "--track-file-attrs/--no-track-file-attrs", 

110 default=True, 

111 help="Indicate to the datastore whether file attributes such as file size" 

112 " or checksum should be tracked or not. Whether this parameter is honored" 

113 " depends on the specific datastore implentation.", 

114) 

115@failfast_option() 

116@click.option( 

117 "--update-records/--no-update-records", 

118 default=False, 

119 help="Use this option to force updates to the exposure records. " 

120 "Should only be used if you know that there has been a change to the " 

121 "exposure records, such as a change to the metadata translator.", 

122) 

123@click.option( 

124 "--skip-existing/--no-skip-existing", 

125 default=True, 

126 help="Use this option to control whether a raw file already existing " 

127 "in the Butler repository is an error or not.", 

128) 

129@options_file_option() 

130def ingest_raws(*args, **kwargs): 

131 """Ingest raw frames into from a directory into the butler registry.""" 

132 script.ingestRaws(*args, **kwargs) 

133 

134 

135@click.command(short_help="Add an instrument's curated calibrations.", cls=ButlerCommand) 

136@repo_argument(required=True) 

137@instrument_argument(required=True) 

138@labels_argument() 

139@click.option( 

140 "--collection", 

141 required=False, 

142 help="Name of the calibration collection that associates datasets with validity ranges.", 

143) 

144@click.option( 

145 "--label", 

146 "labels", 

147 multiple=True, 

148 help=( 

149 "Extra strings to include (with automatic delimiters) in all RUN collection names, " 

150 "as well as the calibration collection name if it is not provided via --collection. " 

151 "May be provided as a positional argument instead of or in addition to this option, " 

152 "as long as at least one label is provided. Positional-argument labels appear before " 

153 "those provided by this option." 

154 ), 

155) 

156@click.option( 

157 "--prefix", 

158 required=False, 

159 help=( 

160 "Prefix for the collection name. Default is the instrument name. " 

161 "This is ignored if --collection is passed." 

162 ), 

163) 

164@options_file_option() 

165def write_curated_calibrations(*, repo, instrument, collection, labels, labels_arg, prefix): 

166 """Add an instrument's curated calibrations to the data repository.""" 

167 script.writeCuratedCalibrations( 

168 repo=repo, instrument=instrument, collection=collection, labels=labels_arg + labels, prefix=prefix 

169 ) 

170 

171 

172@click.command() 

173@repo_argument(required=True) 

174@click.argument("instrument") 

175@collections_argument(help="Collections to search for visit geometry datasets.") 

176@click.option("--dataset-type", default="visit_geometry", help="Name of the visit geometry dataset type.") 

177@click.option("--batch-size", default=11, help="Number of visits to process in each transaction.") 

178@where_option() 

179def update_dimension_regions( 

180 *, 

181 repo: str, 

182 instrument: str, 

183 collections: Sequence[str], 

184 dataset_type: str, 

185 batch_size: int, 

186 where: str, 

187) -> None: 

188 """Update dimension record regions from visit geometry datasets.""" 

189 from ...visit_geometry import VisitGeometry 

190 

191 butler = Butler.from_config(repo, writeable=True, collections=collections) 

192 VisitGeometry.update_dimension_records( 

193 butler, instrument, where, dataset_type=dataset_type, batch_size=batch_size 

194 )