Coverage for python/lsst/ap/pipe/make_apdb.py: 22%

1# This file is part of ap_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["makeApdb"] 

23 

24import argparse 

25 

26from deprecated.sphinx import deprecated 

27 

28import lsst.dax.apdb as daxApdb 

29from lsst.pipe.base.configOverrides import ConfigOverrides 

30from lsst.ap.association import DiaPipelineConfig 

31 

32 

33class ConfigOnlyParser(argparse.ArgumentParser): 

34 """Argument parser that knows standard config arguments. 

35 """ 

36 

37 def __init__(self, description=None, **kwargs): 

38 if description is None: 

39 # Description must be readable in both Sphinx and make_apdb.py -h 

40 description = """\ 

41Create a Alert Production Database using config overrides for 

42`lsst.dax.apdb.ApdbConfig`. 

43 

44This script takes the same ``--config`` and ``--config-file`` arguments as 

45pipeline runs. However, the configs are at a lower level than the AP pipeline. 

46 

47The config overrides must define ``db_url`` to create a valid config. 

48""" 

49 

50 super().__init__(description=description, **kwargs) 

51 

52 self.add_argument("-c", "--config", nargs="*", action=ConfigValueAction, 

53 help="config override(s), e.g. " 

54 "``-c prefix=fancy db_url=\"sqlite://\"``", 

55 metavar="NAME=VALUE") 

56 self.add_argument("-C", "--config-file", dest="configfile", nargs="*", action=ConfigFileAction, 

57 help="config override file(s) for ApdbConfig") 

58 

59 def parse_args(self, args=None, namespace=None): 

60 """Parse arguments for an `ApdbConfig`. 

61 

62 Parameters 

63 ---------- 

64 args : `list` [`str`], optional 

65 Argument list; if `None` then ``sys.argv`` is used. 

66 namespace : `argparse.Namespace`, optional 

67 An object to take the attributes. The default is a new empty 

68 `~argparse.Namespace` object 

69 

70 Returns 

71 ------- 

72 namespace : `argparse.Namespace` 

73 A `~argparse.Namespace` instance containing fields: 

74 - ``config``: the supplied config with all overrides applied, 

75 validated and frozen. 

76 """ 

77 if not namespace: 

78 namespace = argparse.Namespace() 

79 namespace.overrides = ConfigOverrides() 

80 

81 # ConfigFileAction and ConfigValueAction require namespace.overrides to exist 

82 namespace = super().parse_args(args, namespace) 

83 del namespace.configfile 

84 # Make ApdbConfig as a subconfig of DiaPipelineConfig to ensure correct defaults get set 

85 namespace.config = DiaPipelineConfig().apdb.value 

86 try: 

87 namespace.overrides.applyTo(namespace.config) 

88 except Exception as e: # yes, configs really can raise anything 

89 message = str(e) 

90 if "diaPipe" in message: 

91 message = "it looks like one of the config files uses ApPipeConfig; " \ 

92 "try dropping 'diaPipe.apdb'\n" + message 

93 self.error(f"cannot apply config: {message}") 

94 

95 namespace.config.validate() 

96 namespace.config.freeze() 

97 

98 return namespace 

99 

100 

101@deprecated( 

102 reason=( 

103 "`make_apdb.py` script is deprecated, use `apdb-cli` command to create APDB instances. " 

104 "Will be removed after v27" 

105 ), 

106 version="v27.0", 

107 category=FutureWarning, 

108) 

109def makeApdb(args=None): 

110 """Create an APDB according to a config. 

111 

112 The command-line arguments should provide config values or a config file 

113 for `ApdbConfig`. 

114 

115 Parameters 

116 ---------- 

117 args : `list` [`str`], optional 

118 List of command-line arguments; if `None` use `sys.argv`. 

119 

120 Returns 

121 ------- 

122 apdb : `lsst.dax.apdb.Apdb` 

123 The newly configured APDB object. 

124 """ 

125 

126 parser = ConfigOnlyParser() 

127 parsedCmd = parser.parse_args(args=args) 

128 

129 # `make_apdb` is replaced by `apdb-cli` commands, for now we keep it for 

130 # backward compatibility, but only support SQL implementation here. 

131 init_config = parsedCmd.config 

132 if not isinstance(init_config, daxApdb.ApdbSqlConfig): 

133 raise TypeError(f"Unexpected type of APDB configuration instance {type(init_config)}") 

134 config = daxApdb.ApdbSql.init_database( 

135 db_url=init_config.db_url, 

136 schema_file=init_config.schema_file, 

137 schema_name=init_config.schema_name, 

138 read_sources_months=init_config.read_sources_months, 

139 read_forced_sources_months=init_config.read_forced_sources_months, 

140 use_insert_id=init_config.use_insert_id, 

141 connection_timeout=init_config.connection_timeout, 

142 dia_object_index=init_config.dia_object_index, 

143 htm_level=init_config.htm_level, 

144 htm_index_column=init_config.htm_index_column, 

145 ra_dec_columns=init_config.ra_dec_columns, 

146 prefix=init_config.prefix, 

147 namespace=init_config.namespace, 

148 ) 

149 apdb = daxApdb.Apdb.from_config(config) 

150 return apdb 

151 

152 

153# -------------------------------------------------------------------- 

154# argparse.Actions for use with ConfigOverrides 

155# ConfigOverrides is normally used with Click; there is no built-in 

156# argparse support. 

157 

158 

159class ConfigValueAction(argparse.Action): 

160 """argparse action to override config parameters using 

161 name=value pairs from the command-line. 

162 """ 

163 

164 def __call__(self, parser, namespace, values, option_string): 

165 if namespace.overrides is None: 

166 return 

167 for nameValue in values: 

168 name, sep, valueStr = nameValue.partition("=") 

169 if not valueStr: 

170 parser.error(f"{option_string} value {nameValue} must be in form name=value") 

171 if "diaPipe.apdb" in name: 

172 parser.error(f"{nameValue} looks like it uses ApPipeConfig; try dropping 'diaPipe.apdb'") 

173 

174 namespace.overrides.addValueOverride(name, valueStr) 

175 

176 

177class ConfigFileAction(argparse.Action): 

178 """argparse action to load config overrides from one or more files. 

179 """ 

180 

181 def __call__(self, parser, namespace, values, option_string=None): 

182 if namespace.overrides is None: 

183 return 

184 for configfile in values: 

185 namespace.overrides.addFileOverride(configfile)