Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 78%

90 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-01 11:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = [ 

31 "ConfigCollectionType", 

32 "DatasetTypeConfig", 

33 "ExtraColumnConfig", 

34 "ExtraColumnType", 

35 "ObsCoreConfig", 

36 "ObsCoreManagerConfig", 

37 "SpatialPluginConfig", 

38] 

39 

40import enum 

41from collections.abc import Mapping 

42from typing import Any 

43 

44import pydantic 

45from lsst.daf.butler._compat import PYDANTIC_V2, _BaseModelCompat 

46from pydantic import StrictBool, StrictFloat, StrictInt, StrictStr 

47 

48 

49class ExtraColumnType(str, enum.Enum): 

50 """Enum class defining possible values for types of extra columns.""" 

51 

52 bool = "bool" 

53 int = "int" 

54 float = "float" 

55 string = "string" 

56 

57 

58class ExtraColumnConfig(_BaseModelCompat): 

59 """Configuration class describing specification of additional column in 

60 obscore table. 

61 """ 

62 

63 template: str 

64 """Template string for formatting the column value.""" 

65 

66 type: ExtraColumnType = ExtraColumnType.string 

67 """Column type, formatted string will be converted to this actual type.""" 

68 

69 length: int | None = None 

70 """Optional length qualifier for a column, only used for strings.""" 

71 

72 doc: str | None = None 

73 """Documentation string for this column.""" 

74 

75 

76class DatasetTypeConfig(_BaseModelCompat): 

77 """Configuration describing dataset type-related options.""" 

78 

79 dataproduct_type: str 

80 """Value for the ``dataproduct_type`` column.""" 

81 

82 dataproduct_subtype: str | None = None 

83 """Value for the ``dataproduct_subtype`` column, optional.""" 

84 

85 calib_level: int 

86 """Value for the ``calib_level`` column.""" 

87 

88 o_ucd: str | None = None 

89 """Value for the ``o_ucd`` column, optional.""" 

90 

91 access_format: str | None = None 

92 """Value for the ``access_format`` column, optional.""" 

93 

94 obs_id_fmt: str | None = None 

95 """Format string for ``obs_id`` column, optional. Uses `str.format` 

96 syntax. 

97 """ 

98 

99 datalink_url_fmt: str | None = None 

100 """Format string for ``access_url`` column for DataLink.""" 

101 

102 obs_collection: str | None = None 

103 """Value for the ``obs_collection`` column, if specified it overrides 

104 global value in `ObsCoreConfig`.""" 

105 

106 extra_columns: None | ( 

107 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

108 ) = None 

109 """Description for additional columns, optional. 

110 

111 Keys are the names of the columns, values can be literal constants with the 

112 values, or ExtraColumnConfig mappings.""" 

113 

114 

115class SpatialPluginConfig(_BaseModelCompat): 

116 """Configuration class for a spatial plugin.""" 

117 

118 cls: str 

119 """Name of the class implementing plugin methods.""" 

120 

121 config: dict[str, Any] = {} 

122 """Configuration object passed to plugin ``initialize()`` method.""" 

123 

124 

125class ObsCoreConfig(_BaseModelCompat): 

126 """Configuration which controls conversion of Registry datasets into 

127 obscore records. 

128 

129 This configuration is a base class for ObsCore manager configuration class. 

130 It can also be used by other tools that use `RecordFactory` to convert 

131 datasets into obscore records. 

132 """ 

133 

134 collections: list[str] | None = None 

135 """Registry collections to include, if missing then all collections are 

136 used. Depending on implementation the name in the list can be either a 

137 full collection name or a regular expression. 

138 """ 

139 

140 dataset_types: dict[str, DatasetTypeConfig] 

141 """Per-dataset type configuration, key is the dataset type name.""" 

142 

143 obs_collection: str | None = None 

144 """Value for the ``obs_collection`` column. This can be overridden in 

145 dataset type configuration. 

146 """ 

147 

148 facility_name: str 

149 """Value for the ``facility_name`` column.""" 

150 

151 extra_columns: None | ( 

152 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

153 ) = None 

154 """Description for additional columns, optional. 

155 

156 Keys are the names of the columns, values can be literal constants with the 

157 values, or ExtraColumnConfig mappings.""" 

158 

159 indices: dict[str, str | list[str]] | None = None 

160 """Description of indices, key is the index name, value is the list of 

161 column names or a single column name. The index name may not be used for 

162 an actual index. 

163 """ 

164 

165 spectral_ranges: dict[str, tuple[float | None, float | None]] = {} 

166 """Maps band name or filter name to a min/max of spectral range. One or 

167 both ends can be specified as `None`. 

168 """ 

169 

170 spatial_plugins: dict[str, SpatialPluginConfig] = {} 

171 """Optional configuration for plugins managing spatial columns and 

172 indices. The key is an arbitrary name and the value is an object describing 

173 plugin class and its configuration options. By default there is no spatial 

174 indexing support, but a standard ``s_region`` column is always included. 

175 """ 

176 

177 

178class ConfigCollectionType(str, enum.Enum): 

179 """Enum class defining possible values for configuration attributes.""" 

180 

181 RUN = "RUN" 

182 TAGGED = "TAGGED" 

183 

184 

185class ObsCoreManagerConfig(ObsCoreConfig): 

186 """Complete configuration for ObsCore manager.""" 

187 

188 namespace: str = "daf_butler_obscore" 

189 """Unique namespace to distinguish different instances, used for schema 

190 migration purposes. 

191 """ 

192 

193 version: int 

194 """Version of configuration, used for schema migration purposes. It needs 

195 to be incremented on every change of configuration that causes a schema or 

196 data migration. 

197 """ 

198 

199 table_name: str = "obscore" 

200 """Name of the table for ObsCore records.""" 

201 

202 collection_type: ConfigCollectionType 

203 """Type of the collections that can appear in ``collections`` attribute. 

204 

205 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

206 expressions that will be used to match RUN collections only. When 

207 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

208 exactly one collection name which must be TAGGED collection. 

209 """ 

210 

211 if PYDANTIC_V2: 211 ↛ 213line 211 didn't jump to line 213, because the condition on line 211 was never true

212 

213 @pydantic.model_validator(mode="after") # type: ignore[attr-defined] 

214 def validate_collection_type(self) -> ObsCoreManagerConfig: 

215 """Check that contents of ``collections`` is consistent with 

216 ``collection_type``. 

217 """ 

218 if self.collection_type is ConfigCollectionType.TAGGED: 

219 collections: list[str] | None = self.collections 

220 if collections is None or len(collections) != 1: 

221 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

222 return self 

223 

224 else: 

225 

226 @pydantic.validator("collection_type") 

227 def validate_collection_type( 

228 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

229 ) -> Any: 

230 """Check that contents of ``collections`` is consistent with 

231 ``collection_type``. 

232 """ 

233 if value is ConfigCollectionType.TAGGED: 

234 collections: list[str] | None = values["collections"] 

235 if collections is None or len(collections) != 1: 

236 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

237 return value