Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 90%

80 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 10:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = [ 

31 "ConfigCollectionType", 

32 "DatasetTypeConfig", 

33 "ExtraColumnConfig", 

34 "ExtraColumnType", 

35 "ObsCoreConfig", 

36 "ObsCoreManagerConfig", 

37 "SpatialPluginConfig", 

38] 

39 

40import enum 

41from typing import Any 

42 

43import pydantic 

44from pydantic import StrictBool, StrictFloat, StrictInt, StrictStr 

45 

46 

47class ExtraColumnType(str, enum.Enum): 

48 """Enum class defining possible values for types of extra columns.""" 

49 

50 bool = "bool" 

51 int = "int" 

52 float = "float" 

53 string = "string" 

54 

55 

56class ExtraColumnConfig(pydantic.BaseModel): 

57 """Configuration class describing specification of additional column in 

58 obscore table. 

59 """ 

60 

61 template: str 

62 """Template string for formatting the column value.""" 

63 

64 type: ExtraColumnType = ExtraColumnType.string 

65 """Column type, formatted string will be converted to this actual type.""" 

66 

67 length: int | None = None 

68 """Optional length qualifier for a column, only used for strings.""" 

69 

70 doc: str | None = None 

71 """Documentation string for this column.""" 

72 

73 

74class DatasetTypeConfig(pydantic.BaseModel): 

75 """Configuration describing dataset type-related options.""" 

76 

77 dataproduct_type: str 

78 """Value for the ``dataproduct_type`` column.""" 

79 

80 dataproduct_subtype: str | None = None 

81 """Value for the ``dataproduct_subtype`` column, optional.""" 

82 

83 calib_level: int 

84 """Value for the ``calib_level`` column.""" 

85 

86 o_ucd: str | None = None 

87 """Value for the ``o_ucd`` column, optional.""" 

88 

89 access_format: str | None = None 

90 """Value for the ``access_format`` column, optional.""" 

91 

92 obs_id_fmt: str | None = None 

93 """Format string for ``obs_id`` column, optional. Uses `str.format` 

94 syntax. 

95 """ 

96 

97 datalink_url_fmt: str | None = None 

98 """Format string for ``access_url`` column for DataLink.""" 

99 

100 obs_collection: str | None = None 

101 """Value for the ``obs_collection`` column, if specified it overrides 

102 global value in `ObsCoreConfig`.""" 

103 

104 extra_columns: None | ( 

105 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

106 ) = None 

107 """Description for additional columns, optional. 

108 

109 Keys are the names of the columns, values can be literal constants with the 

110 values, or ExtraColumnConfig mappings.""" 

111 

112 

113class SpatialPluginConfig(pydantic.BaseModel): 

114 """Configuration class for a spatial plugin.""" 

115 

116 cls: str 

117 """Name of the class implementing plugin methods.""" 

118 

119 config: dict[str, Any] = {} 

120 """Configuration object passed to plugin ``initialize()`` method.""" 

121 

122 

123class ObsCoreConfig(pydantic.BaseModel): 

124 """Configuration which controls conversion of Registry datasets into 

125 obscore records. 

126 

127 This configuration is a base class for ObsCore manager configuration class. 

128 It can also be used by other tools that use `RecordFactory` to convert 

129 datasets into obscore records. 

130 """ 

131 

132 collections: list[str] | None = None 

133 """Registry collections to include, if missing then all collections are 

134 used. Depending on implementation the name in the list can be either a 

135 full collection name or a regular expression. 

136 """ 

137 

138 dataset_types: dict[str, DatasetTypeConfig] 

139 """Per-dataset type configuration, key is the dataset type name.""" 

140 

141 obs_collection: str | None = None 

142 """Value for the ``obs_collection`` column. This can be overridden in 

143 dataset type configuration. 

144 """ 

145 

146 facility_name: str 

147 """Value for the ``facility_name`` column.""" 

148 

149 extra_columns: None | ( 

150 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

151 ) = None 

152 """Description for additional columns, optional. 

153 

154 Keys are the names of the columns, values can be literal constants with the 

155 values, or ExtraColumnConfig mappings.""" 

156 

157 indices: dict[str, str | list[str]] | None = None 

158 """Description of indices, key is the index name, value is the list of 

159 column names or a single column name. The index name may not be used for 

160 an actual index. 

161 """ 

162 

163 spectral_ranges: dict[str, tuple[float | None, float | None]] = {} 

164 """Maps band name or filter name to a min/max of spectral range. One or 

165 both ends can be specified as `None`. 

166 """ 

167 

168 spatial_plugins: dict[str, SpatialPluginConfig] = {} 

169 """Optional configuration for plugins managing spatial columns and 

170 indices. The key is an arbitrary name and the value is an object describing 

171 plugin class and its configuration options. By default there is no spatial 

172 indexing support, but a standard ``s_region`` column is always included. 

173 """ 

174 

175 

176class ConfigCollectionType(str, enum.Enum): 

177 """Enum class defining possible values for configuration attributes.""" 

178 

179 RUN = "RUN" 

180 TAGGED = "TAGGED" 

181 

182 

183class ObsCoreManagerConfig(ObsCoreConfig): 

184 """Complete configuration for ObsCore manager.""" 

185 

186 namespace: str = "daf_butler_obscore" 

187 """Unique namespace to distinguish different instances, used for schema 

188 migration purposes. 

189 """ 

190 

191 version: int 

192 """Version of configuration, used for schema migration purposes. It needs 

193 to be incremented on every change of configuration that causes a schema or 

194 data migration. 

195 """ 

196 

197 table_name: str = "obscore" 

198 """Name of the table for ObsCore records.""" 

199 

200 collection_type: ConfigCollectionType 

201 """Type of the collections that can appear in ``collections`` attribute. 

202 

203 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

204 expressions that will be used to match RUN collections only. When 

205 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

206 exactly one collection name which must be TAGGED collection. 

207 """ 

208 

209 @pydantic.model_validator(mode="after") 

210 def validate_collection_type(self) -> ObsCoreManagerConfig: 

211 """Check that contents of ``collections`` is consistent with 

212 ``collection_type``. 

213 """ 

214 if self.collection_type is ConfigCollectionType.TAGGED: 

215 collections: list[str] | None = self.collections 

216 if collections is None or len(collections) != 1: 

217 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

218 return self