Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 78%

90 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 15:14 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ConfigCollectionType", 

26 "DatasetTypeConfig", 

27 "ExtraColumnConfig", 

28 "ExtraColumnType", 

29 "ObsCoreConfig", 

30 "ObsCoreManagerConfig", 

31 "SpatialPluginConfig", 

32] 

33 

34import enum 

35from collections.abc import Mapping 

36from typing import Any 

37 

38import pydantic 

39from lsst.daf.butler._compat import PYDANTIC_V2, _BaseModelCompat 

40from pydantic import StrictBool, StrictFloat, StrictInt, StrictStr 

41 

42 

43class ExtraColumnType(str, enum.Enum): 

44 """Enum class defining possible values for types of extra columns.""" 

45 

46 bool = "bool" 

47 int = "int" 

48 float = "float" 

49 string = "string" 

50 

51 

52class ExtraColumnConfig(_BaseModelCompat): 

53 """Configuration class describing specification of additional column in 

54 obscore table. 

55 """ 

56 

57 template: str 

58 """Template string for formatting the column value.""" 

59 

60 type: ExtraColumnType = ExtraColumnType.string 

61 """Column type, formatted string will be converted to this actual type.""" 

62 

63 length: int | None = None 

64 """Optional length qualifier for a column, only used for strings.""" 

65 

66 doc: str | None = None 

67 """Documentation string for this column.""" 

68 

69 

70class DatasetTypeConfig(_BaseModelCompat): 

71 """Configuration describing dataset type-related options.""" 

72 

73 dataproduct_type: str 

74 """Value for the ``dataproduct_type`` column.""" 

75 

76 dataproduct_subtype: str | None = None 

77 """Value for the ``dataproduct_subtype`` column, optional.""" 

78 

79 calib_level: int 

80 """Value for the ``calib_level`` column.""" 

81 

82 o_ucd: str | None = None 

83 """Value for the ``o_ucd`` column, optional.""" 

84 

85 access_format: str | None = None 

86 """Value for the ``access_format`` column, optional.""" 

87 

88 obs_id_fmt: str | None = None 

89 """Format string for ``obs_id`` column, optional. Uses `str.format` 

90 syntax. 

91 """ 

92 

93 datalink_url_fmt: str | None = None 

94 """Format string for ``access_url`` column for DataLink.""" 

95 

96 obs_collection: str | None = None 

97 """Value for the ``obs_collection`` column, if specified it overrides 

98 global value in `ObsCoreConfig`.""" 

99 

100 extra_columns: None | ( 

101 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

102 ) = None 

103 """Description for additional columns, optional. 

104 

105 Keys are the names of the columns, values can be literal constants with the 

106 values, or ExtraColumnConfig mappings.""" 

107 

108 

109class SpatialPluginConfig(_BaseModelCompat): 

110 """Configuration class for a spatial plugin.""" 

111 

112 cls: str 

113 """Name of the class implementing plugin methods.""" 

114 

115 config: dict[str, Any] = {} 

116 """Configuration object passed to plugin ``initialize()`` method.""" 

117 

118 

119class ObsCoreConfig(_BaseModelCompat): 

120 """Configuration which controls conversion of Registry datasets into 

121 obscore records. 

122 

123 This configuration is a base class for ObsCore manager configuration class. 

124 It can also be used by other tools that use `RecordFactory` to convert 

125 datasets into obscore records. 

126 """ 

127 

128 collections: list[str] | None = None 

129 """Registry collections to include, if missing then all collections are 

130 used. Depending on implementation the name in the list can be either a 

131 full collection name or a regular expression. 

132 """ 

133 

134 dataset_types: dict[str, DatasetTypeConfig] 

135 """Per-dataset type configuration, key is the dataset type name.""" 

136 

137 obs_collection: str | None = None 

138 """Value for the ``obs_collection`` column. This can be overridden in 

139 dataset type configuration. 

140 """ 

141 

142 facility_name: str 

143 """Value for the ``facility_name`` column.""" 

144 

145 extra_columns: None | ( 

146 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

147 ) = None 

148 """Description for additional columns, optional. 

149 

150 Keys are the names of the columns, values can be literal constants with the 

151 values, or ExtraColumnConfig mappings.""" 

152 

153 indices: dict[str, str | list[str]] | None = None 

154 """Description of indices, key is the index name, value is the list of 

155 column names or a single column name. The index name may not be used for 

156 an actual index. 

157 """ 

158 

159 spectral_ranges: dict[str, tuple[float | None, float | None]] = {} 

160 """Maps band name or filter name to a min/max of spectral range. One or 

161 both ends can be specified as `None`. 

162 """ 

163 

164 spatial_plugins: dict[str, SpatialPluginConfig] = {} 

165 """Optional configuration for plugins managing spatial columns and 

166 indices. The key is an arbitrary name and the value is an object describing 

167 plugin class and its configuration options. By default there is no spatial 

168 indexing support, but a standard ``s_region`` column is always included. 

169 """ 

170 

171 

172class ConfigCollectionType(str, enum.Enum): 

173 """Enum class defining possible values for configuration attributes.""" 

174 

175 RUN = "RUN" 

176 TAGGED = "TAGGED" 

177 

178 

179class ObsCoreManagerConfig(ObsCoreConfig): 

180 """Complete configuration for ObsCore manager.""" 

181 

182 namespace: str = "daf_butler_obscore" 

183 """Unique namespace to distinguish different instances, used for schema 

184 migration purposes. 

185 """ 

186 

187 version: int 

188 """Version of configuration, used for schema migration purposes. It needs 

189 to be incremented on every change of configuration that causes a schema or 

190 data migration. 

191 """ 

192 

193 table_name: str = "obscore" 

194 """Name of the table for ObsCore records.""" 

195 

196 collection_type: ConfigCollectionType 

197 """Type of the collections that can appear in ``collections`` attribute. 

198 

199 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

200 expressions that will be used to match RUN collections only. When 

201 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

202 exactly one collection name which must be TAGGED collection. 

203 """ 

204 

205 if PYDANTIC_V2: 205 ↛ 207line 205 didn't jump to line 207, because the condition on line 205 was never true

206 

207 @pydantic.model_validator(mode="after") # type: ignore[attr-defined] 

208 def validate_collection_type(self) -> ObsCoreManagerConfig: 

209 """Check that contents of ``collections`` is consistent with 

210 ``collection_type``. 

211 """ 

212 if self.collection_type is ConfigCollectionType.TAGGED: 

213 collections: list[str] | None = self.collections 

214 if collections is None or len(collections) != 1: 

215 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

216 return self 

217 

218 else: 

219 

220 @pydantic.validator("collection_type") 

221 def validate_collection_type( 

222 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

223 ) -> Any: 

224 """Check that contents of ``collections`` is consistent with 

225 ``collection_type``. 

226 """ 

227 if value is ConfigCollectionType.TAGGED: 

228 collections: list[str] | None = values["collections"] 

229 if collections is None or len(collections) != 1: 

230 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

231 return value