Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 89%

80 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-08 05:05 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ConfigCollectionType", 

26 "DatasetTypeConfig", 

27 "ExtraColumnConfig", 

28 "ExtraColumnType", 

29 "ObsCoreConfig", 

30 "ObsCoreManagerConfig", 

31 "SpatialPluginConfig", 

32] 

33 

34import enum 

35from collections.abc import Mapping 

36from typing import Any, Dict, List, Optional, Tuple, Union 

37 

38from pydantic import BaseModel, StrictBool, StrictFloat, StrictInt, StrictStr, validator 

39 

40 

41class ExtraColumnType(str, enum.Enum): 

42 """Enum class defining possible values for types of extra columns.""" 

43 

44 bool = "bool" 

45 int = "int" 

46 float = "float" 

47 string = "string" 

48 

49 

50class ExtraColumnConfig(BaseModel): 

51 """Configuration class describing specification of additional column in 

52 obscore table. 

53 """ 

54 

55 template: str 

56 """Template string for formatting the column value.""" 

57 

58 type: ExtraColumnType = ExtraColumnType.string 

59 """Column type, formatted string will be converted to this actual type.""" 

60 

61 length: Optional[int] = None 

62 """Optional length qualifier for a column, only used for strings.""" 

63 

64 doc: Optional[str] = None 

65 """Documentation string for this column.""" 

66 

67 

68class DatasetTypeConfig(BaseModel): 

69 """Configuration describing dataset type-related options.""" 

70 

71 dataproduct_type: str 

72 """Value for the ``dataproduct_type`` column.""" 

73 

74 dataproduct_subtype: Optional[str] = None 

75 """Value for the ``dataproduct_subtype`` column, optional.""" 

76 

77 calib_level: int 

78 """Value for the ``calib_level`` column.""" 

79 

80 o_ucd: Optional[str] = None 

81 """Value for the ``o_ucd`` column, optional.""" 

82 

83 access_format: Optional[str] = None 

84 """Value for the ``access_format`` column, optional.""" 

85 

86 obs_id_fmt: Optional[str] = None 

87 """Format string for ``obs_id`` column, optional. Uses `str.format` 

88 syntax. 

89 """ 

90 

91 datalink_url_fmt: Optional[str] = None 

92 """Format string for ``access_url`` column for DataLink.""" 

93 

94 obs_collection: Optional[str] = None 

95 """Value for the ``obs_collection`` column, if specified it overrides 

96 global value in `ObsCoreConfig`.""" 

97 

98 extra_columns: Optional[ 

99 Dict[str, Union[StrictFloat, StrictInt, StrictBool, StrictStr, ExtraColumnConfig]] 

100 ] = None 

101 """Description for additional columns, optional. 

102 

103 Keys are the names of the columns, values can be literal constants with the 

104 values, or ExtraColumnConfig mappings.""" 

105 

106 

107class SpatialPluginConfig(BaseModel): 

108 """Configuration class for a spatial plugin.""" 

109 

110 cls: str 

111 """Name of the class implementing plugin methods.""" 

112 

113 config: Dict[str, Any] = {} 

114 """Configuration object passed to plugin ``initialize()`` method.""" 

115 

116 

117class ObsCoreConfig(BaseModel): 

118 """Configuration which controls conversion of Registry datasets into 

119 obscore records. 

120 

121 This configuration is a base class for ObsCore manager configuration class. 

122 It can also be used by other tools that use `RecordFactory` to convert 

123 datasets into obscore records. 

124 """ 

125 

126 collections: Optional[List[str]] = None 

127 """Registry collections to include, if missing then all collections are 

128 used. Depending on implementation the name in the list can be either a 

129 full collection name or a regular expression. 

130 """ 

131 

132 dataset_types: Dict[str, DatasetTypeConfig] 

133 """Per-dataset type configuration, key is the dataset type name.""" 

134 

135 obs_collection: Optional[str] = None 

136 """Value for the ``obs_collection`` column. This can be overridden in 

137 dataset type configuration. 

138 """ 

139 

140 facility_name: str 

141 """Value for the ``facility_name`` column.""" 

142 

143 extra_columns: Optional[ 

144 Dict[str, Union[StrictFloat, StrictInt, StrictBool, StrictStr, ExtraColumnConfig]] 

145 ] = None 

146 """Description for additional columns, optional. 

147 

148 Keys are the names of the columns, values can be literal constants with the 

149 values, or ExtraColumnConfig mappings.""" 

150 

151 indices: Optional[Dict[str, Union[str, List[str]]]] = None 

152 """Description of indices, key is the index name, value is the list of 

153 column names or a single column name. The index name may not be used for 

154 an actual index. 

155 """ 

156 

157 spectral_ranges: Dict[str, Tuple[float | None, float | None]] = {} 

158 """Maps band name or filter name to a min/max of spectral range. One or 

159 both ends can be specified as `None`. 

160 """ 

161 

162 spatial_plugins: Dict[str, SpatialPluginConfig] = {} 

163 """Optional configuration for plugins managing spatial columns and 

164 indices. The key is an arbitrary name and the value is an object describing 

165 plugin class and its configuration options. By default there is no spatial 

166 indexing support, but a standard ``s_region`` column is always included. 

167 """ 

168 

169 

170class ConfigCollectionType(str, enum.Enum): 

171 """Enum class defining possible values for configuration attributes.""" 

172 

173 RUN = "RUN" 

174 TAGGED = "TAGGED" 

175 

176 

177class ObsCoreManagerConfig(ObsCoreConfig): 

178 """Complete configuration for ObsCore manager.""" 

179 

180 namespace: str = "daf_butler_obscore" 

181 """Unique namespace to distinguish different instances, used for schema 

182 migration purposes. 

183 """ 

184 

185 version: int 

186 """Version of configuration, used for schema migration purposes. It needs 

187 to be incremented on every change of configuration that causes a schema or 

188 data migration. 

189 """ 

190 

191 table_name: str = "obscore" 

192 """Name of the table for ObsCore records.""" 

193 

194 collection_type: ConfigCollectionType 

195 """Type of the collections that can appear in ``collections`` attribute. 

196 

197 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

198 expressions that will be used to match RUN collections only. When 

199 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

200 exactly one collection name which must be TAGGED collection. 

201 """ 

202 

203 @validator("collection_type") 

204 def validate_collection_type( 

205 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

206 ) -> Any: 

207 """Check that contents of ``collections`` is consistent with 

208 ``collection_type``. 

209 """ 

210 if value is ConfigCollectionType.TAGGED: 

211 collections: Optional[List[str]] = values["collections"] 

212 if collections is None or len(collections) != 1: 

213 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

214 return value