Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 90%

83 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:56 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ConfigCollectionType", 

26 "DatasetTypeConfig", 

27 "ExtraColumnConfig", 

28 "ExtraColumnType", 

29 "ObsCoreConfig", 

30 "ObsCoreManagerConfig", 

31 "SpatialPluginConfig", 

32] 

33 

34import enum 

35from collections.abc import Mapping 

36from typing import Any 

37 

38try: 

39 from pydantic.v1 import BaseModel, StrictBool, StrictFloat, StrictInt, StrictStr, validator 

40except ModuleNotFoundError: 

41 from pydantic import BaseModel, StrictBool, StrictFloat, StrictInt, StrictStr, validator # type: ignore 

42 

43 

44class ExtraColumnType(str, enum.Enum): 

45 """Enum class defining possible values for types of extra columns.""" 

46 

47 bool = "bool" 

48 int = "int" 

49 float = "float" 

50 string = "string" 

51 

52 

53class ExtraColumnConfig(BaseModel): 

54 """Configuration class describing specification of additional column in 

55 obscore table. 

56 """ 

57 

58 template: str 

59 """Template string for formatting the column value.""" 

60 

61 type: ExtraColumnType = ExtraColumnType.string 

62 """Column type, formatted string will be converted to this actual type.""" 

63 

64 length: int | None = None 

65 """Optional length qualifier for a column, only used for strings.""" 

66 

67 doc: str | None = None 

68 """Documentation string for this column.""" 

69 

70 

71class DatasetTypeConfig(BaseModel): 

72 """Configuration describing dataset type-related options.""" 

73 

74 dataproduct_type: str 

75 """Value for the ``dataproduct_type`` column.""" 

76 

77 dataproduct_subtype: str | None = None 

78 """Value for the ``dataproduct_subtype`` column, optional.""" 

79 

80 calib_level: int 

81 """Value for the ``calib_level`` column.""" 

82 

83 o_ucd: str | None = None 

84 """Value for the ``o_ucd`` column, optional.""" 

85 

86 access_format: str | None = None 

87 """Value for the ``access_format`` column, optional.""" 

88 

89 obs_id_fmt: str | None = None 

90 """Format string for ``obs_id`` column, optional. Uses `str.format` 

91 syntax. 

92 """ 

93 

94 datalink_url_fmt: str | None = None 

95 """Format string for ``access_url`` column for DataLink.""" 

96 

97 obs_collection: str | None = None 

98 """Value for the ``obs_collection`` column, if specified it overrides 

99 global value in `ObsCoreConfig`.""" 

100 

101 extra_columns: None | ( 

102 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

103 ) = None 

104 """Description for additional columns, optional. 

105 

106 Keys are the names of the columns, values can be literal constants with the 

107 values, or ExtraColumnConfig mappings.""" 

108 

109 

110class SpatialPluginConfig(BaseModel): 

111 """Configuration class for a spatial plugin.""" 

112 

113 cls: str 

114 """Name of the class implementing plugin methods.""" 

115 

116 config: dict[str, Any] = {} 

117 """Configuration object passed to plugin ``initialize()`` method.""" 

118 

119 

120class ObsCoreConfig(BaseModel): 

121 """Configuration which controls conversion of Registry datasets into 

122 obscore records. 

123 

124 This configuration is a base class for ObsCore manager configuration class. 

125 It can also be used by other tools that use `RecordFactory` to convert 

126 datasets into obscore records. 

127 """ 

128 

129 collections: list[str] | None = None 

130 """Registry collections to include, if missing then all collections are 

131 used. Depending on implementation the name in the list can be either a 

132 full collection name or a regular expression. 

133 """ 

134 

135 dataset_types: dict[str, DatasetTypeConfig] 

136 """Per-dataset type configuration, key is the dataset type name.""" 

137 

138 obs_collection: str | None = None 

139 """Value for the ``obs_collection`` column. This can be overridden in 

140 dataset type configuration. 

141 """ 

142 

143 facility_name: str 

144 """Value for the ``facility_name`` column.""" 

145 

146 extra_columns: None | ( 

147 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

148 ) = None 

149 """Description for additional columns, optional. 

150 

151 Keys are the names of the columns, values can be literal constants with the 

152 values, or ExtraColumnConfig mappings.""" 

153 

154 indices: dict[str, str | list[str]] | None = None 

155 """Description of indices, key is the index name, value is the list of 

156 column names or a single column name. The index name may not be used for 

157 an actual index. 

158 """ 

159 

160 spectral_ranges: dict[str, tuple[float | None, float | None]] = {} 

161 """Maps band name or filter name to a min/max of spectral range. One or 

162 both ends can be specified as `None`. 

163 """ 

164 

165 spatial_plugins: dict[str, SpatialPluginConfig] = {} 

166 """Optional configuration for plugins managing spatial columns and 

167 indices. The key is an arbitrary name and the value is an object describing 

168 plugin class and its configuration options. By default there is no spatial 

169 indexing support, but a standard ``s_region`` column is always included. 

170 """ 

171 

172 

173class ConfigCollectionType(str, enum.Enum): 

174 """Enum class defining possible values for configuration attributes.""" 

175 

176 RUN = "RUN" 

177 TAGGED = "TAGGED" 

178 

179 

180class ObsCoreManagerConfig(ObsCoreConfig): 

181 """Complete configuration for ObsCore manager.""" 

182 

183 namespace: str = "daf_butler_obscore" 

184 """Unique namespace to distinguish different instances, used for schema 

185 migration purposes. 

186 """ 

187 

188 version: int 

189 """Version of configuration, used for schema migration purposes. It needs 

190 to be incremented on every change of configuration that causes a schema or 

191 data migration. 

192 """ 

193 

194 table_name: str = "obscore" 

195 """Name of the table for ObsCore records.""" 

196 

197 collection_type: ConfigCollectionType 

198 """Type of the collections that can appear in ``collections`` attribute. 

199 

200 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

201 expressions that will be used to match RUN collections only. When 

202 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

203 exactly one collection name which must be TAGGED collection. 

204 """ 

205 

206 @validator("collection_type") 

207 def validate_collection_type( 

208 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

209 ) -> Any: 

210 """Check that contents of ``collections`` is consistent with 

211 ``collection_type``. 

212 """ 

213 if value is ConfigCollectionType.TAGGED: 

214 collections: list[str] | None = values["collections"] 

215 if collections is None or len(collections) != 1: 

216 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

217 return value