Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 90%

81 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ConfigCollectionType", 

26 "DatasetTypeConfig", 

27 "ExtraColumnConfig", 

28 "ExtraColumnType", 

29 "ObsCoreConfig", 

30 "ObsCoreManagerConfig", 

31 "SpatialPluginConfig", 

32] 

33 

34import enum 

35from collections.abc import Mapping 

36from typing import Any 

37 

38from lsst.daf.butler._compat import _BaseModelCompat 

39from pydantic import StrictBool, StrictFloat, StrictInt, StrictStr, validator 

40 

41 

42class ExtraColumnType(str, enum.Enum): 

43 """Enum class defining possible values for types of extra columns.""" 

44 

45 bool = "bool" 

46 int = "int" 

47 float = "float" 

48 string = "string" 

49 

50 

51class ExtraColumnConfig(_BaseModelCompat): 

52 """Configuration class describing specification of additional column in 

53 obscore table. 

54 """ 

55 

56 template: str 

57 """Template string for formatting the column value.""" 

58 

59 type: ExtraColumnType = ExtraColumnType.string 

60 """Column type, formatted string will be converted to this actual type.""" 

61 

62 length: int | None = None 

63 """Optional length qualifier for a column, only used for strings.""" 

64 

65 doc: str | None = None 

66 """Documentation string for this column.""" 

67 

68 

69class DatasetTypeConfig(_BaseModelCompat): 

70 """Configuration describing dataset type-related options.""" 

71 

72 dataproduct_type: str 

73 """Value for the ``dataproduct_type`` column.""" 

74 

75 dataproduct_subtype: str | None = None 

76 """Value for the ``dataproduct_subtype`` column, optional.""" 

77 

78 calib_level: int 

79 """Value for the ``calib_level`` column.""" 

80 

81 o_ucd: str | None = None 

82 """Value for the ``o_ucd`` column, optional.""" 

83 

84 access_format: str | None = None 

85 """Value for the ``access_format`` column, optional.""" 

86 

87 obs_id_fmt: str | None = None 

88 """Format string for ``obs_id`` column, optional. Uses `str.format` 

89 syntax. 

90 """ 

91 

92 datalink_url_fmt: str | None = None 

93 """Format string for ``access_url`` column for DataLink.""" 

94 

95 obs_collection: str | None = None 

96 """Value for the ``obs_collection`` column, if specified it overrides 

97 global value in `ObsCoreConfig`.""" 

98 

99 extra_columns: None | ( 

100 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

101 ) = None 

102 """Description for additional columns, optional. 

103 

104 Keys are the names of the columns, values can be literal constants with the 

105 values, or ExtraColumnConfig mappings.""" 

106 

107 

108class SpatialPluginConfig(_BaseModelCompat): 

109 """Configuration class for a spatial plugin.""" 

110 

111 cls: str 

112 """Name of the class implementing plugin methods.""" 

113 

114 config: dict[str, Any] = {} 

115 """Configuration object passed to plugin ``initialize()`` method.""" 

116 

117 

118class ObsCoreConfig(_BaseModelCompat): 

119 """Configuration which controls conversion of Registry datasets into 

120 obscore records. 

121 

122 This configuration is a base class for ObsCore manager configuration class. 

123 It can also be used by other tools that use `RecordFactory` to convert 

124 datasets into obscore records. 

125 """ 

126 

127 collections: list[str] | None = None 

128 """Registry collections to include, if missing then all collections are 

129 used. Depending on implementation the name in the list can be either a 

130 full collection name or a regular expression. 

131 """ 

132 

133 dataset_types: dict[str, DatasetTypeConfig] 

134 """Per-dataset type configuration, key is the dataset type name.""" 

135 

136 obs_collection: str | None = None 

137 """Value for the ``obs_collection`` column. This can be overridden in 

138 dataset type configuration. 

139 """ 

140 

141 facility_name: str 

142 """Value for the ``facility_name`` column.""" 

143 

144 extra_columns: None | ( 

145 dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] 

146 ) = None 

147 """Description for additional columns, optional. 

148 

149 Keys are the names of the columns, values can be literal constants with the 

150 values, or ExtraColumnConfig mappings.""" 

151 

152 indices: dict[str, str | list[str]] | None = None 

153 """Description of indices, key is the index name, value is the list of 

154 column names or a single column name. The index name may not be used for 

155 an actual index. 

156 """ 

157 

158 spectral_ranges: dict[str, tuple[float | None, float | None]] = {} 

159 """Maps band name or filter name to a min/max of spectral range. One or 

160 both ends can be specified as `None`. 

161 """ 

162 

163 spatial_plugins: dict[str, SpatialPluginConfig] = {} 

164 """Optional configuration for plugins managing spatial columns and 

165 indices. The key is an arbitrary name and the value is an object describing 

166 plugin class and its configuration options. By default there is no spatial 

167 indexing support, but a standard ``s_region`` column is always included. 

168 """ 

169 

170 

171class ConfigCollectionType(str, enum.Enum): 

172 """Enum class defining possible values for configuration attributes.""" 

173 

174 RUN = "RUN" 

175 TAGGED = "TAGGED" 

176 

177 

178class ObsCoreManagerConfig(ObsCoreConfig): 

179 """Complete configuration for ObsCore manager.""" 

180 

181 namespace: str = "daf_butler_obscore" 

182 """Unique namespace to distinguish different instances, used for schema 

183 migration purposes. 

184 """ 

185 

186 version: int 

187 """Version of configuration, used for schema migration purposes. It needs 

188 to be incremented on every change of configuration that causes a schema or 

189 data migration. 

190 """ 

191 

192 table_name: str = "obscore" 

193 """Name of the table for ObsCore records.""" 

194 

195 collection_type: ConfigCollectionType 

196 """Type of the collections that can appear in ``collections`` attribute. 

197 

198 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

199 expressions that will be used to match RUN collections only. When 

200 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

201 exactly one collection name which must be TAGGED collection. 

202 """ 

203 

204 @validator("collection_type") 

205 def validate_collection_type( 

206 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

207 ) -> Any: 

208 """Check that contents of ``collections`` is consistent with 

209 ``collection_type``. 

210 """ 

211 if value is ConfigCollectionType.TAGGED: 

212 collections: list[str] | None = values["collections"] 

213 if collections is None or len(collections) != 1: 

214 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

215 return value