Coverage for python/lsst/daf/butler/registry/obscore/_config.py: 88%

73 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-07 02:47 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = [ 

25 "ConfigCollectionType", 

26 "DatasetTypeConfig", 

27 "ExtraColumnConfig", 

28 "ExtraColumnType", 

29 "ObsCoreConfig", 

30 "ObsCoreManagerConfig", 

31] 

32 

33import enum 

34from collections.abc import Mapping 

35from typing import Any, Dict, List, Optional, Tuple, Union 

36 

37from pydantic import BaseModel, StrictBool, StrictFloat, StrictInt, StrictStr, validator 

38 

39 

40class ExtraColumnType(str, enum.Enum): 

41 """Enum class defining possible values for types of extra columns.""" 

42 

43 bool = "bool" 

44 int = "int" 

45 float = "float" 

46 string = "string" 

47 

48 

49class ExtraColumnConfig(BaseModel): 

50 """Configuration class describing specification of additional column in 

51 obscore table. 

52 """ 

53 

54 template: str 

55 """Template string for formatting the column value.""" 

56 

57 type: ExtraColumnType = ExtraColumnType.string 

58 """Column type, formatted string will be converted to this actual type.""" 

59 

60 length: Optional[int] = None 

61 """Optional length qualifier for a column, only used for strings.""" 

62 

63 

64class DatasetTypeConfig(BaseModel): 

65 """Configuration describing dataset type-related options.""" 

66 

67 dataproduct_type: str 

68 """Value for the ``dataproduct_type`` column.""" 

69 

70 dataproduct_subtype: Optional[str] = None 

71 """Value for the ``dataproduct_subtype`` column, optional.""" 

72 

73 calib_level: int 

74 """Value for the ``calib_level`` column.""" 

75 

76 o_ucd: Optional[str] = None 

77 """Value for the ``o_ucd`` column, optional.""" 

78 

79 access_format: Optional[str] = None 

80 """Value for the ``access_format`` column, optional.""" 

81 

82 obs_id_fmt: Optional[str] = None 

83 """Format string for ``obs_id`` column, optional. Uses `str.format` 

84 syntax. 

85 """ 

86 

87 datalink_url_fmt: Optional[str] = None 

88 """Format string for ``access_url`` column for DataLink.""" 

89 

90 obs_collection: Optional[str] = None 

91 """Value for the ``obs_collection`` column, if specified it overrides 

92 global value in `ObsCoreConfig`.""" 

93 

94 extra_columns: Optional[ 

95 Dict[str, Union[StrictFloat, StrictInt, StrictBool, StrictStr, ExtraColumnConfig]] 

96 ] = None 

97 """Description for additional columns, optional. 

98 

99 Keys are the names of the columns, values can be literal constants with the 

100 values, or ExtraColumnConfig mappings.""" 

101 

102 

103class ObsCoreConfig(BaseModel): 

104 """Configuration which controls conversion of Registry datasets into 

105 obscore records. 

106 

107 This configuration is a base class for ObsCore manager configuration class. 

108 It can also be used by other tools that use `RecordFactory` to convert 

109 datasets into obscore records. 

110 """ 

111 

112 collections: Optional[List[str]] = None 

113 """Registry collections to include, if missing then all collections are 

114 used. Depending on implementation the name in the list can be either a 

115 full collection name or a regular expression. 

116 """ 

117 

118 dataset_types: Dict[str, DatasetTypeConfig] 

119 """Per-dataset type configuration, key is the dataset type name.""" 

120 

121 obs_collection: Optional[str] = None 

122 """Value for the ``obs_collection`` column. This can be overridden in 

123 dataset type configuration. 

124 """ 

125 

126 facility_name: str 

127 """Value for the ``facility_name`` column.""" 

128 

129 extra_columns: Optional[ 

130 Dict[str, Union[StrictFloat, StrictInt, StrictBool, StrictStr, ExtraColumnConfig]] 

131 ] = None 

132 """Description for additional columns, optional. 

133 

134 Keys are the names of the columns, values can be literal constants with the 

135 values, or ExtraColumnConfig mappings.""" 

136 

137 indices: Optional[Dict[str, Union[str, List[str]]]] = None 

138 """Description of indices, key is the index name, value is the list of 

139 column names or a single column name. The index name may not be used for 

140 an actual index. 

141 """ 

142 

143 spectral_ranges: Dict[str, Tuple[float, float]] = {} 

144 """Maps band name or filter name to a min/max of spectral range.""" 

145 

146 spatial_backend: Optional[str] = None 

147 """The name of a spatial backend which manages additional spatial 

148 columns and indices (e.g. "pgsphere"). By default there is no spatial 

149 indexing support, but a standard ``s_region`` column is always included. 

150 """ 

151 

152 

153class ConfigCollectionType(str, enum.Enum): 

154 """Enum class defining possible values for configuration attributes.""" 

155 

156 RUN = "RUN" 

157 TAGGED = "TAGGED" 

158 

159 

160class ObsCoreManagerConfig(ObsCoreConfig): 

161 """Complete configuration for ObsCore manager.""" 

162 

163 namespace: str = "daf_butler_obscore" 

164 """Unique namespace to distinguish different instances, used for schema 

165 migration purposes. 

166 """ 

167 

168 version: int 

169 """Version of configuration, used for schema migration purposes. It needs 

170 to be incremented on every change of configuration that causes a schema or 

171 data migration. 

172 """ 

173 

174 table_name: str = "obscore" 

175 """Name of the table for ObsCore records.""" 

176 

177 collection_type: ConfigCollectionType 

178 """Type of the collections that can appear in ``collections`` attribute. 

179 

180 When ``collection_type`` is ``RUN`` then ``collections`` contains regular 

181 expressions that will be used to match RUN collections only. When 

182 ``collection_type`` is ``TAGGED`` then ``collections`` must contain 

183 exactly one collection name which must be TAGGED collection. 

184 """ 

185 

186 @validator("collection_type") 

187 def validate_collection_type( 

188 cls, value: ConfigCollectionType, values: Mapping[str, Any] # noqa: N805 

189 ) -> Any: 

190 """Check that contents of ``collections`` is consistent with 

191 ``collection_type``. 

192 """ 

193 if value is ConfigCollectionType.TAGGED: 

194 collections: Optional[List[str]] = values["collections"] 

195 if collections is None or len(collections) != 1: 

196 raise ValueError("'collections' must have one element when 'collection_type' is TAGGED") 

197 return value