Coverage for python/lsst/daf/butler/registry/obscore/_schema.py: 24%

47 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-04 02:19 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["ObsCoreSchema"] 

25 

26from typing import TYPE_CHECKING, List, Optional, Tuple, Type 

27 

28import sqlalchemy 

29from lsst.daf.butler import ddl 

30from lsst.utils.iteration import ensure_iterable 

31 

32from ._config import DatasetTypeConfig, ExtraColumnConfig, ObsCoreConfig 

33 

34if TYPE_CHECKING: 34 ↛ 35line 34 didn't jump to line 35, because the condition on line 34 was never true

35 from ..interfaces import DatasetRecordStorageManager 

36 

37 

38# List of standard columns in output file. This should include at least all 

39# mandatory columns defined in ObsCore note (revision 1.1, Appendix B). Extra 

40# columns can be added via `extra_columns` parameters in configuration. 

41_STATIC_COLUMNS = ( 

42 ddl.FieldSpec( 

43 name="dataproduct_type", dtype=sqlalchemy.String, length=255, doc="Logical data product type" 

44 ), 

45 ddl.FieldSpec( 

46 name="dataproduct_subtype", dtype=sqlalchemy.String, length=255, doc="Data product specific type" 

47 ), 

48 ddl.FieldSpec( 

49 name="facility_name", 

50 dtype=sqlalchemy.String, 

51 length=255, 

52 doc="The name of the facility used for the observation", 

53 ), 

54 ddl.FieldSpec(name="calib_level", dtype=sqlalchemy.SmallInteger, doc="Calibration level {0, 1, 2, 3, 4}"), 

55 ddl.FieldSpec(name="target_name", dtype=sqlalchemy.String, length=255, doc="Object of interest"), 

56 ddl.FieldSpec(name="obs_id", dtype=sqlalchemy.String, length=255, doc="Observation ID"), 

57 ddl.FieldSpec( 

58 name="obs_collection", dtype=sqlalchemy.String, length=255, doc="Name of the data collection" 

59 ), 

60 ddl.FieldSpec( 

61 name="obs_publisher_did", 

62 dtype=sqlalchemy.String, 

63 length=255, 

64 doc="Dataset identifier given by the publisher", 

65 ), 

66 ddl.FieldSpec( 

67 name="access_url", dtype=sqlalchemy.String, length=65535, doc="URL used to access (download) dataset" 

68 ), 

69 ddl.FieldSpec(name="access_format", dtype=sqlalchemy.String, length=255, doc="File content format"), 

70 ddl.FieldSpec(name="s_ra", dtype=sqlalchemy.Float, doc="Central right ascension, ICRS (deg)"), 

71 ddl.FieldSpec(name="s_dec", dtype=sqlalchemy.Float, doc="Central declination, ICRS (deg)"), 

72 ddl.FieldSpec(name="s_fov", dtype=sqlalchemy.Float, doc="Diameter (bounds) of the covered region (deg)"), 

73 ddl.FieldSpec( 

74 name="s_region", 

75 dtype=sqlalchemy.String, 

76 length=65535, 

77 doc="Sky region covered by the data product (expressed in ICRS frame)", 

78 ), 

79 ddl.FieldSpec( 

80 name="s_resolution", dtype=sqlalchemy.Float, doc="Spatial resolution of data as FWHM (arcsec)" 

81 ), 

82 ddl.FieldSpec( 

83 name="s_xel1", dtype=sqlalchemy.Integer, doc="Number of elements along the first spatial axis" 

84 ), 

85 ddl.FieldSpec( 

86 name="s_xel2", dtype=sqlalchemy.Integer, doc="Number of elements along the second spatial axis" 

87 ), 

88 ddl.FieldSpec(name="t_xel", dtype=sqlalchemy.Integer, doc="Number of elements along the time axis"), 

89 ddl.FieldSpec(name="t_min", dtype=sqlalchemy.Float, doc="Start time in MJD"), 

90 ddl.FieldSpec(name="t_max", dtype=sqlalchemy.Float, doc="Stop time in MJD"), 

91 ddl.FieldSpec(name="t_exptime", dtype=sqlalchemy.Float, doc="Total exposure time (sec)"), 

92 ddl.FieldSpec(name="t_resolution", dtype=sqlalchemy.Float, doc="Temporal resolution (sec)"), 

93 ddl.FieldSpec(name="em_xel", dtype=sqlalchemy.Integer, doc="Number of elements along the spectral axis"), 

94 ddl.FieldSpec(name="em_min", dtype=sqlalchemy.Float, doc="Start in spectral coordinates (m)"), 

95 ddl.FieldSpec(name="em_max", dtype=sqlalchemy.Float, doc="Stop in spectral coordinates (m)"), 

96 ddl.FieldSpec(name="em_res_power", dtype=sqlalchemy.Float, doc="Spectral resolving power"), 

97 ddl.FieldSpec( 

98 name="em_filter_name", dtype=sqlalchemy.String, length=255, doc="Filter name (non-standard column)" 

99 ), 

100 ddl.FieldSpec(name="o_ucd", dtype=sqlalchemy.String, length=255, doc="UCD of observable"), 

101 ddl.FieldSpec(name="pol_xel", dtype=sqlalchemy.Integer, doc="Number of polarization samples"), 

102 ddl.FieldSpec( 

103 name="instrument_name", 

104 dtype=sqlalchemy.String, 

105 length=255, 

106 doc="Name of the instrument used for this observation", 

107 ), 

108) 

109 

110_TYPE_MAP = { 

111 int: sqlalchemy.BigInteger, 

112 float: sqlalchemy.Float, 

113 bool: sqlalchemy.Boolean, 

114 str: sqlalchemy.String, 

115} 

116 

117 

118class ObsCoreSchema: 

119 """Generate table specification for an ObsCore table based on its 

120 configuration. 

121 

122 Parameters 

123 ---------- 

124 config : `ObsCoreConfig` 

125 ObsCore configuration instance. 

126 datasets : `type`, optional 

127 Type of dataset records manager. If specified, the ObsCore table will 

128 define a foreign key to ``datasets`` table with "ON DELETE CASCADE" 

129 constraint. 

130 collections : `type`, optional 

131 Manager of Registry collections. If specified, the ObsCore table will 

132 define a foreign key to ``run`` table with "ON DELETE CASCADE" 

133 constraint. 

134 

135 Notes 

136 ----- 

137 This class is designed to support both "live" obscore table which is 

138 located in the same database as the Registry, and standalone table in a 

139 completely separate database. Live obscore table depends on foreign key 

140 constraints with "ON DELETE CASCADE" option to manage lifetime of obscore 

141 records when their original datasets are removed. 

142 """ 

143 

144 def __init__( 

145 self, 

146 config: ObsCoreConfig, 

147 datasets: Optional[Type[DatasetRecordStorageManager]] = None, 

148 ): 

149 

150 fields = list(_STATIC_COLUMNS) 

151 

152 column_names = set(col.name for col in fields) 

153 

154 all_configs: List[ObsCoreConfig | DatasetTypeConfig] = [config] 

155 if config.dataset_types: 

156 all_configs += list(config.dataset_types.values()) 

157 for cfg in all_configs: 

158 if cfg.extra_columns: 

159 for col_name, col_value in cfg.extra_columns.items(): 

160 if col_name in column_names: 

161 continue 

162 if isinstance(col_value, ExtraColumnConfig): 

163 col_type = ddl.VALID_CONFIG_COLUMN_TYPES.get(col_value.type.name) 

164 col_length = col_value.length 

165 else: 

166 # Only value is provided, guess type from Python, and 

167 # use a fixed length of 255 for strings. 

168 col_type = _TYPE_MAP.get(type(col_value)) 

169 col_length = 255 if isinstance(col_value, str) else None 

170 if col_type is None: 

171 raise TypeError( 

172 f"Unexpected type in extra_columns: column={col_name}, value={col_value}" 

173 ) 

174 fields.append(ddl.FieldSpec(name=col_name, dtype=col_type, length=col_length, doc="")) 

175 column_names.add(col_name) 

176 

177 indices: List[Tuple[str, ...]] = [] 

178 if config.indices: 

179 for columns in config.indices.values(): 

180 indices.append(tuple(ensure_iterable(columns))) 

181 

182 self._table_spec = ddl.TableSpec(fields=fields, indexes=indices) 

183 

184 self._dataset_fk: Optional[ddl.FieldSpec] = None 

185 if datasets is not None: 

186 # Add FK to datasets, is also a PK for this table 

187 self._dataset_fk = datasets.addDatasetForeignKey( 

188 self._table_spec, name="registry_dataset", onDelete="CASCADE" 

189 ) 

190 self._dataset_fk.primaryKey = True 

191 

192 @property 

193 def table_spec(self) -> ddl.TableSpec: 

194 return self._table_spec 

195 

196 @property 

197 def dataset_fk(self) -> Optional[ddl.FieldSpec]: 

198 return self._dataset_fk