Coverage for python/lsst/daf/butler/registry/obscore/_schema.py: 24%
47 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-12 09:01 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["ObsCoreSchema"]
26from typing import TYPE_CHECKING, List, Optional, Tuple, Type
28import sqlalchemy
29from lsst.daf.butler import ddl
30from lsst.utils.iteration import ensure_iterable
32from ._config import DatasetTypeConfig, ExtraColumnConfig, ObsCoreConfig
34if TYPE_CHECKING: 34 ↛ 35line 34 didn't jump to line 35, because the condition on line 34 was never true
35 from ..interfaces import DatasetRecordStorageManager
38# List of standard columns in output file. This should include at least all
39# mandatory columns defined in ObsCore note (revision 1.1, Appendix B). Extra
40# columns can be added via `extra_columns` parameters in configuration.
41_STATIC_COLUMNS = (
42 ddl.FieldSpec(
43 name="dataproduct_type", dtype=sqlalchemy.String, length=255, doc="Logical data product type"
44 ),
45 ddl.FieldSpec(
46 name="dataproduct_subtype", dtype=sqlalchemy.String, length=255, doc="Data product specific type"
47 ),
48 ddl.FieldSpec(
49 name="facility_name",
50 dtype=sqlalchemy.String,
51 length=255,
52 doc="The name of the facility used for the observation",
53 ),
54 ddl.FieldSpec(name="calib_level", dtype=sqlalchemy.SmallInteger, doc="Calibration level {0, 1, 2, 3, 4}"),
55 ddl.FieldSpec(name="target_name", dtype=sqlalchemy.String, length=255, doc="Object of interest"),
56 ddl.FieldSpec(name="obs_id", dtype=sqlalchemy.String, length=255, doc="Observation ID"),
57 ddl.FieldSpec(
58 name="obs_collection", dtype=sqlalchemy.String, length=255, doc="Name of the data collection"
59 ),
60 ddl.FieldSpec(
61 name="obs_publisher_did",
62 dtype=sqlalchemy.String,
63 length=255,
64 doc="Dataset identifier given by the publisher",
65 ),
66 ddl.FieldSpec(
67 name="access_url", dtype=sqlalchemy.String, length=65535, doc="URL used to access (download) dataset"
68 ),
69 ddl.FieldSpec(name="access_format", dtype=sqlalchemy.String, length=255, doc="File content format"),
70 ddl.FieldSpec(name="s_ra", dtype=sqlalchemy.Float, doc="Central right ascension, ICRS (deg)"),
71 ddl.FieldSpec(name="s_dec", dtype=sqlalchemy.Float, doc="Central declination, ICRS (deg)"),
72 ddl.FieldSpec(name="s_fov", dtype=sqlalchemy.Float, doc="Diameter (bounds) of the covered region (deg)"),
73 ddl.FieldSpec(
74 name="s_region",
75 dtype=sqlalchemy.String,
76 length=65535,
77 doc="Sky region covered by the data product (expressed in ICRS frame)",
78 ),
79 ddl.FieldSpec(
80 name="s_resolution", dtype=sqlalchemy.Float, doc="Spatial resolution of data as FWHM (arcsec)"
81 ),
82 ddl.FieldSpec(
83 name="s_xel1", dtype=sqlalchemy.Integer, doc="Number of elements along the first spatial axis"
84 ),
85 ddl.FieldSpec(
86 name="s_xel2", dtype=sqlalchemy.Integer, doc="Number of elements along the second spatial axis"
87 ),
88 ddl.FieldSpec(name="t_xel", dtype=sqlalchemy.Integer, doc="Number of elements along the time axis"),
89 ddl.FieldSpec(name="t_min", dtype=sqlalchemy.Float, doc="Start time in MJD"),
90 ddl.FieldSpec(name="t_max", dtype=sqlalchemy.Float, doc="Stop time in MJD"),
91 ddl.FieldSpec(name="t_exptime", dtype=sqlalchemy.Float, doc="Total exposure time (sec)"),
92 ddl.FieldSpec(name="t_resolution", dtype=sqlalchemy.Float, doc="Temporal resolution (sec)"),
93 ddl.FieldSpec(name="em_xel", dtype=sqlalchemy.Integer, doc="Number of elements along the spectral axis"),
94 ddl.FieldSpec(name="em_min", dtype=sqlalchemy.Float, doc="Start in spectral coordinates (m)"),
95 ddl.FieldSpec(name="em_max", dtype=sqlalchemy.Float, doc="Stop in spectral coordinates (m)"),
96 ddl.FieldSpec(name="em_res_power", dtype=sqlalchemy.Float, doc="Spectral resolving power"),
97 ddl.FieldSpec(
98 name="em_filter_name", dtype=sqlalchemy.String, length=255, doc="Filter name (non-standard column)"
99 ),
100 ddl.FieldSpec(name="o_ucd", dtype=sqlalchemy.String, length=255, doc="UCD of observable"),
101 ddl.FieldSpec(name="pol_xel", dtype=sqlalchemy.Integer, doc="Number of polarization samples"),
102 ddl.FieldSpec(
103 name="instrument_name",
104 dtype=sqlalchemy.String,
105 length=255,
106 doc="Name of the instrument used for this observation",
107 ),
108)
110_TYPE_MAP = {
111 int: sqlalchemy.BigInteger,
112 float: sqlalchemy.Float,
113 bool: sqlalchemy.Boolean,
114 str: sqlalchemy.String,
115}
118class ObsCoreSchema:
119 """Generate table specification for an ObsCore table based on its
120 configuration.
122 Parameters
123 ----------
124 config : `ObsCoreConfig`
125 ObsCore configuration instance.
126 datasets : `type`, optional
127 Type of dataset records manager. If specified, the ObsCore table will
128 define a foreign key to ``datasets`` table with "ON DELETE CASCADE"
129 constraint.
130 collections : `type`, optional
131 Manager of Registry collections. If specified, the ObsCore table will
132 define a foreign key to ``run`` table with "ON DELETE CASCADE"
133 constraint.
135 Notes
136 -----
137 This class is designed to support both "live" obscore table which is
138 located in the same database as the Registry, and standalone table in a
139 completely separate database. Live obscore table depends on foreign key
140 constraints with "ON DELETE CASCADE" option to manage lifetime of obscore
141 records when their original datasets are removed.
142 """
144 def __init__(
145 self,
146 config: ObsCoreConfig,
147 datasets: Optional[Type[DatasetRecordStorageManager]] = None,
148 ):
150 fields = list(_STATIC_COLUMNS)
152 column_names = set(col.name for col in fields)
154 all_configs: List[ObsCoreConfig | DatasetTypeConfig] = [config]
155 if config.dataset_types:
156 all_configs += list(config.dataset_types.values())
157 for cfg in all_configs:
158 if cfg.extra_columns:
159 for col_name, col_value in cfg.extra_columns.items():
160 if col_name in column_names:
161 continue
162 if isinstance(col_value, ExtraColumnConfig):
163 col_type = ddl.VALID_CONFIG_COLUMN_TYPES.get(col_value.type.name)
164 col_length = col_value.length
165 else:
166 # Only value is provided, guess type from Python, and
167 # use a fixed length of 255 for strings.
168 col_type = _TYPE_MAP.get(type(col_value))
169 col_length = 255 if isinstance(col_value, str) else None
170 if col_type is None:
171 raise TypeError(
172 f"Unexpected type in extra_columns: column={col_name}, value={col_value}"
173 )
174 fields.append(ddl.FieldSpec(name=col_name, dtype=col_type, length=col_length, doc=""))
175 column_names.add(col_name)
177 indices: List[Tuple[str, ...]] = []
178 if config.indices:
179 for columns in config.indices.values():
180 indices.append(tuple(ensure_iterable(columns)))
182 self._table_spec = ddl.TableSpec(fields=fields, indexes=indices)
184 self._dataset_fk: Optional[ddl.FieldSpec] = None
185 if datasets is not None:
186 # Add FK to datasets, is also a PK for this table
187 self._dataset_fk = datasets.addDatasetForeignKey(
188 self._table_spec, name="registry_dataset", onDelete="CASCADE"
189 )
190 self._dataset_fk.primaryKey = True
192 @property
193 def table_spec(self) -> ddl.TableSpec:
194 return self._table_spec
196 @property
197 def dataset_fk(self) -> Optional[ddl.FieldSpec]:
198 return self._dataset_fk