21 from __future__
import annotations
23 __all__ = [
"CalibRepoConverter"]
25 from collections
import defaultdict
28 from typing
import TYPE_CHECKING, Dict, Iterator, List, Mapping, Tuple, Optional
32 from lsst.daf.butler
import DataCoordinate, FileDataset, Timespan
33 from .repoConverter
import RepoConverter
34 from .repoWalker
import RepoWalker
37 from lsst.daf.butler
import DatasetType, StorageClass, FormatterParameter
38 from .repoWalker.scanner
import PathElementHandler
39 from ..cameraMapper
import CameraMapper
40 from ..mapping
import Mapping
as CameraMapperMapping
44 """A specialization of `RepoConverter` for calibration repositories.
48 mapper : `CameraMapper`
49 Gen2 mapper for the data repository. The root associated with the
50 mapper is ignored and need not match the root of the repository.
52 Additional keyword arguments are forwarded to (and required by)
56 def __init__(self, *, mapper: CameraMapper, collection: str, **kwds):
64 return datasetTypeName
in self.
instrument.getCuratedCalibrationNames()
66 def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
68 yield from self.
mapper.calibrations.items()
71 storageClass: StorageClass, formatter: FormatterParameter =
None,
72 targetHandler: Optional[PathElementHandler] =
None,
73 ) -> RepoWalker.Target:
75 target = RepoWalker.Target(
76 datasetTypeName=datasetTypeName,
77 storageClass=storageClass,
80 instrument=self.
task.instrument.getName(),
81 universe=self.
task.registry.dimensions,
83 targetHandler=targetHandler,
84 translatorFactory=self.
task.translatorFactory,
89 def _queryGen2CalibRegistry(self, db: sqlite3.Connection, datasetType: DatasetType, calibDate: str
90 ) -> Iterator[sqlite3.Row]:
92 fields = [
"validStart",
"validEnd"]
93 if "detector" in datasetType.dimensions.names:
94 fields.append(self.
task.config.ccdKey)
96 fields.append(f
"NULL AS {self.task.config.ccdKey}")
97 if "physical_filter" in datasetType.dimensions.names:
98 fields.append(
"filter")
100 assert "band" not in datasetType.dimensions.names
101 fields.append(
"NULL AS filter")
102 tables = self.
mapper.mappings[datasetType.name].tables
103 if tables
is None or len(tables) == 0:
104 self.
task.log.warn(
"Could not extract calibration ranges for %s in %s; "
105 "no tables in Gen2 mapper.",
106 datasetType.name, self.
root, tables[0])
108 query = f
"SELECT DISTINCT {', '.join(fields)} FROM {tables[0]} WHERE calibDate = ?;"
110 results = db.execute(query, (calibDate,))
111 except sqlite3.OperationalError
as e:
112 self.
task.log.warn(
"Could not extract calibration ranges for %s in %s from table %s: %r",
113 datasetType.name, self.
root, tables[0], e)
117 def _finish(self, datasets: Mapping[DatasetType, Mapping[Optional[str], List[FileDataset]]]):
120 calibFile = os.path.join(self.
root,
"calibRegistry.sqlite3")
124 if not os.path.exists(calibFile):
125 raise RuntimeError(
"Attempting to convert calibrations but no registry database"
126 f
" found in {self.root}")
130 refsByTimespan = defaultdict(list)
131 db = sqlite3.connect(calibFile)
132 db.row_factory = sqlite3.Row
133 day = astropy.time.TimeDelta(1, format=
"jd", scale=
"tai")
134 for datasetType, datasetsByCalibDate
in datasets.items():
135 if not datasetType.isCalibration():
138 if "detector" in datasetType.dimensions.names:
139 gen2keys[self.
task.config.ccdKey] = int
140 if "physical_filter" in datasetType.dimensions.names:
141 gen2keys[
"filter"] = str
142 translator = self.
instrument.makeDataIdTranslatorFactory().makeMatching(
147 for calibDate, datasetsForCalibDate
in datasetsByCalibDate.items():
148 assert calibDate
is not None, (
"datasetType.isCalibration() is set by "
149 "the presence of calibDate in the Gen2 template")
155 for dataset
in datasetsForCalibDate:
156 refsByDataId.update((ref.dataId, ref)
for ref
in dataset.refs)
164 astropy.time.Time(row[
"validStart"], format=
"iso", scale=
"tai"),
165 astropy.time.Time(row[
"validEnd"], format=
"iso", scale=
"tai") + day,
169 if "detector" in datasetType.dimensions.names:
170 gen2id[self.
task.config.ccdKey] = row[self.
task.config.ccdKey]
171 if "physical_filter" in datasetType.dimensions.names:
172 gen2id[
"filter"] = row[
"filter"]
174 gen3id, _ = translator(gen2id)
175 dataId = DataCoordinate.standardize(gen3id, graph=datasetType.dimensions)
176 ref = refsByDataId.get(dataId)
178 refsByTimespan[timespan].append(ref)
188 "Gen2 calibration registry entry has no dataset: %s for calibDate=%s, %s.",
189 datasetType.name, calibDate, dataId
192 for timespan, refs
in refsByTimespan.items():
195 def getRun(self, datasetTypeName: str, calibDate: Optional[str] =
None) -> str:
196 if calibDate
is None:
197 return super().
getRun(datasetTypeName)
199 return self.
instrument.makeCollectionName(
"calib",
"gen2", calibDate)
205 """Gen2 mapper associated with this repository.