21 from __future__
import annotations
23 __all__ = [
"RootRepoConverter"]
28 from typing
import TYPE_CHECKING, Iterator, Tuple, List
30 from lsst.daf.butler
import DatasetType, DatasetRef, FileDataset
31 from .calibRepoConverter
import CURATED_CALIBRATION_DATASET_TYPES
32 from .standardRepoConverter
import StandardRepoConverter
34 SKYMAP_DATASET_TYPES = {
35 coaddName: f
"{coaddName}Coadd_skyMap" for coaddName
in (
"deep",
"goodSeeing",
"dcr")
39 from lsst.daf.butler
import SkyPixDimension
40 from ..ingest
import RawExposureData
44 """A specialization of `RepoConverter` for root data repositories. 46 `RootRepoConverter` adds support for raw images (mostly delegated to the 47 parent task's `RawIngestTask` subtask) and reference catalogs. 52 Keyword arguments are forwarded to (and required by) `RepoConverter`. 57 self._exposureData: List[RawExposureData] = []
58 self._refCats: List[Tuple[str, SkyPixDimension]] = []
64 datasetTypeName
in (
"raw",
"ref_cat",
"ref_cat_config")
or 66 datasetTypeName
in CURATED_CALIBRATION_DATASET_TYPES
71 return subdirectory ==
"ref_cats" 76 if self.
task.raws
is not None:
77 self.
task.log.info(f
"Preparing raws from root {self.root}.")
78 if self.
subset is not None:
79 dataRefs = itertools.chain.from_iterable(
84 self._exposureData.extend(self.
task.raws.prep(dataRef.getUri()
for dataRef
in dataRefs))
86 if self.
task.isDatasetTypeIncluded(
"ref_cat"):
87 from lsst.meas.algorithms
import DatasetConfig
as RefCatDatasetConfig
88 for refCat
in os.listdir(os.path.join(self.
root,
"ref_cats")):
89 self.
task.log.info(f
"Preparing ref_cat {refCat} from root {self.root}.")
90 path = os.path.join(self.
root,
"ref_cats", refCat)
91 configFile = os.path.join(path,
"config.py")
92 if not os.path.exists(configFile):
94 if not self.
task.isDatasetTypeIncluded(refCat):
99 onDiskConfig = RefCatDatasetConfig()
100 onDiskConfig.load(configFile)
101 if onDiskConfig.indexer.name !=
"HTM":
102 raise ValueError(f
"Reference catalog '{refCat}' uses unsupported " 103 f
"pixelization '{onDiskConfig.indexer.name}'.")
104 level = onDiskConfig.indexer[
"HTM"].depth
106 dimension = self.
task.universe[f
"htm{level}"]
107 except KeyError
as err:
108 raise ValueError(f
"Reference catalog {refCat} uses HTM level {level}, but no htm{level} " 109 f
"skypix dimension is configured for this registry.")
from err
110 self.
task.useSkyPix(dimension)
111 self._refCats.append((refCat, dimension))
116 self.
task.log.info(f
"Inserting observation dimension records from {self.root}.")
117 records = {
"visit": [],
"exposure": [],
"visit_detector_region": []}
118 for exposure
in self._exposureData:
119 for dimension, recordsForDimension
in exposure.records.items():
120 records[dimension].extend(recordsForDimension)
121 self.
task.raws.insertDimensionData(records)
126 for refCat, dimension
in self._refCats:
127 datasetType = DatasetType(refCat, dimensions=[dimension], universe=self.
task.universe,
128 storageClass=
"SimpleCatalog")
130 regex = re.compile(
r"(\d+)\.fits")
131 for fileName
in os.listdir(os.path.join(self.
root,
"ref_cats", refCat)):
132 m = regex.match(fileName)
134 htmId = int(m.group(1))
135 dataId = self.
task.registry.expandDataId({dimension: htmId})
136 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, fileName),
137 ref=DatasetRef(datasetType, dataId))
139 for htmId
in self.
subset.skypix[dimension]:
140 dataId = self.
task.registry.expandDataId({dimension: htmId})
141 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, f
"{htmId}.fits"),
142 ref=DatasetRef(datasetType, dataId))
147 if self.
task.raws
is not None:
148 self.
task.log.info(f
"Ingesting raws from root {self.root}.")
149 self.
task.registry.registerDatasetType(self.
task.raws.datasetType)
155 butler, collections = self.
getButler(
"raw")
156 for exposure
in self._exposureData:
157 refs.extend(self.
task.raws.ingestExposureDatasets(exposure))
158 for collection
in collections:
159 self.
task.registry.associate(collection, refs)
def insertDimensionData(self)