21 from __future__
import annotations
23 __all__ = [
"RootRepoConverter"]
28 from typing
import TYPE_CHECKING, Iterator, Optional, Tuple, List
30 from lsst.skymap
import BaseSkyMap
31 from lsst.daf.butler
import DatasetType, DatasetRef, FileDataset
32 from .standardRepoConverter
import StandardRepoConverter
34 SKYMAP_DATASET_TYPES = {
35 coaddName: f
"{coaddName}Coadd_skyMap" for coaddName
in (
"deep",
"goodSeeing",
"dcr")
39 from lsst.daf.butler
import SkyPixDimension
40 from ..ingest
import RawExposureData
44 """Strip HDU identifiers from paths and return a unique set of paths. 48 dataRefs : `lsst.daf.persistence.ButlerDataRef` 49 The gen2 datarefs to strip "[HDU]" values from. 54 The unique file paths without appended "[HDU]". 57 for dataRef
in dataRefs:
58 path = dataRef.getUri()
60 paths.add(path.split(
'[')[0])
65 """A specialization of `RepoConverter` for root data repositories. 67 `RootRepoConverter` adds support for raw images (mostly delegated to the 68 parent task's `RawIngestTask` subtask) and reference catalogs. 73 Keyword arguments are forwarded to (and required by) `RepoConverter`. 78 self._exposureData: List[RawExposureData] = []
79 self._refCats: List[Tuple[str, SkyPixDimension]] = []
80 if self.
task.config.rootSkyMapName
is not None:
89 or datasetTypeName
in (
"raw",
"ref_cat",
"ref_cat_config")
91 or datasetTypeName
in self.
task.config.curatedCalibrations
98 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
101 if skyMap
is None and self.
task.config.rootSkyMapName
is not None:
103 (
"Assuming configured root skymap with name '%s' for dataset %s."),
104 self.
task.config.rootSkyMapName, datasetTypeName
107 name = self.
task.config.rootSkyMapName
113 if self.
task.raws
is not None:
114 self.
task.log.info(f
"Preparing raws from root {self.root}.")
115 if self.
subset is not None:
116 dataRefs = itertools.chain.from_iterable(
118 visit=visit)
for visit
in self.
subset.visits
123 self.
task.log.debug(
"Prepping files: %s", dataPaths)
124 self._exposureData.extend(self.
task.raws.prep(dataPaths))
126 if self.
task.isDatasetTypeIncluded(
"ref_cat")
and len(self.
task.config.refCats) != 0:
127 from lsst.meas.algorithms
import DatasetConfig
as RefCatDatasetConfig
128 for refCat
in os.listdir(os.path.join(self.
root,
"ref_cats")):
129 path = os.path.join(self.
root,
"ref_cats", refCat)
130 configFile = os.path.join(path,
"config.py")
131 if not os.path.exists(configFile):
133 if refCat
not in self.
task.config.refCats:
135 self.
task.log.info(f
"Preparing ref_cat {refCat} from root {self.root}.")
136 onDiskConfig = RefCatDatasetConfig()
137 onDiskConfig.load(configFile)
138 if onDiskConfig.indexer.name !=
"HTM":
139 raise ValueError(f
"Reference catalog '{refCat}' uses unsupported " 140 f
"pixelization '{onDiskConfig.indexer.name}'.")
141 level = onDiskConfig.indexer[
"HTM"].depth
143 dimension = self.
task.universe[f
"htm{level}"]
144 except KeyError
as err:
145 raise ValueError(f
"Reference catalog {refCat} uses HTM level {level}, but no htm{level} " 146 f
"skypix dimension is configured for this registry.")
from err
147 self.
task.useSkyPix(dimension)
148 self._refCats.append((refCat, dimension))
149 if self.
task.isDatasetTypeIncluded(
"brightObjectMask")
and self.
task.config.rootSkyMapName:
155 self.
task.log.info(f
"Inserting observation dimension records from {self.root}.")
156 records = {
"visit": [],
"exposure": [],
"visit_detector_region": []}
157 for exposure
in self._exposureData:
158 for dimension, recordsForDimension
in exposure.records.items():
159 records[dimension].extend(recordsForDimension)
160 self.
task.raws.insertDimensionData(records)
165 for refCat, dimension
in self._refCats:
166 datasetType = DatasetType(refCat, dimensions=[dimension], universe=self.
task.universe,
167 storageClass=
"SimpleCatalog")
169 regex = re.compile(
r"(\d+)\.fits")
170 for fileName
in os.listdir(os.path.join(self.
root,
"ref_cats", refCat)):
171 m = regex.match(fileName)
173 htmId = int(m.group(1))
174 dataId = self.
task.registry.expandDataId({dimension: htmId})
175 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, fileName),
176 refs=DatasetRef(datasetType, dataId))
178 for begin, end
in self.
subset.skypix[dimension]:
179 for htmId
in range(begin, end):
180 dataId = self.
task.registry.expandDataId({dimension: htmId})
181 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, f
"{htmId}.fits"),
182 refs=DatasetRef(datasetType, dataId))
187 if self.
task.raws
is not None:
188 self.
task.log.info(f
"Ingesting raws from root {self.root}.")
189 self.
task.registry.registerDatasetType(self.
task.raws.datasetType)
196 for exposure
in self._exposureData:
197 refs.extend(self.
task.raws.ingestExposureDatasets(exposure))
198 for collection
in collections[1:]:
199 self.
task.registry.associate(collection, refs)
204 if datasetTypeName
in self.
task.config.refCats:
def insertDimensionData(self)
def getDataPaths(dataRefs)
def getSpecialDirectories(self)