21 from __future__
import annotations
23 __all__ = [
"RootRepoConverter"]
28 from typing
import TYPE_CHECKING, Iterator, Optional, Tuple, List
30 from lsst.skymap
import BaseSkyMap
31 from lsst.daf.butler
import DatasetType, DatasetRef, DimensionGraph, FileDataset
32 from .standardRepoConverter
import StandardRepoConverter
34 SKYMAP_DATASET_TYPES = {
35 coaddName: f
"{coaddName}Coadd_skyMap" for coaddName
in (
"deep",
"goodSeeing",
"dcr")
39 from lsst.daf.butler
import SkyPixDimension
43 """Strip HDU identifiers from paths and return a unique set of paths.
47 dataRefs : `lsst.daf.persistence.ButlerDataRef`
48 The gen2 datarefs to strip "[HDU]" values from.
53 The unique file paths without appended "[HDU]".
56 for dataRef
in dataRefs:
57 path = dataRef.getUri()
59 paths.add(path.split(
'[')[0])
64 """A specialization of `RepoConverter` for root data repositories.
66 `RootRepoConverter` adds support for raw images (mostly delegated to the
67 parent task's `RawIngestTask` subtask) and reference catalogs.
72 Keyword arguments are forwarded to (and required by) `RepoConverter`.
77 self._refCats: List[Tuple[str, SkyPixDimension]] = []
78 if self.
task.config.rootSkyMapName
is not None:
88 or datasetTypeName
in (
"raw",
"ref_cat",
"ref_cat_config")
90 or datasetTypeName
in self.
instrument.getCuratedCalibrationNames()
97 def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap], Optional[str]]:
100 if skyMap
is None and self.
task.config.rootSkyMapName
is not None:
102 (
"Assuming configured root skymap with name '%s' for dataset %s."),
103 self.
task.config.rootSkyMapName, datasetTypeName
106 name = self.
task.config.rootSkyMapName
110 if self.
task.raws
is None:
112 self.
task.log.info(f
"Finding raws in root {self.root}.")
113 if self.
subset is not None:
114 dataRefs = itertools.chain.from_iterable(
116 visit=visit)
for visit
in self.
subset.visits
121 self.
task.log.info(
"Ingesting raws from root %s into run %s.", self.
root, self.
task.raws.butler.run)
122 self.
_rawRefs.extend(self.
task.raws.run(dataPaths, pool=pool))
126 if self.
task.defineVisits
is None:
128 dimensions = DimensionGraph(self.
task.universe, names=[
"exposure"])
129 exposureDataIds = set(ref.dataId.subset(dimensions)
for ref
in self.
_rawRefs)
130 self.
task.log.info(
"Defining visits from exposures.")
131 self.
task.defineVisits.run(exposureDataIds, pool=pool)
136 if self.
task.isDatasetTypeIncluded(
"ref_cat")
and len(self.
task.config.refCats) != 0:
137 from lsst.meas.algorithms
import DatasetConfig
as RefCatDatasetConfig
138 for refCat
in os.listdir(os.path.join(self.
root,
"ref_cats")):
139 path = os.path.join(self.
root,
"ref_cats", refCat)
140 configFile = os.path.join(path,
"config.py")
141 if not os.path.exists(configFile):
143 if refCat
not in self.
task.config.refCats:
145 self.
task.log.info(f
"Preparing ref_cat {refCat} from root {self.root}.")
146 onDiskConfig = RefCatDatasetConfig()
147 onDiskConfig.load(configFile)
148 if onDiskConfig.indexer.name !=
"HTM":
149 raise ValueError(f
"Reference catalog '{refCat}' uses unsupported "
150 f
"pixelization '{onDiskConfig.indexer.name}'.")
151 level = onDiskConfig.indexer[
"HTM"].depth
153 dimension = self.
task.universe[f
"htm{level}"]
154 except KeyError
as err:
155 raise ValueError(f
"Reference catalog {refCat} uses HTM level {level}, but no htm{level} "
156 f
"skypix dimension is configured for this registry.")
from err
157 self.
task.useSkyPix(dimension)
158 self._refCats.append((refCat, dimension))
159 if self.
task.isDatasetTypeIncluded(
"brightObjectMask")
and self.
task.config.rootSkyMapName:
166 for refCat, dimension
in self._refCats:
167 datasetType = DatasetType(refCat, dimensions=[dimension], universe=self.
task.universe,
168 storageClass=
"SimpleCatalog")
170 regex = re.compile(
r"(\d+)\.fits")
171 for fileName
in os.listdir(os.path.join(self.
root,
"ref_cats", refCat)):
172 m = regex.match(fileName)
174 htmId = int(m.group(1))
175 dataId = self.
task.registry.expandDataId({dimension: htmId})
176 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, fileName),
177 refs=DatasetRef(datasetType, dataId))
179 for begin, end
in self.
subset.skypix[dimension]:
180 for htmId
in range(begin, end):
181 dataId = self.
task.registry.expandDataId({dimension: htmId})
182 yield FileDataset(path=os.path.join(self.
root,
"ref_cats", refCat, f
"{htmId}.fits"),
183 refs=DatasetRef(datasetType, dataId))