21 from __future__
import annotations
23 __all__ = [
"ConvertRepoConfig",
"ConvertRepoTask",
"ConvertRepoSkyMapConfig"]
27 from dataclasses
import dataclass
28 from typing
import Iterable, Optional, List, Dict
31 from lsst.daf.butler
import (
35 from lsst.pex.config
import Config, ConfigurableField, ConfigDictField, DictField, ListField, Field
36 from lsst.pipe.base
import Task
37 from lsst.skymap
import skyMapRegistry, BaseSkyMap
39 from ..ingest
import RawIngestTask
40 from .repoConverter
import ConversionSubset
41 from .rootRepoConverter
import RootRepoConverter
42 from .calibRepoConverter
import CalibRepoConverter
43 from .standardRepoConverter
import StandardRepoConverter
48 """Struct containing information about a skymap that may appear in a Gen2 53 """Name of the skymap used in Gen3 data IDs. 57 """Hash computed by `BaseSkyMap.getSha1`. 61 """Name of the skymap used in Gen3 data IDs. 65 """Whether this skymap has been found in at least one repository being 71 """Sub-config used to hold the parameters of a SkyMap. 75 This config only needs to exist because we can't put a 76 `~lsst.pex.config.RegistryField` directly inside a 77 `~lsst.pex.config.ConfigDictField`. 79 It needs to have its only field named "skyMap" for compatibility with the 80 configuration of `lsst.pipe.tasks.MakeSkyMapTask`, which we want so we can 81 use one config file in an obs package to configure both. 83 This name leads to unfortunate repetition with the field named 84 "skymap" that holds it - "skyMap[name].skyMap" - but that seems 87 skyMap = skyMapRegistry.makeField(
88 doc=
"Type and parameters for the SkyMap itself.",
94 raws = ConfigurableField(
95 "Configuration for subtask responsible for ingesting raws and adding " 96 "visit and exposure dimension entries.",
99 skyMaps = ConfigDictField(
100 "Mapping from Gen3 skymap name to the parameters used to construct a " 101 "BaseSkyMap instance. This will be used to associated names with " 102 "existing skymaps found in the Gen2 repo.",
104 itemtype=ConvertRepoSkyMapConfig,
107 collections = DictField(
108 "Special collections (values) for certain dataset types (keys). " 109 "These are used in addition to rerun collections for datasets in " 110 "reruns. The 'raw' dataset must have an entry here if it is to be " 115 "deepCoadd_skyMap":
"skymaps",
116 "brightObjectMask":
"masks",
119 storageClasses = DictField(
120 "Mapping from dataset type name or Gen2 policy entry (e.g. 'python' " 121 "or 'persistable') to the Gen3 StorageClass name.",
125 "BaseSkyMap":
"SkyMap",
126 "BaseCatalog":
"Catalog",
127 "BackgroundList":
"Background",
131 doRegisterInstrument = Field(
132 "If True (default), add dimension records for the Instrument and its " 133 "filters and detectors to the registry instead of assuming they are " 138 doWriteCuratedCalibrations = Field(
139 "If True (default), ingest human-curated calibrations directly via " 140 "the Instrument interface. Note that these calibrations are never " 141 "converted from Gen2 repositories.",
146 "The names of reference catalogs (subdirectories under ref_cats) to " 151 fileIgnorePatterns = ListField(
152 "Filename globs that should be ignored instead of being treated as " 155 default=[
"README.txt",
"*~?",
"butler.yaml",
"gen3.sqlite3"]
157 datasetIncludePatterns = ListField(
158 "Glob-style patterns for dataset type names that should be converted.",
162 datasetIgnorePatterns = ListField(
163 "Glob-style patterns for dataset type names that should not be " 164 "converted despite matching a pattern in datasetIncludePatterns.",
169 "Key used for the Gen2 equivalent of 'detector' in data IDs.",
174 "If True (default), only convert datasets that are related to the " 175 "ingested visits. Ignored unless a list of visits is passed to " 183 return self.
raws.transfer
187 self.
raws.transfer = value
191 return self.
raws.instrument
195 self.
raws.instrument = value
205 """A task that converts one or more related Gen2 data repositories to a 206 single Gen3 data repository (with multiple collections). 210 config: `ConvertRepoConfig` 211 Configuration for this task. 212 butler3: `lsst.daf.butler.Butler` 213 Gen3 Butler instance that represents the data repository datasets will 214 be ingested into. The collection and/or run associated with this 215 Butler will be ignored in favor of collections/runs passed via config 218 Other keyword arguments are forwarded to the `Task` constructor. 222 Most of the work of converting repositories is delegated to instances of 223 the `RepoConverter` hierarchy. The `ConvertRepoTask` instance itself holds 224 only state that is relevant for all Gen2 repositories being ingested, while 225 each `RepoConverter` instance holds only state relevant for the conversion 226 of a single Gen2 repository. Both the task and the `RepoConverter` 227 instances are single use; `ConvertRepoTask.run` and most `RepoConverter` 228 methods may only be called once on a particular instance. 231 ConfigClass = ConvertRepoConfig
233 _DefaultName =
"convertRepo" 235 def __init__(self, config=None, *, butler3: Butler3, **kwds):
241 self.makeSubtask(
"raws", butler=butler3)
245 self.
instrument = doImport(self.config.instrument)()
248 for name, config
in self.config.skyMaps.items():
249 instance = config.skyMap.apply()
250 struct =
ConfiguredSkyMap(name=name, sha1=instance.getSha1(), instance=instance)
256 """Return `True` if configuration indicates that the given dataset type 259 This method is intended to be called primarily by the 260 `RepoConverter` instances used interally by the task. 265 Name of the dataset type. 270 Whether the dataset should be included in the conversion. 273 any(fnmatch.fnmatchcase(datasetTypeName, pattern)
274 for pattern
in self.config.datasetIncludePatterns)
and 275 not any(fnmatch.fnmatchcase(datasetTypeName, pattern)
276 for pattern
in self.config.datasetIgnorePatterns)
280 """Indicate that a repository uses the given SkyMap. 282 This method is intended to be called primarily by the 283 `RepoConverter` instances used interally by the task. 287 skyMap : `lsst.skymap.BaseSkyMap` 288 SkyMap instance being used, typically retrieved from a Gen2 294 The name of the skymap in Gen3 data IDs. 296 sha1 = skyMap.getSha1()
299 except KeyError
as err:
300 raise LookupError(f
"SkyMap with sha1={sha1} not included in configuration.")
from err
305 """Register all skymaps that have been marked as used. 307 This method is intended to be called primarily by the 308 `RepoConverter` instances used interally by the task. 312 subset : `ConversionSubset`, optional 313 Object that will be used to filter converted datasets by data ID. 314 If given, it will be updated with the tracts of this skymap that 315 overlap the visits in the subset. 319 struct.instance.register(struct.name, self.
registry)
320 if subset
is not None and self.config.relatedOnly:
321 subset.addSkyMap(self.
registry, struct.name)
324 """Indicate that a repository uses the given SkyPix dimension. 326 This method is intended to be called primarily by the 327 `RepoConverter` instances used interally by the task. 331 dimension : `lsst.daf.butler.SkyPixDimension` 332 Dimension represening a pixelization of the sky. 337 """Register all skymaps that have been marked as used. 339 This method is intended to be called primarily by the 340 `RepoConverter` instances used interally by the task. 344 subset : `ConversionSubset`, optional 345 Object that will be used to filter converted datasets by data ID. 346 If given, it will be updated with the pixelization IDs that 347 overlap the visits in the subset. 349 if subset
is not None and self.config.relatedOnly:
351 subset.addSkyPix(self.
registry, dimension)
353 def run(self, root: str, collections: List[str], *,
354 calibs: Dict[str, List[str]] =
None,
355 reruns: Dict[str, List[str]] =
None,
356 visits: Optional[Iterable[int]] =
None):
357 """Convert a group of related data repositories. 362 Complete path to the root Gen2 data repository. This should be 363 a data repository that includes a Gen2 registry and any raw files 364 and/or reference catalogs. 365 collections : `list` of `str` 366 Gen3 collections that datasets from the root repository should be 367 associated with. This should include any rerun collection that 368 these datasets should also be considered to be part of; because of 369 structural difference between Gen2 parent/child relationships and 370 Gen3 collections, these cannot be reliably inferred. 372 Dictionary mapping calibration repository path to the collections 373 that the repository's datasets should be associated with. The path 374 may be relative to ``root`` or absolute. Collections should 375 include child repository collections as appropriate (see 376 documentation for ``collections``). 378 Dictionary mapping rerun repository path to the collections that 379 the repository's datasets should be associated with. The path may 380 be relative to ``root`` or absolute. Collections should include 381 child repository collections as appropriate (see documentation for 383 visits : iterable of `int`, optional 384 The integer IDs of visits to convert. If not provided, all visits 385 in the Gen2 root repository will be converted. 392 if visits
is not None:
395 if self.config.relatedOnly:
396 self.log.warn(
"config.relatedOnly is True but all visits are being ingested; " 397 "no filtering will be done.")
400 if self.config.doRegisterInstrument:
406 rootConverter =
RootRepoConverter(task=self, root=root, collections=collections, subset=subset)
408 converters.append(rootConverter)
410 for root, collections
in calibs.items():
411 if not os.path.isabs(root):
412 root = os.path.join(rootConverter.root, root)
414 mapper=rootConverter.mapper,
415 subset=rootConverter.subset)
417 converters.append(converter)
419 for root, collections
in reruns.items():
420 if not os.path.isabs(root):
421 root = os.path.join(rootConverter.root, root)
423 subset=rootConverter.subset)
425 converters.append(converter)
442 for converter
in converters:
443 converter.insertDimensionData()
454 for converter
in converters:
def isDatasetTypeIncluded