22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
29 import lsst.daf.butler
33 from ..gen2to3
import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34 Translator, ConstantKeyHandler, CopyKeyHandler,
35 CalibKeyHandler, Rerun)
39 parser = argparse.ArgumentParser(description=__doc__,
40 formatter_class=argparse.RawDescriptionHelpFormatter)
41 parser.add_argument(
"instrumentClass", metavar=
"lsst.obs.CAMERA.INSTRUMENT",
42 help=(
"The full import path to the gen3 Instrument class for this camera"
43 " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44 parser.add_argument(
"--gen2root", required=
True,
45 help=
"Root path of the gen 2 repo to be converted.")
46 parser.add_argument(
"--gen3root", required=
True,
47 help=
"Root path of the gen 3 repo to be produced.")
48 parser.add_argument(
"--skymapName", default=
None,
49 help=
"Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50 parser.add_argument(
"--skymapConfig", default=
None,
51 help=
"Path to skymap config file defining the new gen3 skymap.")
52 parser.add_argument(
"--calibs", default=
None,
53 help=
"Path to the gen 2 calibration repo; absolute, or relative to gen2root.")
54 parser.add_argument(
"--reruns", default=[], nargs=
"*",
55 help=
"List of gen 2 reruns to convert.")
56 parser.add_argument(
"--transferMode", default=
"auto",
57 choices=[
"auto",
"link",
"symlink",
"hardlink",
"copy",
"move",
"relsymlink"],
58 help=
"Mode to use to transfer files into the new repository.")
59 parser.add_argument(
"-v",
"--verbose", action=
"store_const", dest=
"verbose",
60 default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
61 help=
"Set the log level to DEBUG.")
62 parser.add_argument(
"--calibFilterType", default=
"physical_filter",
63 help=
"physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
64 parser.add_argument(
"-c",
"--config", default=
None,
65 help=(
"Path to a `ConvertRepoConfig` override to be included after "
66 "the Instrument config overrides are applied."))
72 args = parser.parse_args()
74 skymapList = [args.skymapName, args.skymapConfig]
75 if not all(x
is None for x
in skymapList)
and not all(x
is not None for x
in skymapList):
76 parser.error(
"Must specify both --skymapName and --skymapConfig, or neither.")
82 """Configure the gen3 translators so they know the correct instrument name.
86 instrument : `lsst.obs.base.Instrument`
87 The instrument that conversion is going to be run on.
88 calibFilterType : `str`
89 Whether the gen2 calibRegistry uses ``physical_filter`` or
90 ``abstract_filter`` as the ``filter`` key.
91 ccdKey : `str`, optional
92 The gen2 key used to identify what in gen3 is `detector`.
96 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
97 instrument=instrument.getName(), gen2keys=(
"visit",), consume=
False)
98 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
99 instrument=instrument.getName(), gen2keys=(ccdKey,), consume=
False)
100 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
101 instrument=instrument.getName(), gen2keys=(
"calibDate",), consume=
False)
105 Translator.addRule(CopyKeyHandler(
"exposure",
"visit"),
106 instrument=instrument.getName(), datasetTypeName=
"raw", gen2keys=(
"visit",),
107 consume=(
"visit",
"filter"))
110 Translator.addRule(CopyKeyHandler(
"visit"), instrument=instrument.getName(), gen2keys=(
"visit",),
111 consume=(
"visit",
"filter"))
114 Translator.addRule(CopyKeyHandler(
"detector", ccdKey),
115 instrument=instrument.getName(),
120 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
121 instrument=instrument.getName(), datasetTypeName=
"transmission_optics")
122 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
123 instrument=instrument.getName(), datasetTypeName=
"transmission_atmosphere")
124 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
125 instrument=instrument.getName(), datasetTypeName=
"transmission_filter")
126 Translator.addRule(CopyKeyHandler(
"physical_filter",
"filter"),
127 instrument=instrument.getName(), datasetTypeName=
"transmission_filter")
130 for calibType
in (
'flat',
'sky',
'fringe'):
131 Translator.addRule(CopyKeyHandler(calibFilterType,
"filter"),
132 instrument=instrument.getName(), datasetTypeName=calibType)
135 Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=(
"calibDate",))
138 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
139 skymapName=None, skymapConfig=None,
140 calibs=None, reruns=[], config=None, transferMode="auto"):
141 """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
147 Root path to the gen2 repo to be converted.
149 Root path to the gen3 output repo.
150 instrumentClass : `str`
151 Full python path to the `lsst.obs.base.Instrument` class of the repo
153 calibFilterType : `str`
154 `abstract_filter` or `physical_filter`, depending on the type of
155 ``filter`` in the gen2 calib registry.
156 skymapName : `str`, optional
157 Name of the skymap to be converted in the repo.
158 skymapConfig : `str`, optional
159 Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
161 calibs : `str`, optional
162 Path to the gen2 calibration repository to be converted.
163 If a relative path, it is assumed to be relative to ``gen2root``.
164 reruns : `list` [`str`], optional
165 List of reruns to convert. They will be placed in the
166 ``shared/INSTRUMENT/RERUN`` collection.
167 config : `str`, optional
168 Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
169 after all default/instrument configurations.
170 transferMode : `str`, optional
171 Mode to use when transferring data into the gen3 repository.
176 convertRepoConfig = ConvertRepoTask.ConfigClass()
177 instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
178 convertRepoConfig.instrument = instrumentClass
179 convertRepoConfig.raws.transfer = transferMode
180 if skymapName
is not None:
181 convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
182 convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
183 convertRepoConfig.rootSkyMapName = skymapName
184 if config
is not None:
185 convertRepoConfig.load(config)
189 rerunsArg = [Rerun(rerun, runName=f
"shared/{instrument.getName()}/{rerun}",
190 chainName=f
"shared/{instrument.getName()}", parents=[])
for rerun
in reruns]
194 butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
195 except FileExistsError:
197 butlerConfig = gen3root
198 butler = lsst.daf.butler.Butler(butlerConfig, run=f
"raw/{instrument.getName()}")
199 convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
203 calibs=
None if calibs
is None else {calibs: f
"calib/{instrument.getName()}"}
208 """To be run by the commandline script in `bin/`.
211 args = parser.parse_args()
213 log = lsst.log.Log.getLogger(
"convertRepo")
214 log.setLevel(args.verbose)
216 logger = logging.getLogger(
"convertRepo")
217 logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
218 logger.addHandler(lsst.log.LogHandler())
220 convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
221 skymapName=args.skymapName, skymapConfig=args.skymapConfig,
222 calibs=args.calibs, reruns=args.reruns, config=args.config, transferMode=args.transferMode)