22 """Convert a gen2 butler repo to gen 3. See 23 `lsst.obs.base.ConvertRepoConfig` for most of the config options. 29 import lsst.daf.butler
33 from ..gen2to3
import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34 Translator, ConstantKeyHandler, CopyKeyHandler,
35 CalibKeyHandler, Rerun)
39 parser = argparse.ArgumentParser(description=__doc__,
40 formatter_class=argparse.RawDescriptionHelpFormatter)
41 parser.add_argument(
"instrumentClass", metavar=
"lsst.obs.CAMERA.INSTRUMENT",
42 help=(
"The full import path to the gen3 Instrument class for this camera" 43 " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44 parser.add_argument(
"--gen2root", required=
True,
45 help=
"Root path of the gen 2 repo to be converted.")
46 parser.add_argument(
"--gen3root", required=
True,
47 help=
"Root path of the gen 3 repo to be produced.")
48 parser.add_argument(
"--skymapName", default=
None,
49 help=
"Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50 parser.add_argument(
"--skymapConfig", default=
None,
51 help=
"Path to skymap config file defining the new gen3 skymap.")
52 parser.add_argument(
"--calibs", default=
None,
53 help=
"Path to the gen 2 calibration repo; absolute, or relative to gen2root.")
54 parser.add_argument(
"--reruns", default=[], nargs=
"*",
55 help=
"List of gen 2 reruns to convert.")
56 parser.add_argument(
"--transferMode", default=
"auto",
57 choices=[
"auto",
"link",
"symlink",
"hardlink",
"copy",
"move",
"relsymlink"],
58 help=
"Mode to use to transfer files into the new repository.")
59 parser.add_argument(
"-v",
"--verbose", action=
"store_const", dest=
"verbose",
60 default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
61 help=
"Set the log level to DEBUG.")
62 parser.add_argument(
"--calibFilterType", default=
"physical_filter",
63 help=
"physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
64 parser.add_argument(
"-c",
"--config", default=
None,
65 help=(
"Path to a `ConvertRepoConfig` override to be included after " 66 "the Instrument config overrides are applied."))
72 args = parser.parse_args()
74 skymapList = [args.skymapName, args.skymapConfig]
75 if not all(x
is None for x
in skymapList)
and not all(x
is not None for x
in skymapList):
76 parser.error(
"Must specify both --skymapName and --skymapConfig, or neither.")
82 """Configure the gen3 translators so they know the correct instrument name. 86 instrument : `lsst.obs.base.Instrument` 87 The instrument that conversion is going to be run on. 88 calibFilterType : `str` 89 Whether the gen2 calibRegistry uses ``physical_filter`` or 90 ``abstract_filter`` as the ``filter`` key. 91 ccdKey : `str`, optional 92 The gen2 key used to identify what in gen3 is `detector`. 96 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
97 instrument=instrument.getName(), gen2keys=(
"visit",), consume=
False)
98 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
99 instrument=instrument.getName(), gen2keys=(ccdKey,), consume=
False)
100 Translator.addRule(ConstantKeyHandler(
"instrument", instrument.getName()),
101 instrument=instrument.getName(), gen2keys=(
"calibDate",), consume=
False)
105 Translator.addRule(CopyKeyHandler(
"exposure",
"visit"),
106 instrument=instrument.getName(), datasetTypeName=
"raw", gen2keys=(
"visit",),
107 consume=(
"visit",
"filter"))
110 Translator.addRule(CopyKeyHandler(
"visit"), instrument=instrument.getName(), gen2keys=(
"visit",),
111 consume=(
"visit",
"filter"))
114 Translator.addRule(CopyKeyHandler(
"detector", ccdKey),
115 instrument=instrument.getName(),
120 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
121 instrument=instrument.getName(), datasetTypeName=
"transmission_optics")
122 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
123 instrument=instrument.getName(), datasetTypeName=
"transmission_atmosphere")
124 Translator.addRule(ConstantKeyHandler(
"instrument", instrument),
125 instrument=instrument.getName(), datasetTypeName=
"transmission_filter")
126 Translator.addRule(CopyKeyHandler(
"physical_filter",
"filter"),
127 instrument=instrument.getName(), datasetTypeName=
"transmission_filter")
130 for calibType
in (
'flat',
'sky',
'fringe'):
131 Translator.addRule(CopyKeyHandler(calibFilterType,
"filter"),
132 instrument=instrument.getName(), datasetTypeName=calibType)
135 Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=(
"calibDate",))
138 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
139 skymapName=None, skymapConfig=None,
140 calibs=None, reruns=[], config=None, transferMode="auto"):
141 """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo 147 Root path to the gen2 repo to be converted. 149 Root path to the gen3 output repo. 150 instrumentClass : `str` 151 Full python path to the `lsst.obs.base.Instrument` class of the repo 153 calibFilterType : `str` 154 `abstract_filter` or `physical_filter`, depending on the type of 155 ``filter`` in the gen2 calib registry. 156 skymapName : `str`, optional 157 Name of the skymap to be converted in the repo. 158 skymapConfig : `str`, optional 159 Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be 161 calibs : `str`, optional 162 Path to the gen2 calibration repository to be converted. 163 If a relative path, it is assumed to be relative to ``gen2root``. 164 reruns : `list` [`str`], optional 165 List of reruns to convert. They will be placed in the 166 ``shared/INSTRUMENT/RERUN`` collection. 167 config : `str`, optional 168 Path to `lsst.obs.base.ConvertRepoConfig` configuration to load 169 after all default/instrument configurations. 170 transferMode : `str`, optional 171 Mode to use when transferring data into the gen3 repository. 176 convertRepoConfig = ConvertRepoTask.ConfigClass()
177 instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
178 if convertRepoConfig.raws.instrument
is None:
179 convertRepoConfig.raws.instrument = instrumentClass
180 convertRepoConfig.raws.transfer = transferMode
181 if skymapName
is not None:
182 convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
183 convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
184 convertRepoConfig.rootSkyMapName = skymapName
185 if config
is not None:
186 convertRepoConfig.load(config)
190 rerunsArg = [Rerun(rerun, runName=f
"shared/{instrument.getName()}/{rerun}",
191 chainName=f
"shared/{instrument.getName()}", parents=[])
for rerun
in reruns]
195 butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
196 except FileExistsError:
198 butlerConfig = gen3root
199 butler = lsst.daf.butler.Butler(butlerConfig, run=f
"raw/{instrument.getName()}")
200 convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
204 calibs=
None if calibs
is None else {calibs: f
"calib/{instrument.getName()}"}
209 """To be run by the commandline script in `bin/`. 212 args = parser.parse_args()
214 log = lsst.log.Log.getLogger(
"convertRepo")
215 log.setLevel(args.verbose)
217 logger = logging.getLogger(
"convertRepo")
218 logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
219 logger.addHandler(lsst.log.LogHandler())
221 convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
222 skymapName=args.skymapName, skymapConfig=args.skymapConfig,
223 calibs=args.calibs, reruns=args.reruns, config=args.config, transferMode=args.transferMode)
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, reruns=[], config=None, transferMode="auto")