lsst.obs.base  19.0.0-51-gb87bce2
convertGen2RepoToGen3.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 import argparse
27 import logging
28 
29 import lsst.daf.butler
30 import lsst.log
31 import lsst.utils
32 
33 from ..gen2to3 import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34  Translator, ConstantKeyHandler, CopyKeyHandler,
35  CalibKeyHandler, Rerun)
36 
37 
39  parser = argparse.ArgumentParser(description=__doc__,
40  formatter_class=argparse.RawDescriptionHelpFormatter)
41  parser.add_argument("instrumentClass", metavar="lsst.obs.CAMERA.INSTRUMENT",
42  help=("The full import path to the gen3 Instrument class for this camera"
43  " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44  parser.add_argument("--gen2root", required=True,
45  help="Root path of the gen 2 repo to be converted.")
46  parser.add_argument("--gen3root", required=True,
47  help="Root path of the gen 3 repo to be produced.")
48  parser.add_argument("--skymapName", default=None,
49  help="Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50  parser.add_argument("--skymapConfig", default=None,
51  help="Path to skymap config file defining the new gen3 skymap.")
52  parser.add_argument("--calibs", default=None,
53  help="Path to the gen 2 calibration repo; absolute, or relative to gen2root.")
54  parser.add_argument("--reruns", default=[], nargs="*",
55  help="List of gen 2 reruns to convert.")
56  parser.add_argument("--transferMode", default="auto",
57  choices=["auto", "link", "symlink", "hardlink", "copy", "move", "relsymlink"],
58  help="Mode to use to transfer files into the new repository.")
59  parser.add_argument("-v", "--verbose", action="store_const", dest="verbose",
60  default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
61  help="Set the log level to DEBUG.")
62  parser.add_argument("--calibFilterType", default="physical_filter",
63  help="physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
64  parser.add_argument("-c", "--config", default=None,
65  help=("Path to a `ConvertRepoConfig` override to be included after "
66  "the Instrument config overrides are applied."))
67 
68  return parser
69 
70 
71 def parse_args(parser):
72  args = parser.parse_args()
73 
74  skymapList = [args.skymapName, args.skymapConfig]
75  if not all(x is None for x in skymapList) and not all(x is not None for x in skymapList):
76  parser.error("Must specify both --skymapName and --skymapConfig, or neither.")
77 
78  return args
79 
80 
81 def configure_translators(instrument, calibFilterType, ccdKey="ccd"):
82  """Configure the gen3 translators so they know the correct instrument name.
83 
84  Parameters
85  ----------
86  instrument : `lsst.obs.base.Instrument`
87  The instrument that conversion is going to be run on.
88  calibFilterType : `str`
89  Whether the gen2 calibRegistry uses ``physical_filter`` or
90  ``abstract_filter`` as the ``filter`` key.
91  ccdKey : `str`, optional
92  The gen2 key used to identify what in gen3 is `detector`.
93  """
94  # Add instrument to Gen3 data ID if Gen2 contains "visit" or ccdKey.
95  # (Both rules will match, so we'll actually set instrument in the same dict twice).
96  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
97  instrument=instrument.getName(), gen2keys=("visit",), consume=False)
98  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
99  instrument=instrument.getName(), gen2keys=(ccdKey,), consume=False)
100  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
101  instrument=instrument.getName(), gen2keys=("calibDate",), consume=False)
102 
103  # Copy Gen2 'visit' to Gen3 'exposure' for raw only. Also consume filter,
104  # since that's implied by 'exposure' in Gen3.
105  Translator.addRule(CopyKeyHandler("exposure", "visit"),
106  instrument=instrument.getName(), datasetTypeName="raw", gen2keys=("visit",),
107  consume=("visit", "filter"))
108 
109  # Copy Gen2 'visit' to Gen3 'visit' otherwise. Also consume filter.
110  Translator.addRule(CopyKeyHandler("visit"), instrument=instrument.getName(), gen2keys=("visit",),
111  consume=("visit", "filter"))
112 
113  # Copy Gen2 'ccd' to Gen3 'detector;
114  Translator.addRule(CopyKeyHandler("detector", ccdKey),
115  instrument=instrument.getName(),
116  gen2keys=(ccdKey,))
117 
118  # Add instrument for transmission curve datasets (transmission_sensor is
119  # already handled by the above translators).
120  Translator.addRule(ConstantKeyHandler("instrument", instrument),
121  instrument=instrument.getName(), datasetTypeName="transmission_optics")
122  Translator.addRule(ConstantKeyHandler("instrument", instrument),
123  instrument=instrument.getName(), datasetTypeName="transmission_atmosphere")
124  Translator.addRule(ConstantKeyHandler("instrument", instrument),
125  instrument=instrument.getName(), datasetTypeName="transmission_filter")
126  Translator.addRule(CopyKeyHandler("physical_filter", "filter"),
127  instrument=instrument.getName(), datasetTypeName="transmission_filter")
128 
129  # Add calibration mapping for filter dependent types
130  for calibType in ('flat', 'sky', 'fringe'):
131  Translator.addRule(CopyKeyHandler(calibFilterType, "filter"),
132  instrument=instrument.getName(), datasetTypeName=calibType)
133 
134  # Translate Gen2 calibDate and datasetType to Gen3 calibration_label.
135  Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=("calibDate",))
136 
137 
138 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
139  skymapName=None, skymapConfig=None,
140  calibs=None, reruns=[], config=None, transferMode="auto"):
141  """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
142  living at gen3root.
143 
144  Parameters
145  ----------
146  gen2root : `str`
147  Root path to the gen2 repo to be converted.
148  gen3root : `str`
149  Root path to the gen3 output repo.
150  instrumentClass : `str`
151  Full python path to the `lsst.obs.base.Instrument` class of the repo
152  being converted.
153  calibFilterType : `str`
154  `abstract_filter` or `physical_filter`, depending on the type of
155  ``filter`` in the gen2 calib registry.
156  skymapName : `str`, optional
157  Name of the skymap to be converted in the repo.
158  skymapConfig : `str`, optional
159  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
160  converted.
161  calibs : `str`, optional
162  Path to the gen2 calibration repository to be converted.
163  If a relative path, it is assumed to be relative to ``gen2root``.
164  reruns : `list` [`str`], optional
165  List of reruns to convert. They will be placed in the
166  ``shared/INSTRUMENT/RERUN`` collection.
167  config : `str`, optional
168  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
169  after all default/instrument configurations.
170  transferMode : `str`, optional
171  Mode to use when transferring data into the gen3 repository.
172  """
173  # instantiate the correct instrument
174  instrument = lsst.utils.doImport(instrumentClass)()
175 
176  convertRepoConfig = ConvertRepoTask.ConfigClass()
177  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
178  convertRepoConfig.instrument = instrumentClass
179  convertRepoConfig.raws.transfer = transferMode
180  if skymapName is not None:
181  convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
182  convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
183  convertRepoConfig.rootSkyMapName = skymapName
184  if config is not None:
185  convertRepoConfig.load(config)
186 
187  configure_translators(instrument, calibFilterType, convertRepoConfig.ccdKey)
188 
189  rerunsArg = [Rerun(rerun, runName=f"shared/{instrument.getName()}/{rerun}",
190  chainName=f"shared/{instrument.getName()}", parents=[]) for rerun in reruns]
191 
192  # Allow a gen3 butler to be reused
193  try:
194  butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
195  except FileExistsError:
196  # Use the existing butler configuration
197  butlerConfig = gen3root
198  butler = lsst.daf.butler.Butler(butlerConfig, run=f"raw/{instrument.getName()}")
199  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
200  convertRepoTask.run(
201  root=gen2root,
202  reruns=rerunsArg,
203  calibs=None if calibs is None else {calibs: f"calib/{instrument.getName()}"}
204  )
205 
206 
207 def main():
208  """To be run by the commandline script in `bin/`.
209  """
210  parser = build_argparser()
211  args = parser.parse_args()
212 
213  log = lsst.log.Log.getLogger("convertRepo")
214  log.setLevel(args.verbose)
215  # Forward python logging to lsst logger
216  logger = logging.getLogger("convertRepo")
217  logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
218  logger.addHandler(lsst.log.LogHandler())
219 
220  convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
221  skymapName=args.skymapName, skymapConfig=args.skymapConfig,
222  calibs=args.calibs, reruns=args.reruns, config=args.config, transferMode=args.transferMode)
convertGen2RepoToGen3.configure_translators
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
Definition: convertGen2RepoToGen3.py:81
convertGen2RepoToGen3.parse_args
def parse_args(parser)
Definition: convertGen2RepoToGen3.py:71
convertGen2RepoToGen3.build_argparser
def build_argparser()
Definition: convertGen2RepoToGen3.py:38
lsst::utils
convertGen2RepoToGen3.main
def main()
Definition: convertGen2RepoToGen3.py:207
convertGen2RepoToGen3.convert
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, reruns=[], config=None, transferMode="auto")
Definition: convertGen2RepoToGen3.py:138
lsst::utils::doImport