lsst.obs.base  20.0.0-68-ga3f3dda+0059e82a5d
convert.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 from lsst.daf.persistence import Butler as Butler2
27 import lsst.daf.butler
28 from lsst.log import Log
29 import lsst.utils
30 from lsst.log.utils import temporaryLogLevel
31 
32 from ..gen2to3 import ConvertRepoTask, ConvertRepoSkyMapConfig, Rerun
33 
34 
35 def convert(repo, gen2root, skymap_name, skymap_config, calibs, reruns, config_file, transfer, processes=1):
36  """Implements the command line interface `butler convert` subcommand,
37  should only be called by command line tools and unit test code that tests
38  this function.
39 
40  Convert the gen 2 Butler repo at `gen2root` into a gen 3 repo
41  living at `repo`.
42 
43  Parameters
44  ----------
45  repo : `str`
46  URI to the gen 3 repository.
47  gen2root : `str`
48  URI to the gen 2 repository.
49  skymap_name : `str` or None
50  Name of the skymap to be converted in the repo.
51  skymap_config : `str` or None
52  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
53  converted.
54  calibs : `str` or None
55  Path to the gen2 calibration repository to be converted.
56  If a relative path, it is assumed to be relative to `gen2root`.
57  reruns : `list` [`str`] or None
58  List of reruns to convert. They will be placed in the
59  ``shared/INSTRUMENT/RERUN`` collection.
60  config_file : `str` or None
61  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
62  after all default/instrument configurations.
63  transfer : `str` or None
64  Mode to use when transferring data into the gen3 repository.
65  processess : `int`
66  Number of processes to use for conversion.
67  """
68  # Allow a gen3 butler to be reused
69  try:
70  butlerConfig = lsst.daf.butler.Butler.makeRepo(repo)
71  except FileExistsError:
72  # Use the existing butler configuration
73  butlerConfig = repo
74 
75  butler = lsst.daf.butler.Butler(butlerConfig)
76 
77  # Derive the gen3 instrument from the gen2root
78  # This requires we instantiate a gen2 butler solely to get its mapper
79  # Hide all logging -- the later call will show them
80  with temporaryLogLevel("", Log.ERROR):
81  butler2 = Butler2(gen2root)
82  gen2mapperClass = butler2.getMapperClass(gen2root)
83  del butler2
84 
85  instrument = gen2mapperClass.getGen3Instrument()()
86 
87  convertRepoConfig = ConvertRepoTask.ConfigClass()
88  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
89  convertRepoConfig.raws.transfer = transfer
90  if skymap_name is not None:
91  convertRepoConfig.skyMaps[skymap_name] = ConvertRepoSkyMapConfig()
92  convertRepoConfig.skyMaps[skymap_name].load(skymap_config)
93  convertRepoConfig.rootSkyMapName = skymap_name
94  if config_file is not None:
95  convertRepoConfig.load(config_file)
96 
97  if reruns is None:
98  rerunsArg = []
99  else:
100  rerunsArg = [Rerun(rerun, runName=f"shared/{instrument.getName()}/{rerun}",
101  chainName=f"shared/{instrument.getName()}", parents=[]) for rerun in reruns]
102 
103  # create a new butler instance for running the convert repo task
104  butler = lsst.daf.butler.Butler(butlerConfig, run=instrument.makeDefaultRawIngestRunName())
105  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler, instrument=instrument)
106  convertRepoTask.run(
107  root=gen2root,
108  reruns=rerunsArg,
109  calibs=None if calibs is None else {calibs: instrument.makeCollectionName("calib")},
110  processes=processes,
111  )
lsst.obs.base.gen2to3.convertRepo.ConvertRepoSkyMapConfig
Definition: convertRepo.py:103
lsst::utils
lsst.obs.base.gen2to3.convertRepo.Rerun
Definition: convertRepo.py:74
lsst.obs.base.gen2to3.convertRepo.ConvertRepoTask
Definition: convertRepo.py:279
lsst.obs.base.script.convert.convert
def convert(repo, gen2root, skymap_name, skymap_config, calibs, reruns, config_file, transfer, processes=1)
Definition: convert.py:35