lsst.obs.base  19.0.0-43-gbcf6a3c+1
convertGen2RepoToGen3.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 import argparse
27 import logging
28 
29 import lsst.daf.butler
30 import lsst.log
31 import lsst.utils
32 
33 from ..gen2to3 import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34  Translator, ConstantKeyHandler, CopyKeyHandler,
35  CalibKeyHandler, Rerun)
36 
37 
39  parser = argparse.ArgumentParser(description=__doc__,
40  formatter_class=argparse.RawDescriptionHelpFormatter)
41  parser.add_argument("instrumentClass", metavar="lsst.obs.CAMERA.INSTRUMENT",
42  help=("The full import path to the gen3 Instrument class for this camera"
43  " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44  parser.add_argument("--gen2root", required=True,
45  help="Root path of the gen 2 repo to be converted.")
46  parser.add_argument("--gen3root", required=True,
47  help="Root path of the gen 3 repo to be produced.")
48  parser.add_argument("--skymapName", default=None,
49  help="Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50  parser.add_argument("--skymapConfig", default=None,
51  help="Path to skymap config file defining the new gen3 skymap.")
52  parser.add_argument("--calibs", default=None,
53  help="Path to the gen 2 calibration repo; absolute, or relative to gen2root.")
54  parser.add_argument("--reruns", default=[], nargs="*",
55  help="List of gen 2 reruns to convert.")
56  parser.add_argument("--transferMode", default="auto",
57  choices=["auto", "link", "symlink", "hardlink", "copy", "move", "relsymlink"],
58  help="Mode to use to transfer files into the new repository.")
59  parser.add_argument("-v", "--verbose", action="store_const", dest="verbose",
60  default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
61  help="Set the log level to DEBUG.")
62  parser.add_argument("--calibFilterType", default="physical_filter",
63  help="physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
64  parser.add_argument("-c", "--config", default=None,
65  help=("Path to a `ConvertRepoConfig` override to be included after "
66  "the Instrument config overrides are applied."))
67 
68  return parser
69 
70 
71 def parse_args(parser):
72  args = parser.parse_args()
73 
74  skymapList = [args.skymapName, args.skymapConfig]
75  if not all(x is None for x in skymapList) and not all(x is not None for x in skymapList):
76  parser.error("Must specify both --skymapName and --skymapConfig, or neither.")
77 
78  return args
79 
80 
81 def configure_translators(instrument, calibFilterType, ccdKey="ccd"):
82  """Configure the gen3 translators so they know the correct instrument name.
83 
84  Parameters
85  ----------
86  instrument : `lsst.obs.base.Instrument`
87  The instrument that conversion is going to be run on.
88  calibFilterType : `str`
89  Whether the gen2 calibRegistry uses ``physical_filter`` or
90  ``abstract_filter`` as the ``filter`` key.
91  ccdKey : `str`, optional
92  The gen2 key used to identify what in gen3 is `detector`.
93  """
94  # Add instrument to Gen3 data ID if Gen2 contains "visit" or ccdKey.
95  # (Both rules will match, so we'll actually set instrument in the same dict twice).
96  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
97  instrument=instrument.getName(), gen2keys=("visit",), consume=False)
98  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
99  instrument=instrument.getName(), gen2keys=(ccdKey,), consume=False)
100  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
101  instrument=instrument.getName(), gen2keys=("calibDate",), consume=False)
102 
103  # Copy Gen2 'visit' to Gen3 'exposure' for raw only. Also consume filter,
104  # since that's implied by 'exposure' in Gen3.
105  Translator.addRule(CopyKeyHandler("exposure", "visit"),
106  instrument=instrument.getName(), datasetTypeName="raw", gen2keys=("visit",),
107  consume=("visit", "filter"))
108 
109  # Copy Gen2 'visit' to Gen3 'visit' otherwise. Also consume filter.
110  Translator.addRule(CopyKeyHandler("visit"), instrument=instrument.getName(), gen2keys=("visit",),
111  consume=("visit", "filter"))
112 
113  # Copy Gen2 'ccd' to Gen3 'detector;
114  Translator.addRule(CopyKeyHandler("detector", ccdKey),
115  instrument=instrument.getName(),
116  gen2keys=(ccdKey,))
117 
118  # Add instrument for transmission curve datasets (transmission_sensor is
119  # already handled by the above translators).
120  Translator.addRule(ConstantKeyHandler("instrument", instrument),
121  instrument=instrument.getName(), datasetTypeName="transmission_optics")
122  Translator.addRule(ConstantKeyHandler("instrument", instrument),
123  instrument=instrument.getName(), datasetTypeName="transmission_atmosphere")
124  Translator.addRule(ConstantKeyHandler("instrument", instrument),
125  instrument=instrument.getName(), datasetTypeName="transmission_filter")
126  Translator.addRule(CopyKeyHandler("physical_filter", "filter"),
127  instrument=instrument.getName(), datasetTypeName="transmission_filter")
128 
129  # Add calibration mapping for filter dependent types
130  for calibType in ('flat', 'sky', 'fringe'):
131  Translator.addRule(CopyKeyHandler(calibFilterType, "filter"),
132  instrument=instrument.getName(), datasetTypeName=calibType)
133 
134  # Translate Gen2 calibDate and datasetType to Gen3 calibration_label.
135  Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=("calibDate",))
136 
137 
138 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
139  skymapName=None, skymapConfig=None,
140  calibs=None, reruns=[], config=None, transferMode="auto"):
141  """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
142  living at gen3root.
143 
144  Parameters
145  ----------
146  gen2root : `str`
147  Root path to the gen2 repo to be converted.
148  gen3root : `str`
149  Root path to the gen3 output repo.
150  instrumentClass : `str`
151  Full python path to the `lsst.obs.base.Instrument` class of the repo
152  being converted.
153  calibFilterType : `str`
154  `abstract_filter` or `physical_filter`, depending on the type of
155  ``filter`` in the gen2 calib registry.
156  skymapName : `str`, optional
157  Name of the skymap to be converted in the repo.
158  skymapConfig : `str`, optional
159  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
160  converted.
161  calibs : `str`, optional
162  Path to the gen2 calibration repository to be converted.
163  If a relative path, it is assumed to be relative to ``gen2root``.
164  reruns : `list` [`str`], optional
165  List of reruns to convert. They will be placed in the
166  ``shared/INSTRUMENT/RERUN`` collection.
167  config : `str`, optional
168  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
169  after all default/instrument configurations.
170  transferMode : `str`, optional
171  Mode to use when transferring data into the gen3 repository.
172  """
173  # instantiate the correct instrument
174  instrument = lsst.utils.doImport(instrumentClass)()
175 
176  convertRepoConfig = ConvertRepoTask.ConfigClass()
177  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
178  if convertRepoConfig.raws.instrument is None:
179  convertRepoConfig.raws.instrument = instrumentClass
180  convertRepoConfig.raws.transfer = transferMode
181  if skymapName is not None:
182  convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
183  convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
184  convertRepoConfig.rootSkyMapName = skymapName
185  if config is not None:
186  convertRepoConfig.load(config)
187 
188  configure_translators(instrument, calibFilterType, convertRepoConfig.ccdKey)
189 
190  rerunsArg = [Rerun(rerun, runName=f"shared/{instrument.getName()}/{rerun}",
191  chainName=f"shared/{instrument.getName()}", parents=[]) for rerun in reruns]
192 
193  # Allow a gen3 butler to be reused
194  try:
195  butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
196  except FileExistsError:
197  # Use the existing butler configuration
198  butlerConfig = gen3root
199  butler = lsst.daf.butler.Butler(butlerConfig, run=f"raw/{instrument.getName()}")
200  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
201  convertRepoTask.run(
202  root=gen2root,
203  reruns=rerunsArg,
204  calibs=None if calibs is None else {calibs: f"calib/{instrument.getName()}"}
205  )
206 
207 
208 def main():
209  """To be run by the commandline script in `bin/`.
210  """
211  parser = build_argparser()
212  args = parser.parse_args()
213 
214  log = lsst.log.Log.getLogger("convertRepo")
215  log.setLevel(args.verbose)
216  # Forward python logging to lsst logger
217  logger = logging.getLogger("convertRepo")
218  logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
219  logger.addHandler(lsst.log.LogHandler())
220 
221  convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
222  skymapName=args.skymapName, skymapConfig=args.skymapConfig,
223  calibs=args.calibs, reruns=args.reruns, config=args.config, transferMode=args.transferMode)
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, reruns=[], config=None, transferMode="auto")