lsst.obs.base  19.0.0-32-g3b6bf2d
convertGen2RepoToGen3.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 import argparse
27 import logging
28 
29 import lsst.daf.butler
30 import lsst.log
31 import lsst.utils
32 
33 from ..gen2to3 import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34  Translator, ConstantKeyHandler, CopyKeyHandler,
35  CalibKeyHandler)
36 
37 
39  parser = argparse.ArgumentParser(description=__doc__,
40  formatter_class=argparse.RawDescriptionHelpFormatter)
41  parser.add_argument("instrumentClass", metavar="lsst.obs.CAMERA.INSTRUMENT",
42  help=("The full import path to the gen3 Instrument class for this camera"
43  " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44  parser.add_argument("--gen2root", required=True,
45  help="Root path of the gen 2 repo to be converted.")
46  parser.add_argument("--gen3root", required=True,
47  help="Root path of the gen 3 repo to be produced.")
48  parser.add_argument("--skymapName", default=None,
49  help="Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50  parser.add_argument("--skymapConfig", default=None,
51  help="Path to skymap config file defining the new gen3 skymap.")
52  parser.add_argument("--calibs", default=None,
53  help="Path to calibration repo; absolute, or relative to gen2root.")
54  parser.add_argument("-v", "--verbose", action="store_const", dest="verbose",
55  default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
56  help="Set the log level to DEBUG.")
57  parser.add_argument("--calibFilterType", default="physical_filter",
58  help="physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
59  parser.add_argument("-c", "--config", default=None,
60  help=("Path to a `ConvertRepoConfig` override to be included after "
61  "the Instrument config overrides are applied."))
62 
63  return parser
64 
65 
66 def parse_args(parser):
67  args = parser.parse_args()
68 
69  skymapList = [args.skymapName, args.skymapConfig]
70  if not all(x is None for x in skymapList) and not all(x is not None for x in skymapList):
71  parser.error("Must specify both --skymapName and --skymapConfig, or neither.")
72 
73  return args
74 
75 
76 def configure_translators(instrument, calibFilterType, ccdKey="ccd"):
77  """Configure the gen3 translators so they know the correct instrument name.
78 
79  Parameters
80  ----------
81  instrument : `lsst.obs.base.Instrument`
82  The instrument that conversion is going to be run on.
83  calibFilterType : `str`
84  Whether the gen2 calibRegistry uses ``physical_filter`` or
85  ``abstract_filter`` as the ``filter`` key.
86  ccdKey : `str`, optional
87  The gen2 key used to identify what in gen3 is `detector`.
88  """
89  # Add instrument to Gen3 data ID if Gen2 contains "visit" or ccdKey.
90  # (Both rules will match, so we'll actually set instrument in the same dict twice).
91  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
92  instrument=instrument.getName(), gen2keys=("visit",), consume=False)
93  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
94  instrument=instrument.getName(), gen2keys=(ccdKey,), consume=False)
95  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
96  instrument=instrument.getName(), gen2keys=("calibDate",), consume=False)
97 
98  # Copy Gen2 'visit' to Gen3 'exposure' for raw only. Also consume filter,
99  # since that's implied by 'exposure' in Gen3.
100  Translator.addRule(CopyKeyHandler("exposure", "visit"),
101  instrument=instrument.getName(), datasetTypeName="raw", gen2keys=("visit",),
102  consume=("visit", "filter"))
103 
104  # Copy Gen2 'visit' to Gen3 'visit' otherwise. Also consume filter.
105  Translator.addRule(CopyKeyHandler("visit"), instrument=instrument.getName(), gen2keys=("visit",),
106  consume=("visit", "filter"))
107 
108  # Copy Gen2 'ccd' to Gen3 'detector;
109  Translator.addRule(CopyKeyHandler("detector", ccdKey),
110  instrument=instrument.getName(),
111  gen2keys=(ccdKey,))
112 
113  # Add instrument for transmission curve datasets (transmission_sensor is
114  # already handled by the above translators).
115  Translator.addRule(ConstantKeyHandler("instrument", instrument),
116  instrument=instrument.getName(), datasetTypeName="transmission_optics")
117  Translator.addRule(ConstantKeyHandler("instrument", instrument),
118  instrument=instrument.getName(), datasetTypeName="transmission_atmosphere")
119  Translator.addRule(ConstantKeyHandler("instrument", instrument),
120  instrument=instrument.getName(), datasetTypeName="transmission_filter")
121  Translator.addRule(CopyKeyHandler("physical_filter", "filter"),
122  instrument=instrument.getName(), datasetTypeName="transmission_filter")
123 
124  # Add calibration mapping for filter dependent types
125  for calibType in ('flat', 'sky', 'fringe'):
126  Translator.addRule(CopyKeyHandler(calibFilterType, "filter"),
127  instrument=instrument.getName(), datasetTypeName=calibType)
128 
129  # Translate Gen2 calibDate and datasetType to Gen3 calibration_label.
130  Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=("calibDate",))
131 
132 
133 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
134  skymapName=None, skymapConfig=None,
135  calibs=None, config=None):
136  """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
137  living at gen3root.
138 
139  Parameters
140  ----------
141  gen2root : `str`
142  Root path to the gen2 repo to be converted.
143  gen3root : `str`
144  Root path to the gen3 output repo.
145  instrumentClass : `str`
146  Full python path to the `lsst.obs.base.Instrument` class of the repo
147  being converted.
148  calibFilterType : `str`
149  `abstract_filter` or `physical_filter`, depending on the type of
150  ``filter`` in the gen2 calib registry.
151  skymapName : `str`, optional
152  Name of the skymap to be converted in the repo.
153  skymapConfig : `str`, optional
154  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
155  converted.
156  calibs : `str`, optional
157  Path to the gen2 calibration repository to be converted.
158  If a relative path, it is assumed to be relative to ``gen2root``.
159  config : `str`, optional
160  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
161  after all default/instrument configurations.
162  """
163  # instantiate the correct instrument
164  instrument = lsst.utils.doImport(instrumentClass)()
165 
166  convertRepoConfig = ConvertRepoTask.ConfigClass()
167  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
168  if convertRepoConfig.raws.instrument is None:
169  convertRepoConfig.raws.instrument = instrumentClass
170  convertRepoConfig.raws.transfer = "symlink"
171  if skymapName is not None:
172  convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
173  convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
174  if config is not None:
175  convertRepoConfig.load(config)
176 
177  configure_translators(instrument, calibFilterType, convertRepoConfig.ccdKey)
178 
179  # Allow a gen3 butler to be reused
180  try:
181  butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
182  except FileExistsError:
183  # Use the existing butler configuration
184  butlerConfig = gen3root
185  butler = lsst.daf.butler.Butler(butlerConfig, run=f"raw/{instrument.getName()}")
186  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
187  convertRepoTask.run(
188  root=gen2root,
189  reruns=[],
190  calibs=None if calibs is None else {calibs: f"calib/{instrument.getName()}"}
191  )
192 
193 
194 def main():
195  """To be run by the commandline script in `bin/`.
196  """
197  parser = build_argparser()
198  args = parser.parse_args()
199 
200  log = lsst.log.Log.getLogger("convertRepo")
201  log.setLevel(args.verbose)
202  # Forward python logging to lsst logger
203  logger = logging.getLogger("convertRepo")
204  logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
205  logger.addHandler(lsst.log.LogHandler())
206 
207  convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
208  skymapName=args.skymapName, skymapConfig=args.skymapConfig,
209  calibs=args.calibs, config=args.config)
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, config=None)