lsst.obs.base  19.0.0-33-g58bbfa5
convertGen2RepoToGen3.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 import argparse
27 import logging
28 
29 import lsst.daf.butler
30 import lsst.log
31 import lsst.utils
32 
33 from ..gen2to3 import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34  Translator, ConstantKeyHandler, CopyKeyHandler,
35  CalibKeyHandler, Rerun)
36 
37 
39  parser = argparse.ArgumentParser(description=__doc__,
40  formatter_class=argparse.RawDescriptionHelpFormatter)
41  parser.add_argument("instrumentClass", metavar="lsst.obs.CAMERA.INSTRUMENT",
42  help=("The full import path to the gen3 Instrument class for this camera"
43  " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44  parser.add_argument("--gen2root", required=True,
45  help="Root path of the gen 2 repo to be converted.")
46  parser.add_argument("--gen3root", required=True,
47  help="Root path of the gen 3 repo to be produced.")
48  parser.add_argument("--skymapName", default=None,
49  help="Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50  parser.add_argument("--skymapConfig", default=None,
51  help="Path to skymap config file defining the new gen3 skymap.")
52  parser.add_argument("--calibs", default=None,
53  help="Path to the gen 2 calibration repo; absolute, or relative to gen2root.")
54  parser.add_argument("--reruns", default=[], nargs="*",
55  help="List of gen 2 reruns to convert.")
56  parser.add_argument("-v", "--verbose", action="store_const", dest="verbose",
57  default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
58  help="Set the log level to DEBUG.")
59  parser.add_argument("--calibFilterType", default="physical_filter",
60  help="physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
61  parser.add_argument("-c", "--config", default=None,
62  help=("Path to a `ConvertRepoConfig` override to be included after "
63  "the Instrument config overrides are applied."))
64 
65  return parser
66 
67 
68 def parse_args(parser):
69  args = parser.parse_args()
70 
71  skymapList = [args.skymapName, args.skymapConfig]
72  if not all(x is None for x in skymapList) and not all(x is not None for x in skymapList):
73  parser.error("Must specify both --skymapName and --skymapConfig, or neither.")
74 
75  return args
76 
77 
78 def configure_translators(instrument, calibFilterType, ccdKey="ccd"):
79  """Configure the gen3 translators so they know the correct instrument name.
80 
81  Parameters
82  ----------
83  instrument : `lsst.obs.base.Instrument`
84  The instrument that conversion is going to be run on.
85  calibFilterType : `str`
86  Whether the gen2 calibRegistry uses ``physical_filter`` or
87  ``abstract_filter`` as the ``filter`` key.
88  ccdKey : `str`, optional
89  The gen2 key used to identify what in gen3 is `detector`.
90  """
91  # Add instrument to Gen3 data ID if Gen2 contains "visit" or ccdKey.
92  # (Both rules will match, so we'll actually set instrument in the same dict twice).
93  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
94  instrument=instrument.getName(), gen2keys=("visit",), consume=False)
95  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
96  instrument=instrument.getName(), gen2keys=(ccdKey,), consume=False)
97  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
98  instrument=instrument.getName(), gen2keys=("calibDate",), consume=False)
99 
100  # Copy Gen2 'visit' to Gen3 'exposure' for raw only. Also consume filter,
101  # since that's implied by 'exposure' in Gen3.
102  Translator.addRule(CopyKeyHandler("exposure", "visit"),
103  instrument=instrument.getName(), datasetTypeName="raw", gen2keys=("visit",),
104  consume=("visit", "filter"))
105 
106  # Copy Gen2 'visit' to Gen3 'visit' otherwise. Also consume filter.
107  Translator.addRule(CopyKeyHandler("visit"), instrument=instrument.getName(), gen2keys=("visit",),
108  consume=("visit", "filter"))
109 
110  # Copy Gen2 'ccd' to Gen3 'detector;
111  Translator.addRule(CopyKeyHandler("detector", ccdKey),
112  instrument=instrument.getName(),
113  gen2keys=(ccdKey,))
114 
115  # Add instrument for transmission curve datasets (transmission_sensor is
116  # already handled by the above translators).
117  Translator.addRule(ConstantKeyHandler("instrument", instrument),
118  instrument=instrument.getName(), datasetTypeName="transmission_optics")
119  Translator.addRule(ConstantKeyHandler("instrument", instrument),
120  instrument=instrument.getName(), datasetTypeName="transmission_atmosphere")
121  Translator.addRule(ConstantKeyHandler("instrument", instrument),
122  instrument=instrument.getName(), datasetTypeName="transmission_filter")
123  Translator.addRule(CopyKeyHandler("physical_filter", "filter"),
124  instrument=instrument.getName(), datasetTypeName="transmission_filter")
125 
126  # Add calibration mapping for filter dependent types
127  for calibType in ('flat', 'sky', 'fringe'):
128  Translator.addRule(CopyKeyHandler(calibFilterType, "filter"),
129  instrument=instrument.getName(), datasetTypeName=calibType)
130 
131  # Translate Gen2 calibDate and datasetType to Gen3 calibration_label.
132  Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=("calibDate",))
133 
134 
135 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
136  skymapName=None, skymapConfig=None,
137  calibs=None, reruns=[], config=None):
138  """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
139  living at gen3root.
140 
141  Parameters
142  ----------
143  gen2root : `str`
144  Root path to the gen2 repo to be converted.
145  gen3root : `str`
146  Root path to the gen3 output repo.
147  instrumentClass : `str`
148  Full python path to the `lsst.obs.base.Instrument` class of the repo
149  being converted.
150  calibFilterType : `str`
151  `abstract_filter` or `physical_filter`, depending on the type of
152  ``filter`` in the gen2 calib registry.
153  skymapName : `str`, optional
154  Name of the skymap to be converted in the repo.
155  skymapConfig : `str`, optional
156  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
157  converted.
158  calibs : `str`, optional
159  Path to the gen2 calibration repository to be converted.
160  If a relative path, it is assumed to be relative to ``gen2root``.
161  reruns : `list` [`str`], optional
162  List of reruns to convert. They will be placed in the
163  ``shared/INSTRUMENT/RERUN`` collection.
164  config : `str`, optional
165  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
166  after all default/instrument configurations.
167  """
168  # instantiate the correct instrument
169  instrument = lsst.utils.doImport(instrumentClass)()
170 
171  convertRepoConfig = ConvertRepoTask.ConfigClass()
172  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
173  if convertRepoConfig.raws.instrument is None:
174  convertRepoConfig.raws.instrument = instrumentClass
175  convertRepoConfig.raws.transfer = "symlink"
176  if skymapName is not None:
177  convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
178  convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
179  convertRepoConfig.rootSkyMapName = skymapName
180  if config is not None:
181  convertRepoConfig.load(config)
182 
183  configure_translators(instrument, calibFilterType, convertRepoConfig.ccdKey)
184 
185  rerunsArg = [Rerun(rerun, runName=f"shared/{instrument.getName()}/{rerun}",
186  chainName=f"shared/{instrument.getName()}", parents=[]) for rerun in reruns]
187 
188  # Allow a gen3 butler to be reused
189  try:
190  butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
191  except FileExistsError:
192  # Use the existing butler configuration
193  butlerConfig = gen3root
194  butler = lsst.daf.butler.Butler(butlerConfig, run=f"raw/{instrument.getName()}")
195  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
196  convertRepoTask.run(
197  root=gen2root,
198  reruns=rerunsArg,
199  calibs=None if calibs is None else {calibs: f"calib/{instrument.getName()}"}
200  )
201 
202 
203 def main():
204  """To be run by the commandline script in `bin/`.
205  """
206  parser = build_argparser()
207  args = parser.parse_args()
208 
209  log = lsst.log.Log.getLogger("convertRepo")
210  log.setLevel(args.verbose)
211  # Forward python logging to lsst logger
212  logger = logging.getLogger("convertRepo")
213  logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
214  logger.addHandler(lsst.log.LogHandler())
215 
216  convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
217  skymapName=args.skymapName, skymapConfig=args.skymapConfig,
218  calibs=args.calibs, reruns=args.reruns, config=args.config)
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, reruns=[], config=None)