lsst.obs.base  19.0.0-27-gde16c09
convertGen2RepoToGen3.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Convert a gen2 butler repo to gen 3. See
23 `lsst.obs.base.ConvertRepoConfig` for most of the config options.
24 """
25 
26 import argparse
27 import logging
28 
29 import lsst.daf.butler
30 import lsst.log
31 import lsst.utils
32 
33 from lsst.obs.base.gen2to3 import (ConvertRepoTask, ConvertRepoSkyMapConfig,
34  Translator, ConstantKeyHandler, CopyKeyHandler,
35  CalibKeyHandler)
36 
37 
39  parser = argparse.ArgumentParser(description=__doc__,
40  formatter_class=argparse.RawDescriptionHelpFormatter)
41  parser.add_argument("instrumentClass", metavar="lsst.obs.CAMERA.INSTRUMENT",
42  help=("The full import path to the gen3 Instrument class for this camera"
43  " (e.g. lsst.obs.decam.DarkEnergyCamera)."))
44  parser.add_argument("--gen2root", required=True,
45  help="Root path of the gen 2 repo to be converted.")
46  parser.add_argument("--gen3root", required=True,
47  help="Root path of the gen 3 repo to be produced.")
48  parser.add_argument("--skymapName", default=None,
49  help="Name of the new gen3 skymap (e.g. 'discrete/ci_hsc').")
50  parser.add_argument("--skymapConfig", default=None,
51  help="Path to skymap config file defining the new gen3 skymap.")
52  parser.add_argument("--calibs", default=None,
53  help="Path to calibration repo; absolute, or relative to gen2root.")
54  parser.add_argument("-v", "--verbose", action="store_const", dest="verbose",
55  default=lsst.log.Log.INFO, const=lsst.log.Log.DEBUG,
56  help="Set the log level to DEBUG.")
57  parser.add_argument("--calibFilterType", default="physical_filter",
58  help="physical_filter or abstract_filter as the id in the gen2 calibRegistry.")
59  parser.add_argument("-c", "--config", default=None,
60  help=("Path to a `ConvertRepoConfig` override to be included after "
61  "the Instrument config overrides are applied."))
62 
63  return parser
64 
65 
66 def parse_args(parser):
67  args = parser.parse_args()
68 
69  skymapList = [args.skymapName, args.skymapConfig]
70  if not all(x is None for x in skymapList) and not all(x is not None for x in skymapList):
71  parser.error("Must specify both --skymapName and --skymapConfig, or neither.")
72 
73  return args
74 
75 
76 def configure_translators(instrument, calibFilterType, ccdKey="ccd"):
77  """Configure the gen3 translators so they know the correct instrument name.
78 
79  Parameters
80  ----------
81  instrument : `lsst.obs.base.Instrument`
82  The instrument that conversion is going to be run on.
83  calibFilterType : `str`
84  Whether the gen2 calibRegistry uses ``physical_filter`` or
85  ``abstract_filter`` as the ``filter`` key.
86  ccdKey : `str`, optional
87  The gen2 key used to identify what in gen3 is `detector`.
88  """
89  # Add instrument to Gen3 data ID if Gen2 contains "visit" or ccdKey.
90  # (Both rules will match, so we'll actually set instrument in the same dict twice).
91  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
92  instrument=instrument.getName(), gen2keys=("visit",), consume=False)
93  Translator.addRule(ConstantKeyHandler("instrument", instrument.getName()),
94  instrument=instrument.getName(), gen2keys=(ccdKey,), consume=False)
95 
96  # Copy Gen2 'visit' to Gen3 'exposure' for raw only. Also consume filter,
97  # since that's implied by 'exposure' in Gen3.
98  Translator.addRule(CopyKeyHandler("exposure", "visit"),
99  instrument=instrument.getName(), datasetTypeName="raw", gen2keys=("visit",),
100  consume=("visit", "filter"))
101 
102  # Copy Gen2 'visit' to Gen3 'visit' otherwise. Also consume filter.
103  Translator.addRule(CopyKeyHandler("visit"), instrument=instrument.getName(), gen2keys=("visit",),
104  consume=("visit", "filter"))
105 
106  # Copy Gen2 'ccd' to Gen3 'detector;
107  Translator.addRule(CopyKeyHandler("detector", ccdKey),
108  instrument=instrument.getName(),
109  gen2keys=(ccdKey,))
110 
111  # Add instrument for transmission curve datasets (transmission_sensor is
112  # already handled by the above translators).
113  Translator.addRule(ConstantKeyHandler("instrument", instrument),
114  instrument=instrument.getName(), datasetTypeName="transmission_optics")
115  Translator.addRule(ConstantKeyHandler("instrument", instrument),
116  instrument=instrument.getName(), datasetTypeName="transmission_atmosphere")
117  Translator.addRule(ConstantKeyHandler("instrument", instrument),
118  instrument=instrument.getName(), datasetTypeName="transmission_filter")
119  Translator.addRule(CopyKeyHandler("physical_filter", "filter"),
120  instrument=instrument.getName(), datasetTypeName="transmission_filter")
121 
122  # Add calibration mapping for filter dependent types
123  for calibType in ('flat', 'sky', 'fringe'):
124  Translator.addRule(CopyKeyHandler(calibFilterType, "filter"),
125  instrument=instrument.getName(), datasetTypeName=calibType)
126 
127  # Translate Gen2 calibDate and datasetType to Gen3 calibration_label.
128  Translator.addRule(CalibKeyHandler(ccdKey), gen2keys=("calibDate",))
129 
130 
131 def convert(gen2root, gen3root, instrumentClass, calibFilterType,
132  skymapName=None, skymapConfig=None,
133  calibs=None, config=None):
134  """Convert the gen 2 Butler repo living at gen2root into a gen 3 repo
135  living at gen3root.
136 
137  Parameters
138  ----------
139  gen2root : `str`
140  Root path to the gen2 repo to be converted.
141  gen3root : `str`
142  Root path to the gen3 output repo.
143  instrumentClass : `str`
144  Full python path to the `lsst.obs.base.Instrument` class of the repo
145  being converted.
146  calibFilterType : `str`
147  `abstract_filter` or `physical_filter`, depending on the type of
148  ``filter`` in the gen2 calib registry.
149  skymapName : `str`, optional
150  Name of the skymap to be converted in the repo.
151  skymapConfig : `str`, optional
152  Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be
153  converted.
154  calibs : `str`, optional
155  Path to the gen2 calibration repository to be converted.
156  If a relative path, it is assumed to be relative to ``gen2root``.
157  config : `str`, optional
158  Path to `lsst.obs.base.ConvertRepoConfig` configuration to load
159  after all default/instrument configurations.
160  """
161  # instantiate the correct instrument
162  instrument = lsst.utils.doImport(instrumentClass)()
163 
164  convertRepoConfig = ConvertRepoTask.ConfigClass()
165  instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig)
166  if convertRepoConfig.raws.instrument is None:
167  convertRepoConfig.raws.instrument = instrumentClass
168  convertRepoConfig.raws.transfer = "symlink"
169  if skymapName is not None:
170  convertRepoConfig.skyMaps[skymapName] = ConvertRepoSkyMapConfig()
171  convertRepoConfig.skyMaps[skymapName].load(skymapConfig)
172  if config is not None:
173  convertRepoConfig.load(config)
174 
175  configure_translators(instrument, calibFilterType, convertRepoConfig.ccdKey)
176 
177  butlerConfig = lsst.daf.butler.Butler.makeRepo(gen3root)
178  butler = lsst.daf.butler.Butler(butlerConfig, run=instrument.getName())
179  convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler)
180  convertRepoTask.run(
181  root=gen2root,
182  # NOTE: we'd like to use `raw/NAME` and `calib/NAME` here, but if I do, I get an error about
183  # `AssertionError: Multiple collections for curated calibrations is not yet supported.`
184  # TODO: This is likely related to DM-23230.
185  # since we also want to specify just `NAME` so that we can instantiate a gen3 Butler
186  # using that as an "umbrella" collection so it knows about everything in it.
187  collections=[instrument.getName()], # ['raw/'+instrument.getName(), instrument.getName()],
188  calibs=None if calibs is None else {calibs: [instrument.getName()]}
189  )
190 
191 
192 def main():
193  """To be run by the commandline script in `bin/`.
194  """
195  parser = build_argparser()
196  args = parser.parse_args()
197 
198  log = lsst.log.Log.getLogger("convertRepo")
199  log.setLevel(args.verbose)
200  # Forward python logging to lsst logger
201  logger = logging.getLogger("convertRepo")
202  logger.setLevel(lsst.log.LevelTranslator.lsstLog2logging(log.getLevel()))
203  logger.addHandler(lsst.log.LogHandler())
204 
205  convert(args.gen2root, args.gen3root, args.instrumentClass, args.calibFilterType,
206  skymapName=args.skymapName, skymapConfig=args.skymapConfig,
207  calibs=args.calibs, config=args.config)
def configure_translators(instrument, calibFilterType, ccdKey="ccd")
def convert(gen2root, gen3root, instrumentClass, calibFilterType, skymapName=None, skymapConfig=None, calibs=None, config=None)