Coverage for python/lsst/obs/base/script/convert.py: 19%

41 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-05 02:40 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Convert a gen2 butler repo to gen 3. See 

23`lsst.obs.base.ConvertRepoConfig` for most of the config options. 

24""" 

25 

26import logging 

27 

28import lsst.daf.butler 

29import lsst.utils 

30from lsst.daf.persistence import Butler as Butler2 

31from lsst.log import Log 

32from lsst.log.utils import temporaryLogLevel 

33 

34from ..gen2to3 import CalibRepo, ConvertRepoSkyMapConfig, ConvertRepoTask, Rerun 

35 

36log = logging.getLogger(__name__) 

37 

38 

39def convert(repo, gen2root, skymap_name, skymap_config, calibs, reruns, config_file, transfer, processes=1): 

40 """Implements the command line interface `butler convert` subcommand, 

41 should only be called by command line tools and unit test code that tests 

42 this function. 

43 

44 Convert the gen 2 Butler repo at `gen2root` into a gen 3 repo 

45 living at `repo`. 

46 

47 Parameters 

48 ---------- 

49 repo : `str` 

50 URI to the gen 3 repository. 

51 gen2root : `str` 

52 URI to the gen 2 repository. 

53 skymap_name : `str` or None 

54 Name of the skymap to be converted in the repo. 

55 skymap_config : `str` or None 

56 Path to the `lsst.skymap.BaseSkyMapConfig` of the gen2 skymap to be 

57 converted. 

58 calibs : `str` or None 

59 Path to the gen2 calibration repository to be converted. 

60 If a relative path, it is assumed to be relative to `gen2root`. 

61 reruns : `list` [`str`] or None 

62 List of rerun paths to convert. Output collection names will be 

63 guessed, which can fail if the Gen2 repository paths do not follow a 

64 recognized convention. In this case, the command-line interface cannot 

65 be used. 

66 config_file : `str` or None 

67 Path to `lsst.obs.base.ConvertRepoConfig` configuration to load 

68 after all default/instrument configurations. 

69 transfer : `str` or None 

70 Mode to use when transferring data into the gen3 repository. 

71 processess : `int` 

72 Number of processes to use for conversion. 

73 """ 

74 # Allow a gen3 butler to be reused 

75 try: 

76 butlerConfig = lsst.daf.butler.Butler.makeRepo(repo) 

77 except FileExistsError: 

78 # Use the existing butler configuration 

79 butlerConfig = repo 

80 

81 butler = lsst.daf.butler.Butler(butlerConfig) 

82 

83 # Derive the gen3 instrument from the gen2root 

84 # This requires we instantiate a gen2 butler solely to get its mapper 

85 # Hide all logging -- the later call will show them 

86 with temporaryLogLevel("", Log.ERROR): 

87 butler2 = Butler2(gen2root) 

88 gen2mapperClass = butler2.getMapperClass(gen2root) 

89 del butler2 

90 

91 instrument = gen2mapperClass.getGen3Instrument()() 

92 

93 convertRepoConfig = ConvertRepoTask.ConfigClass() 

94 instrument.applyConfigOverrides(ConvertRepoTask._DefaultName, convertRepoConfig) 

95 convertRepoConfig.raws.transfer = transfer 

96 if skymap_name is not None: 

97 convertRepoConfig.skyMaps[skymap_name] = ConvertRepoSkyMapConfig() 

98 convertRepoConfig.skyMaps[skymap_name].load(skymap_config) 

99 convertRepoConfig.rootSkyMapName = skymap_name 

100 if config_file is not None: 

101 convertRepoConfig.load(config_file) 

102 

103 if reruns is None: 

104 rerunsArg = [] 

105 else: 

106 rerunsArg = [Rerun(rerun, runName=None, chainName=None, parents=[]) for rerun in reruns] 

107 

108 # If this is old schema but is using modern visit grouping algorithm, 

109 # (which is the default for new code) revert to one-to-one (which 

110 # was the old default). 

111 exposure_dimension = butler.registry.dimensions["exposure"] 

112 modern = "one-to-one-and-by-counter" 

113 if ( 

114 "seq_end" not in exposure_dimension.metadata 

115 and convertRepoConfig.defineVisits.groupExposures.name == modern 

116 ): 

117 legacy = "one-to-one" 

118 log.warning( 

119 "Request to use %s grouping algorithm but registry schema is too old. Using %s instead.", 

120 modern, 

121 legacy, 

122 ) 

123 convertRepoConfig.defineVisits.groupExposures.name = legacy 

124 

125 # create a new butler instance for running the convert repo task 

126 butler = lsst.daf.butler.Butler(butlerConfig, run=instrument.makeDefaultRawIngestRunName()) 

127 convertRepoTask = ConvertRepoTask(config=convertRepoConfig, butler3=butler, instrument=instrument) 

128 convertRepoTask.run( 

129 root=gen2root, 

130 reruns=rerunsArg, 

131 calibs=None if calibs is None else [CalibRepo(path=calibs)], 

132 processes=processes, 

133 )