Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24"""Interface between `ap_verify` and `ap_pipe`. 

25 

26This module handles calling `ap_pipe` and converting any information 

27as needed. 

28""" 

29 

30__all__ = ["ApPipeParser", "runApPipe"] 

31 

32import argparse 

33import os 

34 

35import lsst.log 

36import lsst.pipe.base as pipeBase 

37import lsst.ap.pipe as apPipe 

38from lsst.ap.pipe.make_apdb import makeApdb 

39 

40 

41class ApPipeParser(argparse.ArgumentParser): 

42 """An argument parser for data needed by ``ap_pipe`` activities. 

43 

44 This parser is not complete, and is designed to be passed to another parser 

45 using the `parent` parameter. 

46 """ 

47 

48 def __init__(self): 

49 # Help and documentation will be handled by main program's parser 

50 argparse.ArgumentParser.__init__(self, add_help=False) 

51 # namespace.dataIds will always be a list of 0 or more nonempty strings, regardless of inputs. 

52 # TODO: in Python 3.8+, action='extend' handles nargs='?' more naturally than 'append'. 

53 self.add_argument('--id', dest='dataIds', action=self.AppendOptional, nargs='?', default=[], 

54 help='An identifier for the data to process.') 

55 self.add_argument("-j", "--processes", default=1, type=int, 

56 help="Number of processes to use.") 

57 self.add_argument("--skip-pipeline", action="store_true", 

58 help="Do not run the AP pipeline itself. This argument is useful " 

59 "for testing metrics on a fixed data set.") 

60 

61 class AppendOptional(argparse.Action): 

62 """A variant of the built-in "append" action that ignores None values 

63 instead of appending them. 

64 """ 

65 # This class can't safely inherit from the built-in "append" action 

66 # because there is no public class that implements it. 

67 def __call__(self, parser, namespace, values, option_string=None): 

68 if values is not None: 

69 try: 

70 allValues = getattr(namespace, self.dest) 

71 allValues.append(values) 

72 except AttributeError: 

73 setattr(namespace, self.dest, [values]) 

74 

75 

76def runApPipe(workspace, parsedCmdLine): 

77 """Run `ap_pipe` on this object's dataset. 

78 

79 Parameters 

80 ---------- 

81 workspace : `lsst.ap.verify.workspace.WorkspaceGen2` 

82 The abstract location containing input and output repositories. 

83 parsedCmdLine : `argparse.Namespace` 

84 Command-line arguments, including all arguments supported by `ApPipeParser`. 

85 

86 Returns 

87 ------- 

88 apPipeReturn : `Struct` 

89 The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

90 ``doReturnResults=False``. This object is valid even if 

91 `~lsst.ap.pipe.ApPipeTask` was never run. 

92 """ 

93 log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipe') 

94 

95 configArgs = _getConfigArguments(workspace) 

96 makeApdb(configArgs) 

97 

98 pipelineArgs = [workspace.dataRepo, 

99 "--output", workspace.outputRepo, 

100 "--calib", workspace.calibRepo, 

101 "--template", workspace.templateRepo] 

102 pipelineArgs.extend(configArgs) 

103 if parsedCmdLine.dataIds: 

104 for singleId in parsedCmdLine.dataIds: 

105 pipelineArgs.extend(["--id", *singleId.split(" ")]) 

106 else: 

107 pipelineArgs.extend(["--id"]) 

108 pipelineArgs.extend(["--processes", str(parsedCmdLine.processes)]) 

109 pipelineArgs.extend(["--noExit"]) 

110 

111 if not parsedCmdLine.skip_pipeline: 

112 results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

113 log.info('Pipeline complete') 

114 else: 

115 log.info('Skipping AP pipeline entirely.') 

116 apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

117 apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

118 results = pipeBase.Struct( 

119 argumentParser=apPipeParser, 

120 parsedCmd=apPipeParsed, 

121 taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

122 resultList=[], 

123 ) 

124 

125 return results 

126 

127 

128def _getConfigArguments(workspace): 

129 """Return the config options for running ApPipeTask on this workspace, as 

130 command-line arguments. 

131 

132 Parameters 

133 ---------- 

134 workspace : `lsst.ap.verify.workspace.Workspace` 

135 A Workspace whose config directory may contain an 

136 `~lsst.ap.pipe.ApPipeTask` config. 

137 

138 Returns 

139 ------- 

140 args : `list` of `str` 

141 Command-line arguments calling ``--config`` or ``--configFile``, 

142 following the conventions of `sys.argv`. 

143 """ 

144 overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

145 overridePath = os.path.join(workspace.configDir, overrideFile) 

146 

147 args = ["--configfile", overridePath] 

148 # ApVerify will use the sqlite hooks for the Apdb. 

149 args.extend(["--config", "diaPipe.apdb.db_url=sqlite:///" + workspace.dbLocation]) 

150 args.extend(["--config", "diaPipe.apdb.isolation_level=READ_UNCOMMITTED"]) 

151 # Put output alerts into the workspace. 

152 args.extend(["--config", "diaPipe.alertPackager.alertWriteLocation=" + workspace.workDir + "/alerts"]) 

153 args.extend(["--config", "diaPipe.doPackageAlerts=True"]) 

154 

155 return args