Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

# 

# This file is part of ap_verify. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

# 

 

"""Interface between `ap_verify` and `ap_pipe`. 

 

This module handles calling `ap_pipe` and converting any information 

as needed. 

""" 

 

__all__ = ["ApPipeParser", "runApPipe"] 

 

import argparse 

import os 

 

import lsst.log 

import lsst.pipe.base as pipeBase 

import lsst.ap.pipe as apPipe 

from lsst.ap.pipe.make_ppdb import makePpdb 

 

 

class ApPipeParser(argparse.ArgumentParser): 

"""An argument parser for data needed by ``ap_pipe`` activities. 

 

This parser is not complete, and is designed to be passed to another parser 

using the `parent` parameter. 

""" 

 

def __init__(self): 

# Help and documentation will be handled by main program's parser 

argparse.ArgumentParser.__init__(self, add_help=False) 

self.add_argument('--id', dest='dataIds', action='append', default=[], 

help='An identifier for the data to process.') 

self.add_argument("-j", "--processes", default=1, type=int, 

help="Number of processes to use.") 

self.add_argument("--skip-pipeline", action="store_true", 

help="Do not run the AP pipeline itself. This argument is useful " 

"for testing metrics on a fixed data set.") 

 

 

def runApPipe(workspace, parsedCmdLine): 

"""Run `ap_pipe` on this object's dataset. 

 

Parameters 

---------- 

workspace : `lsst.ap.verify.workspace.Workspace` 

The abstract location containing input and output repositories. 

parsedCmdLine : `argparse.Namespace` 

Command-line arguments, including all arguments supported by `ApPipeParser`. 

 

Returns 

------- 

apPipeReturn : `Struct` 

The `Struct` returned from `~lsst.ap.pipe.ApPipeTask.parseAndRun` with 

``doReturnResults=False``. This object is valid even if 

`~lsst.ap.pipe.ApPipeTask` was never run. 

""" 

log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipe') 

 

configArgs = _getConfigArguments(workspace) 

makePpdb(configArgs) 

 

pipelineArgs = [workspace.dataRepo, 

"--output", workspace.outputRepo, 

"--calib", workspace.calibRepo, 

"--template", workspace.templateRepo] 

pipelineArgs.extend(configArgs) 

if parsedCmdLine.dataIds: 

for singleId in parsedCmdLine.dataIds: 

pipelineArgs.extend(["--id", *singleId.split(" ")]) 

else: 

pipelineArgs.extend(["--id"]) 

pipelineArgs.extend(["--processes", str(parsedCmdLine.processes)]) 

pipelineArgs.extend(["--noExit"]) 

 

if not parsedCmdLine.skip_pipeline: 

results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

log.info('Pipeline complete') 

else: 

log.info('Skipping AP pipeline entirely.') 

apPipeParser = apPipe.ApPipeTask._makeArgumentParser() 

apPipeParsed = apPipeParser.parse_args(config=apPipe.ApPipeTask.ConfigClass(), args=pipelineArgs) 

results = pipeBase.Struct( 

argumentParser=apPipeParser, 

parsedCmd=apPipeParsed, 

taskRunner=apPipe.ApPipeTask.RunnerClass(TaskClass=apPipe.ApPipeTask, parsedCmd=apPipeParsed), 

resultList=[], 

) 

 

return results 

 

 

def _getConfigArguments(workspace): 

"""Return the config options for running ApPipeTask on this workspace, as 

command-line arguments. 

 

Parameters 

---------- 

workspace : `lsst.ap.verify.workspace.Workspace` 

A Workspace whose config directory may contain an 

`~lsst.ap.pipe.ApPipeTask` config. 

 

Returns 

------- 

args : `list` of `str` 

Command-line arguments calling ``--config`` or ``--configFile``, 

following the conventions of `sys.argv`. 

""" 

overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

overridePath = os.path.join(workspace.configDir, overrideFile) 

 

args = ["--configfile", overridePath] 

# ApVerify will use the sqlite hooks for the Ppdb. 

args.extend(["--config", "ppdb.db_url=sqlite:///" + workspace.dbLocation]) 

args.extend(["--config", "ppdb.isolation_level=READ_UNCOMMITTED"]) 

 

return args