Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

# 

# This file is part of ap_verify. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

# 

 

"""Interface between `ap_verify` and `ap_pipe`. 

 

This module handles calling `ap_pipe` and converting any information 

as needed. 

""" 

 

__all__ = ["ApPipeParser", "runApPipe"] 

 

import argparse 

import os 

 

import lsst.log 

import lsst.ap.pipe as apPipe 

from lsst.ap.pipe.make_ppdb import makePpdb 

 

 

class ApPipeParser(argparse.ArgumentParser): 

"""An argument parser for data needed by ``ap_pipe`` activities. 

 

This parser is not complete, and is designed to be passed to another parser 

using the `parent` parameter. 

""" 

 

def __init__(self): 

# Help and documentation will be handled by main program's parser 

argparse.ArgumentParser.__init__(self, add_help=False) 

self.add_argument('--id', dest='dataIds', action='append', default=[], 

help='An identifier for the data to process.') 

self.add_argument("-j", "--processes", default=1, type=int, 

help="Number of processes to use.") 

 

 

def runApPipe(workspace, parsedCmdLine): 

"""Run `ap_pipe` on this object's dataset. 

 

Parameters 

---------- 

workspace : `lsst.ap.verify.workspace.Workspace` 

The abstract location containing input and output repositories. 

parsedCmdLine : `argparse.Namespace` 

Command-line arguments, including all arguments supported by `ApPipeParser`. 

 

Returns 

------- 

dataIds : `lsst.pipe.base.DataIdContainer` 

The set of complete data IDs fed into ``ap_pipe``. 

""" 

log = lsst.log.Log.getLogger('ap.verify.pipeline_driver.runApPipe') 

 

configArgs = _getConfigArguments(workspace) 

makePpdb(configArgs) 

 

pipelineArgs = [workspace.dataRepo, 

"--output", workspace.outputRepo, 

"--calib", workspace.calibRepo, 

"--template", workspace.templateRepo] 

pipelineArgs.extend(configArgs) 

if parsedCmdLine.dataIds: 

for singleId in parsedCmdLine.dataIds: 

pipelineArgs.extend(["--id", *singleId.split(" ")]) 

else: 

pipelineArgs.extend(["--id"]) 

pipelineArgs.extend(["--processes", str(parsedCmdLine.processes)]) 

pipelineArgs.extend(["--noExit"]) 

 

results = apPipe.ApPipeTask.parseAndRun(pipelineArgs) 

log.info('Pipeline complete') 

 

return results.parsedCmd.id 

 

 

def _getConfigArguments(workspace): 

"""Return the config options for running ApPipeTask on this workspace, as 

command-line arguments. 

 

Parameters 

---------- 

workspace : `lsst.ap.verify.workspace.Workspace` 

A Workspace whose config directory may contain an 

`~lsst.ap.pipe.ApPipeTask` config. 

 

Returns 

------- 

args : `list` of `str` 

Command-line arguments calling ``--config`` or ``--configFile``, 

following the conventions of `sys.argv`. 

""" 

overrideFile = apPipe.ApPipeTask._DefaultName + ".py" 

overridePath = os.path.join(workspace.configDir, overrideFile) 

 

args = ["--configfile", overridePath] 

# ApVerify will use the sqlite hooks for the Ppdb. 

args.extend(["--config", "ppdb.db_url=sqlite:///" + workspace.dbLocation]) 

args.extend(["--config", "ppdb.isolation_level=READ_UNCOMMITTED"]) 

 

return args