Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

import math 

import collections 

import operator 

from lsst.ctrl.pool.parallel import BatchPoolTask 

from lsst.ctrl.pool.pool import Pool 

from lsst.pipe.base import ArgumentParser 

from lsst.pex.config import Config 

 

__all__ = ["DemoTask", ] 

 

 

class DemoTask(BatchPoolTask): 

"""Task for demonstrating the BatchPoolTask functionality""" 

ConfigClass = Config 

_DefaultName = "demo" 

 

@classmethod 

def _makeArgumentParser(cls, *args, **kwargs): 

kwargs.pop('doBatch', False) # Unused 

parser = ArgumentParser(name="demo", *args, **kwargs) 

parser.add_id_argument("--id", datasetType="raw", level="visit", 

help="data ID, e.g. --id visit=12345") 

return parser 

 

@classmethod 

def batchWallTime(cls, time, parsedCmd, numCores): 

"""Return walltime request for batch job 

 

Subclasses should override if the walltime should be calculated 

differently (e.g., addition of some serial time). 

 

@param time: Requested time per iteration 

@param parsedCmd: Results of argument parsing 

@param numCores: Number of cores 

""" 

numTargets = [sum(1 for ccdRef in visitRef.subItems("ccd") if ccdRef.datasetExists("raw")) for 

visitRef in parsedCmd.id.refList] 

return time*sum(math.ceil(tt/numCores) for tt in numTargets) 

 

def runDataRef(self, visitRef): 

"""Main entry-point 

 

Only the master node runs this method. It will dispatch jobs to the 

slave nodes. 

""" 

pool = Pool("test") 

 

# Less overhead to transfer the butler once rather than in each dataRef 

dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId) 

for ccdRef in visitRef.subItems("ccd") if ccdRef.datasetExists("raw")]) 

dataIdList = collections.OrderedDict(sorted(dataIdList.items())) 

 

with self.logOperation("master"): 

total = pool.reduce(operator.add, self.run, list(dataIdList.values()), 

butler=visitRef.getButler()) 

self.log.info("Total number of pixels read: %d" % (total,)) 

 

def run(self, cache, dataId, butler=None): 

"""Read image and return number of pixels 

 

Only the slave nodes run this method. 

""" 

assert butler is not None 

with self.logOperation("read %s" % (dataId,)): 

raw = butler.get("raw", dataId, immediate=True) 

dims = raw.getDimensions() 

num = dims.getX()*dims.getY() 

self.log.info("Read %d pixels for %s" % (num, dataId,)) 

return num 

 

def _getConfigName(self): 

return None 

 

def _getMetadataName(self): 

return None 

 

def _getEupsVersionsName(self): 

return None