Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

# This file is part of cp_pipe. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (https://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <https://www.gnu.org/licenses/>. 

# 

 

__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

'countMaskedPixels', 'checkExpLengthEqual'] 

 

import re 

import numpy as np 

 

import lsst.pipe.base as pipeBase 

import lsst.ip.isr as ipIsr 

import lsst.log 

 

 

def countMaskedPixels(maskedIm, maskPlane): 

"""Count the number of pixels in a given mask plane.""" 

maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

return nPix 

 

 

class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

"""Subclass of TaskRunner for handling intrinsically paired visits. 

 

This transforms the processed arguments generated by the ArgumentParser 

into the arguments expected by tasks which take visit pairs for their 

run() methods. 

 

Such tasks' run() methods tend to take two arguments, 

one of which is the dataRef (as usual), and the other is the list 

of visit-pairs, in the form of a list of tuples. 

This list is supplied on the command line as documented, 

and this class parses that, and passes the parsed version 

to the run() method. 

 

See pipeBase.TaskRunner for more information. 

""" 

 

@staticmethod 

def getTargetList(parsedCmd, **kwargs): 

"""Parse the visit list and pass through explicitly.""" 

visitPairs = [] 

for visitStringPair in parsedCmd.visitPairs: 

visitStrings = visitStringPair.split(",") 

if len(visitStrings) != 2: 

raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

visitStringPair)) 

try: 

visits = [int(visit) for visit in visitStrings] 

except Exception: 

raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

visitPairs.append(visits) 

 

return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

 

 

def parseCmdlineNumberString(inputString): 

"""Parse command line numerical expression sytax and return as list of int 

 

Take an input of the form "'1..5:2^123..126'" as a string, and return 

a list of ints as [1, 3, 5, 123, 124, 125, 126] 

""" 

outList = [] 

for subString in inputString.split("^"): 

mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

if mat: 

v1 = int(mat.group(1)) 

v2 = int(mat.group(2)) 

v3 = mat.group(3) 

v3 = int(v3) if v3 else 1 

for v in range(v1, v2 + 1, v3): 

outList.append(int(v)) 

else: 

outList.append(int(subString)) 

return outList 

 

 

class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

"""Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

 

This transforms the processed arguments generated by the ArgumentParser 

into the arguments expected by tasks which require a list of visits 

to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

 

Such tasks' run() methods tend to take two arguments, 

one of which is the dataRef (as usual), and the other is the list 

of visits. 

This list is supplied on the command line as documented, 

and this class parses that, and passes the parsed version 

to the run() method. 

 

See `lsst.pipe.base.TaskRunner` for more information. 

""" 

 

@staticmethod 

def getTargetList(parsedCmd, **kwargs): 

"""Parse the visit list and pass through explicitly.""" 

# if this has been pre-parsed and therefore doesn't have length of one 

# then something has gone wrong, so execution should stop here. 

assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

 

return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

 

 

class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

"""A DataIdContainer for the tasks for which the output does 

not yet exist.""" 

 

def makeDataRefList(self, namespace): 

"""Compute refList based on idList. 

 

This method must be defined as the dataset does not exist before this 

task is run. 

 

Parameters 

---------- 

namespace 

Results of parsing the command-line. 

 

Notes 

----- 

Not called if ``add_id_argument`` called 

with ``doMakeDataRefList=False``. 

Note that this is almost a copy-and-paste of the vanilla 

implementation, but without checking if the datasets already exist, 

as this task exists to make them. 

""" 

if self.datasetType is None: 

raise RuntimeError("Must call setDatasetType first") 

butler = namespace.butler 

for dataId in self.idList: 

refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

# exclude nonexistent data 

# this is a recursive test, e.g. for the sake of "raw" data 

if not refList: 

namespace.log.warn("No data found for dataId=%s", dataId) 

continue 

self.refList += refList 

 

 

def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

"""Check the exposure lengths of two exposures are equal. 

 

Parameters: 

----------- 

exp1 : `lsst.afw.image.exposure.ExposureF` 

First exposure to check 

exp2 : `lsst.afw.image.exposure.ExposureF` 

Second exposure to check 

v1 : `int` or `str`, optional 

First visit of the visit pair 

v2 : `int` or `str`, optional 

Second visit of the visit pair 

raiseWithMessage : `bool` 

If True, instead of returning a bool, raise a RuntimeError if exposure 

times are not equal, with a message about which visits mismatch if the 

information is available. 

 

Raises: 

------- 

RuntimeError 

Raised if the exposure lengths of the two exposures are not equal 

""" 

expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

if expTime1 != expTime2: 

if raiseWithMessage: 

msg = "Exposure lengths for visit pairs must be equal. " + \ 

"Found %s and %s" % (expTime1, expTime2) 

if v1 and v2: 

msg += " for visit pair %s, %s" % (v1, v2) 

raise RuntimeError(msg) 

else: 

return False 

return True 

 

 

def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

checkTrim=True, logName=None): 

"""Check that appropriate ISR settings have been selected for the task. 

 

Note that this checks that the task itself is configured correctly rather 

than checking a config. 

 

Parameters 

---------- 

isrTask : `lsst.ip.isr.IsrTask` 

The task whose config is to be validated 

 

mandatory : `iterable` of `str` 

isr steps that must be set to True. Raises if False or missing 

 

forbidden : `iterable` of `str` 

isr steps that must be set to False. Raises if True, warns if missing 

 

desirable : `iterable` of `str` 

isr steps that should probably be set to True. Warns is False, info if 

missing 

 

undesirable : `iterable` of `str` 

isr steps that should probably be set to False. Warns is True, info if 

missing 

 

checkTrim : `bool` 

Check to ensure the isrTask's assembly subtask is trimming the images. 

This is a separate config as it is very ugly to do this within the 

normal configuration lists as it is an option of a sub task. 

 

Raises 

------ 

RuntimeError 

Raised if ``mandatory`` config parameters are False, 

or if ``forbidden`` parameters are True. 

 

TypeError 

Raised if parameter ``isrTask`` is an invalid type. 

 

Notes 

----- 

Logs warnings using an isrValidation logger for desirable/undesirable 

options that are of the wrong polarity or if keys are missing. 

""" 

if not isinstance(isrTask, ipIsr.IsrTask): 

raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

 

configDict = isrTask.config.toDict() 

 

if logName and isinstance(logName, str): 

log = lsst.log.getLogger(logName) 

else: 

log = lsst.log.getLogger("isrValidation") 

 

if mandatory: 

for configParam in mandatory: 

if configParam not in configDict: 

raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

if configDict[configParam] is False: 

raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

 

if forbidden: 

for configParam in forbidden: 

if configParam not in configDict: 

log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

" forbidden list should each have an associated Field in IsrConfig:" 

" check that there is not a typo in this case.") 

continue 

if configDict[configParam] is True: 

raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

 

if desirable: 

for configParam in desirable: 

if configParam not in configDict: 

log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

" to set the equivalent for your obs_package to True.") 

continue 

if configDict[configParam] is False: 

log.warn(f"Found config.isr.{configParam} set to False for this task." + 

" The cp_pipe Config recommends setting this to True.") 

if undesirable: 

for configParam in undesirable: 

if configParam not in configDict: 

log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

" to set the equivalent for your obs_package to False.") 

continue 

if configDict[configParam] is True: 

log.warn(f"Found config.isr.{configParam} set to True for this task." + 

" The cp_pipe Config recommends setting this to False.") 

 

if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

if not isrTask.assembleCcd.config.doTrim: 

raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")