Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual'] 

26 

27import re 

28import numpy as np 

29 

30import lsst.pipe.base as pipeBase 

31import lsst.ip.isr as ipIsr 

32import lsst.log 

33 

34 

35def countMaskedPixels(maskedIm, maskPlane): 

36 """Count the number of pixels in a given mask plane.""" 

37 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

38 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

39 return nPix 

40 

41 

42class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

43 """Subclass of TaskRunner for handling intrinsically paired visits. 

44 

45 This transforms the processed arguments generated by the ArgumentParser 

46 into the arguments expected by tasks which take visit pairs for their 

47 run() methods. 

48 

49 Such tasks' run() methods tend to take two arguments, 

50 one of which is the dataRef (as usual), and the other is the list 

51 of visit-pairs, in the form of a list of tuples. 

52 This list is supplied on the command line as documented, 

53 and this class parses that, and passes the parsed version 

54 to the run() method. 

55 

56 See pipeBase.TaskRunner for more information. 

57 """ 

58 

59 @staticmethod 

60 def getTargetList(parsedCmd, **kwargs): 

61 """Parse the visit list and pass through explicitly.""" 

62 visitPairs = [] 

63 for visitStringPair in parsedCmd.visitPairs: 

64 visitStrings = visitStringPair.split(",") 

65 if len(visitStrings) != 2: 

66 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

67 visitStringPair)) 

68 try: 

69 visits = [int(visit) for visit in visitStrings] 

70 except Exception: 

71 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

72 visitPairs.append(visits) 

73 

74 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

75 

76 

77def parseCmdlineNumberString(inputString): 

78 """Parse command line numerical expression sytax and return as list of int 

79 

80 Take an input of the form "'1..5:2^123..126'" as a string, and return 

81 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

82 """ 

83 outList = [] 

84 for subString in inputString.split("^"): 

85 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

86 if mat: 

87 v1 = int(mat.group(1)) 

88 v2 = int(mat.group(2)) 

89 v3 = mat.group(3) 

90 v3 = int(v3) if v3 else 1 

91 for v in range(v1, v2 + 1, v3): 

92 outList.append(int(v)) 

93 else: 

94 outList.append(int(subString)) 

95 return outList 

96 

97 

98class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

99 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

100 

101 This transforms the processed arguments generated by the ArgumentParser 

102 into the arguments expected by tasks which require a list of visits 

103 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

104 

105 Such tasks' run() methods tend to take two arguments, 

106 one of which is the dataRef (as usual), and the other is the list 

107 of visits. 

108 This list is supplied on the command line as documented, 

109 and this class parses that, and passes the parsed version 

110 to the run() method. 

111 

112 See `lsst.pipe.base.TaskRunner` for more information. 

113 """ 

114 

115 @staticmethod 

116 def getTargetList(parsedCmd, **kwargs): 

117 """Parse the visit list and pass through explicitly.""" 

118 # if this has been pre-parsed and therefore doesn't have length of one 

119 # then something has gone wrong, so execution should stop here. 

120 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

121 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

122 

123 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

124 

125 

126class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

127 """A DataIdContainer for the tasks for which the output does 

128 not yet exist.""" 

129 

130 def makeDataRefList(self, namespace): 

131 """Compute refList based on idList. 

132 

133 This method must be defined as the dataset does not exist before this 

134 task is run. 

135 

136 Parameters 

137 ---------- 

138 namespace 

139 Results of parsing the command-line. 

140 

141 Notes 

142 ----- 

143 Not called if ``add_id_argument`` called 

144 with ``doMakeDataRefList=False``. 

145 Note that this is almost a copy-and-paste of the vanilla 

146 implementation, but without checking if the datasets already exist, 

147 as this task exists to make them. 

148 """ 

149 if self.datasetType is None: 

150 raise RuntimeError("Must call setDatasetType first") 

151 butler = namespace.butler 

152 for dataId in self.idList: 

153 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

154 # exclude nonexistent data 

155 # this is a recursive test, e.g. for the sake of "raw" data 

156 if not refList: 

157 namespace.log.warn("No data found for dataId=%s", dataId) 

158 continue 

159 self.refList += refList 

160 

161 

162def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

163 """Check the exposure lengths of two exposures are equal. 

164 

165 Parameters: 

166 ----------- 

167 exp1 : `lsst.afw.image.exposure.ExposureF` 

168 First exposure to check 

169 exp2 : `lsst.afw.image.exposure.ExposureF` 

170 Second exposure to check 

171 v1 : `int` or `str`, optional 

172 First visit of the visit pair 

173 v2 : `int` or `str`, optional 

174 Second visit of the visit pair 

175 raiseWithMessage : `bool` 

176 If True, instead of returning a bool, raise a RuntimeError if exposure 

177 times are not equal, with a message about which visits mismatch if the 

178 information is available. 

179 

180 Raises: 

181 ------- 

182 RuntimeError 

183 Raised if the exposure lengths of the two exposures are not equal 

184 """ 

185 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

186 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

187 if expTime1 != expTime2: 

188 if raiseWithMessage: 

189 msg = "Exposure lengths for visit pairs must be equal. " + \ 

190 "Found %s and %s" % (expTime1, expTime2) 

191 if v1 and v2: 

192 msg += " for visit pair %s, %s" % (v1, v2) 

193 raise RuntimeError(msg) 

194 else: 

195 return False 

196 return True 

197 

198 

199def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

200 checkTrim=True, logName=None): 

201 """Check that appropriate ISR settings have been selected for the task. 

202 

203 Note that this checks that the task itself is configured correctly rather 

204 than checking a config. 

205 

206 Parameters 

207 ---------- 

208 isrTask : `lsst.ip.isr.IsrTask` 

209 The task whose config is to be validated 

210 

211 mandatory : `iterable` of `str` 

212 isr steps that must be set to True. Raises if False or missing 

213 

214 forbidden : `iterable` of `str` 

215 isr steps that must be set to False. Raises if True, warns if missing 

216 

217 desirable : `iterable` of `str` 

218 isr steps that should probably be set to True. Warns is False, info if 

219 missing 

220 

221 undesirable : `iterable` of `str` 

222 isr steps that should probably be set to False. Warns is True, info if 

223 missing 

224 

225 checkTrim : `bool` 

226 Check to ensure the isrTask's assembly subtask is trimming the images. 

227 This is a separate config as it is very ugly to do this within the 

228 normal configuration lists as it is an option of a sub task. 

229 

230 Raises 

231 ------ 

232 RuntimeError 

233 Raised if ``mandatory`` config parameters are False, 

234 or if ``forbidden`` parameters are True. 

235 

236 TypeError 

237 Raised if parameter ``isrTask`` is an invalid type. 

238 

239 Notes 

240 ----- 

241 Logs warnings using an isrValidation logger for desirable/undesirable 

242 options that are of the wrong polarity or if keys are missing. 

243 """ 

244 if not isinstance(isrTask, ipIsr.IsrTask): 

245 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

246 

247 configDict = isrTask.config.toDict() 

248 

249 if logName and isinstance(logName, str): 

250 log = lsst.log.getLogger(logName) 

251 else: 

252 log = lsst.log.getLogger("isrValidation") 

253 

254 if mandatory: 

255 for configParam in mandatory: 

256 if configParam not in configDict: 

257 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

258 if configDict[configParam] is False: 

259 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

260 

261 if forbidden: 

262 for configParam in forbidden: 

263 if configParam not in configDict: 

264 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

265 " forbidden list should each have an associated Field in IsrConfig:" 

266 " check that there is not a typo in this case.") 

267 continue 

268 if configDict[configParam] is True: 

269 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

270 

271 if desirable: 

272 for configParam in desirable: 

273 if configParam not in configDict: 

274 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

275 " to set the equivalent for your obs_package to True.") 

276 continue 

277 if configDict[configParam] is False: 

278 log.warn(f"Found config.isr.{configParam} set to False for this task." + 

279 " The cp_pipe Config recommends setting this to True.") 

280 if undesirable: 

281 for configParam in undesirable: 

282 if configParam not in configDict: 

283 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

284 " to set the equivalent for your obs_package to False.") 

285 continue 

286 if configDict[configParam] is True: 

287 log.warn(f"Found config.isr.{configParam} set to True for this task." + 

288 " The cp_pipe Config recommends setting this to False.") 

289 

290 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

291 if not isrTask.assembleCcd.config.doTrim: 

292 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")