Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_pipe. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24__all__ = ["ApPipeConfig", "ApPipeTask"] 

25 

26import warnings 

27 

28from sqlalchemy.exc import OperationalError, ProgrammingError 

29 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33from lsst.pipe.tasks.processCcd import ProcessCcdTask 

34from lsst.pipe.tasks.imageDifference import ImageDifferenceTask 

35from lsst.ap.association import DiaPipelineTask 

36from lsst.ap.pipe.apPipeParser import ApPipeParser 

37from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner 

38 

39 

40class ApPipeConfig(pexConfig.Config): 

41 """Settings and defaults for ApPipeTask. 

42 """ 

43 

44 ccdProcessor = pexConfig.ConfigurableField( 

45 target=ProcessCcdTask, 

46 doc="Task used to perform basic image reduction and characterization.", 

47 ) 

48 differencer = pexConfig.ConfigurableField( 

49 target=ImageDifferenceTask, 

50 doc="Task used to do image subtraction and DiaSource detection.", 

51 ) 

52 diaPipe = pexConfig.ConfigurableField( 

53 target=DiaPipelineTask, 

54 doc="Pipeline task for loading/store DiaSources and DiaObjects and " 

55 "spatially associating them.", 

56 ) 

57 

58 def setDefaults(self): 

59 """Settings appropriate for most or all ap_pipe runs. 

60 """ 

61 # Always prefer decorrelation; may eventually become ImageDifferenceTask default 

62 self.differencer.doDecorrelation = True 

63 self.differencer.detection.thresholdValue = 5.0 # needed with doDecorrelation 

64 

65 # Don't have source catalogs for templates 

66 self.differencer.doSelectSources = False 

67 

68 def validate(self): 

69 pexConfig.Config.validate(self) 

70 if not self.ccdProcessor.calibrate.doWrite or not self.ccdProcessor.calibrate.doWriteExposure: 

71 raise ValueError("Differencing needs calexps [ccdProcessor.calibrate.doWrite, doWriteExposure]") 

72 if not self.differencer.doMeasurement: 

73 raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].") 

74 if not self.differencer.doWriteSources: 

75 raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].") 

76 if not self.differencer.doWriteSubtractedExp: 

77 raise ValueError("Source association needs difference exposures " 

78 "[differencer.doWriteSubtractedExp].") 

79 

80 

81class ApPipeTask(pipeBase.CmdLineTask): 

82 """Command-line task representing the entire AP pipeline. 

83 

84 ``ApPipeTask`` processes raw DECam images from basic processing through 

85 source association. Other observatories will be supported in the future. 

86 

87 ``ApPipeTask`` can be run from the command line, but it can also be called 

88 from other pipeline code. It provides public methods for executing each 

89 major step of the pipeline by itself. 

90 

91 Parameters 

92 ---------- 

93 butler : `lsst.daf.persistence.Butler` 

94 A Butler providing access to the science, calibration, and (unless 

95 ``config.differencer.getTemplate`` is overridden) template data to 

96 be processed. Its output repository must be both readable 

97 and writable. 

98 """ 

99 

100 ConfigClass = ApPipeConfig 

101 RunnerClass = ApPipeTaskRunner 

102 _DefaultName = "apPipe" 

103 

104 def __init__(self, butler, *args, **kwargs): 

105 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

106 

107 self.makeSubtask("ccdProcessor", butler=butler) 

108 self.makeSubtask("differencer", butler=butler) 

109 self.makeSubtask("diaPipe", initInputs={"diaSourceSchema": self.differencer.outputSchema}) 

110 

111 @pipeBase.timeMethod 

112 def runDataRef(self, rawRef, templateIds=None, reuse=None): 

113 """Execute the ap_pipe pipeline on a single image. 

114 

115 Parameters 

116 ---------- 

117 rawRef : `lsst.daf.persistence.ButlerDataRef` 

118 A reference to the raw data to process. 

119 templateIds : `list` of `dict`, optional 

120 A list of parsed data IDs for templates to use. Only used if 

121 ``config.differencer`` is configured to do so. ``differencer`` or 

122 its subtasks may restrict the allowed IDs. 

123 reuse : `list` of `str`, optional 

124 The names of all subtasks that may be skipped if their output is 

125 present. Defaults to skipping nothing. 

126 

127 Returns 

128 ------- 

129 result : `lsst.pipe.base.Struct` 

130 Result struct with components: 

131 

132 - l1Database : handle for accessing the final association database, conforming to 

133 `ap_association`'s DB access API 

134 - ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

135 - differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

136 - diaPipe : output of `config.diaPipe.run` (`lsst.pipe.base.Struct` or `None`). 

137 """ 

138 if reuse is None: 

139 reuse = [] 

140 # Work around mismatched HDU lists for raw and processed data 

141 calexpId = rawRef.dataId.copy() 

142 if 'hdu' in calexpId: 

143 del calexpId['hdu'] 

144 calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId) 

145 

146 # Ensure that templateIds make it through basic data reduction 

147 # TODO: treat as independent jobs (may need SuperTask framework?) 

148 if templateIds is not None: 

149 for templateId in templateIds: 

150 # templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef 

151 rawTemplateRef = _siblingRef(rawRef, "raw", templateId) 

152 calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId) 

153 if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp", write=True): 

154 self.runProcessCcd(rawTemplateRef) 

155 

156 if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp", write=True): 

157 self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId)) 

158 processResults = None 

159 else: 

160 processResults = self.runProcessCcd(rawRef) 

161 

162 diffType = self.config.differencer.coaddName 

163 if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc", write=True): 

164 self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId)) 

165 diffImResults = None 

166 else: 

167 diffImResults = self.runDiffIm(calexpRef, templateIds) 

168 

169 try: 

170 if "diaPipe" in reuse: 

171 warnings.warn( 

172 "Reusing association results for some images while rerunning " 

173 "others may change the associations. If exact reproducibility " 

174 "matters, please clear the association database and run " 

175 "ap_pipe.py with --reuse-output-from=differencer to redo all " 

176 "association results consistently.") 

177 if "diaPipe" in reuse and calexpRef.datasetExists("apdb_marker", write=True): 

178 message = "DiaPipeline has already been run for {0}, skipping...".format(calexpRef.dataId) 

179 self.log.info(message) 

180 diaPipeResults = None 

181 else: 

182 diaPipeResults = self.runAssociation(calexpRef) 

183 except (OperationalError, ProgrammingError) as e: 

184 # Don't use lsst.pipe.base.TaskError because it mixes poorly with exception chaining 

185 raise RuntimeError("Database query failed; did you call make_apdb.py first?") from e 

186 

187 return pipeBase.Struct( 

188 l1Database=self.diaPipe.apdb, 

189 ccdProcessor=processResults if processResults else None, 

190 differencer=diffImResults if diffImResults else None, 

191 diaPipe=diaPipeResults.taskResults if diaPipeResults else None 

192 ) 

193 

194 @pipeBase.timeMethod 

195 def runProcessCcd(self, sensorRef): 

196 """Perform ISR with ingested images and calibrations via processCcd. 

197 

198 The output repository associated with ``sensorRef`` will be populated with the 

199 usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR). 

200 

201 Parameters 

202 ---------- 

203 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

204 Data reference for raw data. 

205 

206 Returns 

207 ------- 

208 result : `lsst.pipe.base.Struct` 

209 Output of `config.ccdProcessor.runDataRef`. 

210 

211 Notes 

212 ----- 

213 The input repository corresponding to ``sensorRef`` must already contain the refcats. 

214 """ 

215 self.log.info("Running ProcessCcd...") 

216 return self.ccdProcessor.runDataRef(sensorRef) 

217 

218 @pipeBase.timeMethod 

219 def runDiffIm(self, sensorRef, templateIds=None): 

220 """Do difference imaging with a template and a science image 

221 

222 The output repository associated with ``sensorRef`` will be populated with difference images 

223 and catalogs of detected sources (diaSrc, diffexp, and metadata files) 

224 

225 Parameters 

226 ---------- 

227 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

228 Data reference for multiple dataset types, both input and output. 

229 templateIds : `list` of `dict`, optional 

230 A list of parsed data IDs for templates to use. Only used if 

231 ``config.differencer`` is configured to do so. ``differencer`` or 

232 its subtasks may restrict the allowed IDs. 

233 

234 Returns 

235 ------- 

236 result : `lsst.pipe.base.Struct` 

237 Output of `config.differencer.runDataRef`. 

238 """ 

239 self.log.info("Running ImageDifference...") 

240 return self.differencer.runDataRef(sensorRef, templateIdList=templateIds) 

241 

242 @pipeBase.timeMethod 

243 def runAssociation(self, sensorRef): 

244 """Do source association. 

245 

246 This method writes an ``apdb_marker`` dataset once all changes related 

247 to the current exposure have been committed. 

248 

249 Parameters 

250 ---------- 

251 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

252 Data reference for multiple input dataset types. 

253 

254 Returns 

255 ------- 

256 result : `lsst.pipe.base.Struct` 

257 Result struct with components: 

258 

259 - apdb : `lsst.dax.apdb.Apdb` Initialized association database containing final association 

260 results. 

261 - taskResults : output of `config.diaPipe.run` (`lsst.pipe.base.Struct`). 

262 """ 

263 diffType = self.config.differencer.coaddName 

264 

265 results = self.diaPipe.run( 

266 diaSourceCat=sensorRef.get(diffType + "Diff_diaSrc"), 

267 diffIm=sensorRef.get(diffType + "Diff_differenceExp"), 

268 exposure=sensorRef.get("calexp"), 

269 ccdExposureIdBits=sensorRef.get("ccdExposureId_bits")) 

270 

271 # apdb_marker triggers metrics processing; let them try to read 

272 # something even if association failed 

273 sensorRef.put(results.apdbMarker, "apdb_marker") 

274 

275 return pipeBase.Struct( 

276 l1Database=self.diaPipe.apdb, 

277 taskResults=results 

278 ) 

279 

280 @classmethod 

281 def _makeArgumentParser(cls): 

282 """A parser that can handle extra arguments for ap_pipe. 

283 """ 

284 return ApPipeParser(name=cls._DefaultName) 

285 

286 

287def _siblingRef(original, datasetType, dataId): 

288 """Construct a new dataRef using an existing dataRef as a template. 

289 

290 The typical application is to construct a data ID that differs from an 

291 existing ID in one or two keys, but is more specific than expanding a 

292 partial data ID would be. 

293 

294 Parameters 

295 ---------- 

296 original : `lsst.daf.persistence.ButlerDataRef` 

297 A dataRef related to the desired one. Assumed to represent a unique dataset. 

298 datasetType : `str` 

299 The desired type of the new dataRef. Must be compatible 

300 with ``original``. 

301 dataId : `dict` from `str` to any 

302 A possibly partial data ID for the new dataRef. Any properties left 

303 unspecified shall be copied from ``original``. 

304 

305 Returns 

306 ------- 

307 dataRef : `lsst.daf.persistence.ButlerDataRef` 

308 A dataRef to the same butler as ``original``, but of type 

309 ``datasetType`` and with data ID equivalent to 

310 ``original.dataId.update(dataId)``. 

311 """ 

312 butler = original.getButler() 

313 return butler.dataRef(datasetType, dataId=original.dataId, **dataId)