Coverage for python/lsst/ap/pipe/ap_pipe.py: 28%

1# 

2# This file is part of ap_pipe. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24__all__ = ["ApPipeConfig", "ApPipeTask"] 

25 

26import warnings 

27 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30 

31from lsst.pipe.tasks.processCcd import ProcessCcdTask 

32from lsst.pipe.tasks.imageDifference import ImageDifferenceTask 

33from lsst.ap.association import DiaPipelineTask 

34from lsst.ap.association.transformDiaSourceCatalog import TransformDiaSourceCatalogTask 

35from lsst.ap.pipe.apPipeParser import ApPipeParser 

36from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner 

37 

38 

39class ApPipeConfig(pexConfig.Config): 

40 """Settings and defaults for ApPipeTask. 

41 """ 

42 

43 ccdProcessor = pexConfig.ConfigurableField( 

44 target=ProcessCcdTask, 

45 doc="Task used to perform basic image reduction and characterization.", 

46 ) 

47 differencer = pexConfig.ConfigurableField( 

48 target=ImageDifferenceTask, 

49 doc="Task used to do image subtraction and DiaSource detection.", 

50 ) 

51 transformDiaSrcCat = pexConfig.ConfigurableField( 

52 target=TransformDiaSourceCatalogTask, 

53 doc="Task for converting and calibrating the afw SourceCatalog of " 

54 "DiaSources to Pandas DataFrame for use in Association." 

55 ) 

56 diaPipe = pexConfig.ConfigurableField( 

57 target=DiaPipelineTask, 

58 doc="Pipeline task for loading/store DiaSources and DiaObjects and " 

59 "spatially associating them.", 

60 ) 

61 

62 def setDefaults(self): 

63 """Settings appropriate for most or all ap_pipe runs. 

64 """ 

65 

66 # Write the WarpedExposure to disk for use in Alert Packet creation. 

67 self.differencer.doWriteWarpedExp = True 

68 

69 def validate(self): 

70 pexConfig.Config.validate(self) 

71 if not self.ccdProcessor.calibrate.doWrite or not self.ccdProcessor.calibrate.doWriteExposure: 

72 raise ValueError("Differencing needs calexps [ccdProcessor.calibrate.doWrite, doWriteExposure]") 

73 if not self.differencer.doMeasurement: 

74 raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].") 

75 if not self.differencer.doWriteWarpedExp: 

76 raise ValueError("Alert generation needs warped exposures [differencer.doWriteWarpedExp].") 

77 if not self.differencer.doWriteSources: 

78 raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].") 

79 if not self.differencer.doWriteSubtractedExp: 

80 raise ValueError("Source association needs difference exposures " 

81 "[differencer.doWriteSubtractedExp].") 

82 

83 

84class ApPipeTask(pipeBase.CmdLineTask): 

85 """Command-line task representing the entire AP pipeline. 

86 

87 ``ApPipeTask`` processes raw DECam images from basic processing through 

88 source association. Other observatories will be supported in the future. 

89 

90 ``ApPipeTask`` can be run from the command line, but it can also be called 

91 from other pipeline code. It provides public methods for executing each 

92 major step of the pipeline by itself. 

93 

94 Parameters 

95 ---------- 

96 butler : `lsst.daf.persistence.Butler` 

97 A Butler providing access to the science, calibration, and (unless 

98 ``config.differencer.getTemplate`` is overridden) template data to 

99 be processed. Its output repository must be both readable 

100 and writable. 

101 """ 

102 

103 ConfigClass = ApPipeConfig 

104 RunnerClass = ApPipeTaskRunner 

105 _DefaultName = "apPipe" 

106 

107 def __init__(self, butler, *args, **kwargs): 

108 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

109 

110 self.makeSubtask("ccdProcessor", butler=butler) 

111 self.makeSubtask("differencer", butler=butler) 

112 self.makeSubtask("transformDiaSrcCat", initInputs={"diaSourceSchema": self.differencer.outputSchema}) 

113 self.makeSubtask("diaPipe") 

114 

115 @pipeBase.timeMethod 

116 def runDataRef(self, rawRef, templateIds=None, reuse=None): 

117 """Execute the ap_pipe pipeline on a single image. 

118 

119 Parameters 

120 ---------- 

121 rawRef : `lsst.daf.persistence.ButlerDataRef` 

122 A reference to the raw data to process. 

123 templateIds : `list` of `dict`, optional 

124 A list of parsed data IDs for templates to use. Only used if 

125 ``config.differencer`` is configured to do so. ``differencer`` or 

126 its subtasks may restrict the allowed IDs. 

127 reuse : `list` of `str`, optional 

128 The names of all subtasks that may be skipped if their output is 

129 present. Defaults to skipping nothing. 

130 

131 Returns 

132 ------- 

133 result : `lsst.pipe.base.Struct` 

134 Result struct with components: 

135 

136 - l1Database : handle for accessing the final association database, conforming to 

137 `ap_association`'s DB access API 

138 - ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

139 - differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

140 - diaPipe : output of `config.diaPipe.run` (`lsst.pipe.base.Struct` or `None`). 

141 """ 

142 if reuse is None: 

143 reuse = [] 

144 # Work around mismatched HDU lists for raw and processed data 

145 calexpId = rawRef.dataId.copy() 

146 if 'hdu' in calexpId: 

147 del calexpId['hdu'] 

148 calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId) 

149 

150 # Ensure that templateIds make it through basic data reduction 

151 # TODO: treat as independent jobs (may need SuperTask framework?) 

152 if templateIds is not None: 

153 for templateId in templateIds: 

154 # templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef 

155 rawTemplateRef = _siblingRef(rawRef, "raw", templateId) 

156 calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId) 

157 if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp"): 

158 self.runProcessCcd(rawTemplateRef) 

159 

160 if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp"): 

161 self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId)) 

162 processResults = None 

163 else: 

164 processResults = self.runProcessCcd(rawRef) 

165 

166 diffType = self.config.differencer.coaddName 

167 if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc"): 

168 self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId)) 

169 diffImResults = None 

170 else: 

171 diffImResults = self.runDiffIm(calexpRef, templateIds) 

172 

173 if "diaPipe" in reuse: 

174 warnings.warn( 

175 "Reusing association results for some images while rerunning " 

176 "others may change the associations. If exact reproducibility " 

177 "matters, please clear the association database and run " 

178 "ap_pipe.py with --reuse-output-from=differencer to redo all " 

179 "association results consistently.") 

180 if "diaPipe" in reuse and calexpRef.datasetExists("apdb_marker"): 

181 message = "DiaPipeline has already been run for {0}, skipping...".format(calexpRef.dataId) 

182 self.log.info(message) 

183 diaPipeResults = None 

184 else: 

185 diaPipeResults = self.runAssociation(calexpRef) 

186 

187 return pipeBase.Struct( 

188 l1Database=self.diaPipe.apdb, 

189 ccdProcessor=processResults if processResults else None, 

190 differencer=diffImResults if diffImResults else None, 

191 diaPipe=diaPipeResults.taskResults if diaPipeResults else None 

192 ) 

193 

194 @pipeBase.timeMethod 

195 def runProcessCcd(self, sensorRef): 

196 """Perform ISR with ingested images and calibrations via processCcd. 

197 

198 The output repository associated with ``sensorRef`` will be populated with the 

199 usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR). 

200 

201 Parameters 

202 ---------- 

203 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

204 Data reference for raw data. 

205 

206 Returns 

207 ------- 

208 result : `lsst.pipe.base.Struct` 

209 Output of `config.ccdProcessor.runDataRef`. 

210 

211 Notes 

212 ----- 

213 The input repository corresponding to ``sensorRef`` must already contain the refcats. 

214 """ 

215 self.log.info("Running ProcessCcd...") 

216 return self.ccdProcessor.runDataRef(sensorRef) 

217 

218 @pipeBase.timeMethod 

219 def runDiffIm(self, sensorRef, templateIds=None): 

220 """Do difference imaging with a template and a science image 

221 

222 The output repository associated with ``sensorRef`` will be populated with difference images 

223 and catalogs of detected sources (diaSrc, diffexp, and metadata files) 

224 

225 Parameters 

226 ---------- 

227 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

228 Data reference for multiple dataset types, both input and output. 

229 templateIds : `list` of `dict`, optional 

230 A list of parsed data IDs for templates to use. Only used if 

231 ``config.differencer`` is configured to do so. ``differencer`` or 

232 its subtasks may restrict the allowed IDs. 

233 

234 Returns 

235 ------- 

236 result : `lsst.pipe.base.Struct` 

237 Output of `config.differencer.runDataRef`. 

238 """ 

239 self.log.info("Running ImageDifference...") 

240 return self.differencer.runDataRef(sensorRef, templateIdList=templateIds) 

241 

242 @pipeBase.timeMethod 

243 def runAssociation(self, sensorRef): 

244 """Do source association. 

245 

246 This method writes an ``apdb_marker`` dataset once all changes related 

247 to the current exposure have been committed. 

248 

249 Parameters 

250 ---------- 

251 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

252 Data reference for multiple input dataset types. 

253 

254 Returns 

255 ------- 

256 result : `lsst.pipe.base.Struct` 

257 Result struct with components: 

258 

259 - apdb : `lsst.dax.apdb.Apdb` Initialized association database containing final association 

260 results. 

261 - taskResults : output of `config.diaPipe.run` (`lsst.pipe.base.Struct`). 

262 """ 

263 diffType = self.config.differencer.coaddName 

264 diffIm = sensorRef.get(diffType + "Diff_differenceExp") 

265 

266 transformResult = self.transformDiaSrcCat.run( 

267 diaSourceCat=sensorRef.get(diffType + "Diff_diaSrc"), 

268 diffIm=diffIm, 

269 band=diffIm.getFilterLabel().bandLabel, 

270 ccdVisitId=diffIm.getInfo().getVisitInfo().getExposureId()) 

271 

272 results = self.diaPipe.run( 

273 diaSourceTable=transformResult.diaSourceTable, 

274 diffIm=sensorRef.get(diffType + "Diff_differenceExp"), 

275 exposure=sensorRef.get("calexp"), 

276 warpedExposure=sensorRef.get(diffType + "Diff_warpedExp"), 

277 ccdExposureIdBits=sensorRef.get("ccdExposureId_bits"), 

278 band=diffIm.getFilterLabel().bandLabel, 

279 ) 

280 

281 # apdb_marker triggers metrics processing; let them try to read 

282 # something even if association failed 

283 sensorRef.put(results.apdbMarker, "apdb_marker") 

284 

285 return pipeBase.Struct( 

286 l1Database=self.diaPipe.apdb, 

287 taskResults=results 

288 ) 

289 

290 @classmethod 

291 def _makeArgumentParser(cls): 

292 """A parser that can handle extra arguments for ap_pipe. 

293 """ 

294 return ApPipeParser(name=cls._DefaultName) 

295 

296 

297def _siblingRef(original, datasetType, dataId): 

298 """Construct a new dataRef using an existing dataRef as a template. 

299 

300 The typical application is to construct a data ID that differs from an 

301 existing ID in one or two keys, but is more specific than expanding a 

302 partial data ID would be. 

303 

304 Parameters 

305 ---------- 

306 original : `lsst.daf.persistence.ButlerDataRef` 

307 A dataRef related to the desired one. Assumed to represent a unique dataset. 

308 datasetType : `str` 

309 The desired type of the new dataRef. Must be compatible 

310 with ``original``. 

311 dataId : `dict` from `str` to any 

312 A possibly partial data ID for the new dataRef. Any properties left 

313 unspecified shall be copied from ``original``. 

314 

315 Returns 

316 ------- 

317 dataRef : `lsst.daf.persistence.ButlerDataRef` 

318 A dataRef to the same butler as ``original``, but of type 

319 ``datasetType`` and with data ID equivalent to 

320 ``original.dataId.update(dataId)``. 

321 """ 

322 butler = original.getButler() 

323 return butler.dataRef(datasetType, dataId=original.dataId, **dataId)