Coverage for python/lsst/ap/pipe/ap_pipe.py: 28%

1# 

2# This file is part of ap_pipe. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24__all__ = ["ApPipeConfig", "ApPipeTask"] 

25 

26import warnings 

27 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30 

31from lsst.pipe.tasks.processCcd import ProcessCcdTask 

32from lsst.pipe.tasks.imageDifference import ImageDifferenceTask 

33from lsst.ap.association import DiaPipelineTask 

34from lsst.ap.association.transformDiaSourceCatalog import TransformDiaSourceCatalogTask 

35from lsst.ap.pipe.apPipeParser import ApPipeParser 

36from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner 

37from lsst.utils.timer import timeMethod 

38 

39 

40class ApPipeConfig(pexConfig.Config): 

41 """Settings and defaults for ApPipeTask. 

42 """ 

43 

44 ccdProcessor = pexConfig.ConfigurableField( 

45 target=ProcessCcdTask, 

46 doc="Task used to perform basic image reduction and characterization.", 

47 ) 

48 differencer = pexConfig.ConfigurableField( 

49 target=ImageDifferenceTask, 

50 doc="Task used to do image subtraction and DiaSource detection.", 

51 ) 

52 transformDiaSrcCat = pexConfig.ConfigurableField( 

53 target=TransformDiaSourceCatalogTask, 

54 doc="Task for converting and calibrating the afw SourceCatalog of " 

55 "DiaSources to Pandas DataFrame for use in Association." 

56 ) 

57 diaPipe = pexConfig.ConfigurableField( 

58 target=DiaPipelineTask, 

59 doc="Pipeline task for loading/store DiaSources and DiaObjects and " 

60 "spatially associating them.", 

61 ) 

62 

63 def setDefaults(self): 

64 """Settings appropriate for most or all ap_pipe runs. 

65 """ 

66 

67 # Write the WarpedExposure to disk for use in Alert Packet creation. 

68 self.differencer.doWriteWarpedExp = True 

69 

70 def validate(self): 

71 pexConfig.Config.validate(self) 

72 if not self.ccdProcessor.calibrate.doWrite or not self.ccdProcessor.calibrate.doWriteExposure: 

73 raise ValueError("Differencing needs calexps [ccdProcessor.calibrate.doWrite, doWriteExposure]") 

74 if not self.differencer.doMeasurement: 

75 raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].") 

76 if not self.differencer.doWriteWarpedExp: 

77 raise ValueError("Alert generation needs warped exposures [differencer.doWriteWarpedExp].") 

78 if not self.differencer.doWriteSources: 

79 raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].") 

80 if not self.differencer.doWriteSubtractedExp: 

81 raise ValueError("Source association needs difference exposures " 

82 "[differencer.doWriteSubtractedExp].") 

83 

84 

85class ApPipeTask(pipeBase.CmdLineTask): 

86 """Command-line task representing the entire AP pipeline. 

87 

88 ``ApPipeTask`` processes raw DECam images from basic processing through 

89 source association. Other observatories will be supported in the future. 

90 

91 ``ApPipeTask`` can be run from the command line, but it can also be called 

92 from other pipeline code. It provides public methods for executing each 

93 major step of the pipeline by itself. 

94 

95 Parameters 

96 ---------- 

97 butler : `lsst.daf.persistence.Butler` 

98 A Butler providing access to the science, calibration, and (unless 

99 ``config.differencer.getTemplate`` is overridden) template data to 

100 be processed. Its output repository must be both readable 

101 and writable. 

102 """ 

103 

104 ConfigClass = ApPipeConfig 

105 RunnerClass = ApPipeTaskRunner 

106 _DefaultName = "apPipe" 

107 

108 def __init__(self, butler, *args, **kwargs): 

109 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

110 

111 self.makeSubtask("ccdProcessor", butler=butler) 

112 self.makeSubtask("differencer", butler=butler) 

113 self.makeSubtask("transformDiaSrcCat", initInputs={"diaSourceSchema": self.differencer.outputSchema}) 

114 self.makeSubtask("diaPipe") 

115 

116 @timeMethod 

117 def runDataRef(self, rawRef, templateIds=None, reuse=None): 

118 """Execute the ap_pipe pipeline on a single image. 

119 

120 Parameters 

121 ---------- 

122 rawRef : `lsst.daf.persistence.ButlerDataRef` 

123 A reference to the raw data to process. 

124 templateIds : `list` of `dict`, optional 

125 A list of parsed data IDs for templates to use. Only used if 

126 ``config.differencer`` is configured to do so. ``differencer`` or 

127 its subtasks may restrict the allowed IDs. 

128 reuse : `list` of `str`, optional 

129 The names of all subtasks that may be skipped if their output is 

130 present. Defaults to skipping nothing. 

131 

132 Returns 

133 ------- 

134 result : `lsst.pipe.base.Struct` 

135 Result struct with components: 

136 

137 - l1Database : handle for accessing the final association database, conforming to 

138 `ap_association`'s DB access API 

139 - ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

140 - differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

141 - diaPipe : output of `config.diaPipe.run` (`lsst.pipe.base.Struct` or `None`). 

142 """ 

143 if reuse is None: 

144 reuse = [] 

145 # Work around mismatched HDU lists for raw and processed data 

146 calexpId = rawRef.dataId.copy() 

147 if 'hdu' in calexpId: 

148 del calexpId['hdu'] 

149 calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId) 

150 

151 # Ensure that templateIds make it through basic data reduction 

152 # TODO: treat as independent jobs (may need SuperTask framework?) 

153 if templateIds is not None: 

154 for templateId in templateIds: 

155 # templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef 

156 rawTemplateRef = _siblingRef(rawRef, "raw", templateId) 

157 calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId) 

158 if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp"): 

159 self.runProcessCcd(rawTemplateRef) 

160 

161 if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp"): 

162 self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId)) 

163 processResults = None 

164 else: 

165 processResults = self.runProcessCcd(rawRef) 

166 

167 diffType = self.config.differencer.coaddName 

168 if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc"): 

169 self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId)) 

170 diffImResults = None 

171 else: 

172 diffImResults = self.runDiffIm(calexpRef, templateIds) 

173 

174 if "diaPipe" in reuse: 

175 warnings.warn( 

176 "Reusing association results for some images while rerunning " 

177 "others may change the associations. If exact reproducibility " 

178 "matters, please clear the association database and run " 

179 "ap_pipe.py with --reuse-output-from=differencer to redo all " 

180 "association results consistently.") 

181 if "diaPipe" in reuse and calexpRef.datasetExists("apdb_marker"): 

182 message = "DiaPipeline has already been run for {0}, skipping...".format(calexpRef.dataId) 

183 self.log.info(message) 

184 diaPipeResults = None 

185 else: 

186 diaPipeResults = self.runAssociation(calexpRef) 

187 

188 return pipeBase.Struct( 

189 l1Database=self.diaPipe.apdb, 

190 ccdProcessor=processResults if processResults else None, 

191 differencer=diffImResults if diffImResults else None, 

192 diaPipe=diaPipeResults.taskResults if diaPipeResults else None 

193 ) 

194 

195 @timeMethod 

196 def runProcessCcd(self, sensorRef): 

197 """Perform ISR with ingested images and calibrations via processCcd. 

198 

199 The output repository associated with ``sensorRef`` will be populated with the 

200 usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR). 

201 

202 Parameters 

203 ---------- 

204 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

205 Data reference for raw data. 

206 

207 Returns 

208 ------- 

209 result : `lsst.pipe.base.Struct` 

210 Output of `config.ccdProcessor.runDataRef`. 

211 

212 Notes 

213 ----- 

214 The input repository corresponding to ``sensorRef`` must already contain the refcats. 

215 """ 

216 self.log.info("Running ProcessCcd...") 

217 return self.ccdProcessor.runDataRef(sensorRef) 

218 

219 @timeMethod 

220 def runDiffIm(self, sensorRef, templateIds=None): 

221 """Do difference imaging with a template and a science image 

222 

223 The output repository associated with ``sensorRef`` will be populated with difference images 

224 and catalogs of detected sources (diaSrc, diffexp, and metadata files) 

225 

226 Parameters 

227 ---------- 

228 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

229 Data reference for multiple dataset types, both input and output. 

230 templateIds : `list` of `dict`, optional 

231 A list of parsed data IDs for templates to use. Only used if 

232 ``config.differencer`` is configured to do so. ``differencer`` or 

233 its subtasks may restrict the allowed IDs. 

234 

235 Returns 

236 ------- 

237 result : `lsst.pipe.base.Struct` 

238 Output of `config.differencer.runDataRef`. 

239 """ 

240 self.log.info("Running ImageDifference...") 

241 return self.differencer.runDataRef(sensorRef, templateIdList=templateIds) 

242 

243 @timeMethod 

244 def runAssociation(self, sensorRef): 

245 """Do source association. 

246 

247 This method writes an ``apdb_marker`` dataset once all changes related 

248 to the current exposure have been committed. 

249 

250 Parameters 

251 ---------- 

252 sensorRef : `lsst.daf.persistence.ButlerDataRef` 

253 Data reference for multiple input dataset types. 

254 

255 Returns 

256 ------- 

257 result : `lsst.pipe.base.Struct` 

258 Result struct with components: 

259 

260 - apdb : `lsst.dax.apdb.Apdb` Initialized association database containing final association 

261 results. 

262 - taskResults : output of `config.diaPipe.run` (`lsst.pipe.base.Struct`). 

263 """ 

264 diffType = self.config.differencer.coaddName 

265 diffIm = sensorRef.get(diffType + "Diff_differenceExp") 

266 

267 transformResult = self.transformDiaSrcCat.run( 

268 diaSourceCat=sensorRef.get(diffType + "Diff_diaSrc"), 

269 diffIm=diffIm, 

270 band=diffIm.getFilter().bandLabel, 

271 ccdVisitId=diffIm.getInfo().getVisitInfo().getExposureId()) 

272 

273 results = self.diaPipe.run( 

274 diaSourceTable=transformResult.diaSourceTable, 

275 solarSystemObjectTable=None, 

276 diffIm=sensorRef.get(diffType + "Diff_differenceExp"), 

277 exposure=sensorRef.get("calexp"), 

278 template=sensorRef.get(diffType + "Diff_warpedExp"), 

279 ccdExposureIdBits=sensorRef.get("ccdExposureId_bits"), 

280 band=diffIm.getFilter().bandLabel, 

281 ) 

282 

283 # apdb_marker triggers metrics processing; let them try to read 

284 # something even if association failed 

285 sensorRef.put(results.apdbMarker, "apdb_marker") 

286 

287 return pipeBase.Struct( 

288 l1Database=self.diaPipe.apdb, 

289 taskResults=results 

290 ) 

291 

292 @classmethod 

293 def _makeArgumentParser(cls): 

294 """A parser that can handle extra arguments for ap_pipe. 

295 """ 

296 return ApPipeParser(name=cls._DefaultName) 

297 

298 

299def _siblingRef(original, datasetType, dataId): 

300 """Construct a new dataRef using an existing dataRef as a template. 

301 

302 The typical application is to construct a data ID that differs from an 

303 existing ID in one or two keys, but is more specific than expanding a 

304 partial data ID would be. 

305 

306 Parameters 

307 ---------- 

308 original : `lsst.daf.persistence.ButlerDataRef` 

309 A dataRef related to the desired one. Assumed to represent a unique dataset. 

310 datasetType : `str` 

311 The desired type of the new dataRef. Must be compatible 

312 with ``original``. 

313 dataId : `dict` from `str` to any 

314 A possibly partial data ID for the new dataRef. Any properties left 

315 unspecified shall be copied from ``original``. 

316 

317 Returns 

318 ------- 

319 dataRef : `lsst.daf.persistence.ButlerDataRef` 

320 A dataRef to the same butler as ``original``, but of type 

321 ``datasetType`` and with data ID equivalent to 

322 ``original.dataId.update(dataId)``. 

323 """ 

324 butler = original.getButler() 

325 return butler.dataRef(datasetType, dataId=original.dataId, **dataId)