Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37 

38from lsst.ap.association import ( 

39 AssociationTask, 

40 DiaForcedSourceTask, 

41 LoadDiaCatalogsTask, 

42 make_dia_object_schema, 

43 make_dia_source_schema, 

44 PackageAlertsTask) 

45 

46__all__ = ("DiaPipelineConfig", 

47 "DiaPipelineTask", 

48 "DiaPipelineConnections") 

49 

50 

51class DiaPipelineConnections( 

52 pipeBase.PipelineTaskConnections, 

53 dimensions=("instrument", "visit", "detector"), 

54 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

55 """Butler connections for DiaPipelineTask. 

56 """ 

57 diaSourceTable = connTypes.Input( 

58 doc="Catalog of calibrated DiaSources.", 

59 name="{fakesType}{coaddName}Diff_diaSrcTable", 

60 storageClass="DataFrame", 

61 dimensions=("instrument", "visit", "detector"), 

62 ) 

63 diffIm = connTypes.Input( 

64 doc="Difference image on which the DiaSources were detected.", 

65 name="{fakesType}{coaddName}Diff_differenceExp", 

66 storageClass="ExposureF", 

67 dimensions=("instrument", "visit", "detector"), 

68 ) 

69 exposure = connTypes.Input( 

70 doc="Calibrated exposure differenced with a template image during " 

71 "image differencing.", 

72 name="calexp", 

73 storageClass="ExposureF", 

74 dimensions=("instrument", "visit", "detector"), 

75 ) 

76 warpedExposure = connTypes.Input( 

77 doc="Warped template used to create `subtractedExposure`. Not PSF " 

78 "matched.", 

79 dimensions=("instrument", "visit", "detector"), 

80 storageClass="ExposureF", 

81 name="{fakesType}{coaddName}Diff_warpedExp", 

82 ) 

83 apdbMarker = connTypes.Output( 

84 doc="Marker dataset storing the configuration of the Apdb for each " 

85 "visit/detector. Used to signal the completion of the pipeline.", 

86 name="apdb_marker", 

87 storageClass="Config", 

88 dimensions=("instrument", "visit", "detector"), 

89 ) 

90 associatedDiaSources = connTypes.Output( 

91 doc="Optional output storing the DiaSource catalog after matching, " 

92 "calibration, and standardization for insertation into the Apdb.", 

93 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

94 storageClass="DataFrame", 

95 dimensions=("instrument", "visit", "detector"), 

96 ) 

97 

98 def __init__(self, *, config=None): 

99 super().__init__(config=config) 

100 

101 if not config.doWriteAssociatedSources: 

102 self.outputs.remove("associatedDiaSources") 

103 

104 def adjustQuantum(self, inputs, outputs, label, dataId): 

105 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

106 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

107 of the activator. 

108 

109 This implementation checks to make sure that the filters in the dataset 

110 are compatible with AP processing as set by the Apdb/DPDD schema. 

111 

112 Parameters 

113 ---------- 

114 inputs : `dict` 

115 Dictionary whose keys are an input (regular or prerequisite) 

116 connection name and whose values are a tuple of the connection 

117 instance and a collection of associated `DatasetRef` objects. 

118 The exact type of the nested collections is unspecified; it can be 

119 assumed to be multi-pass iterable and support `len` and ``in``, but 

120 it should not be mutated in place. In contrast, the outer 

121 dictionaries are guaranteed to be temporary copies that are true 

122 `dict` instances, and hence may be modified and even returned; this 

123 is especially useful for delegating to `super` (see notes below). 

124 outputs : `dict` 

125 Dict of output datasets, with the same structure as ``inputs``. 

126 label : `str` 

127 Label for this task in the pipeline (should be used in all 

128 diagnostic messages). 

129 data_id : `lsst.daf.butler.DataCoordinate` 

130 Data ID for this quantum in the pipeline (should be used in all 

131 diagnostic messages). 

132 

133 Returns 

134 ------- 

135 adjusted_inputs : `dict` 

136 Dict of the same form as ``inputs`` with updated containers of 

137 input `DatasetRef` objects. Connections that are not changed 

138 should not be returned at all. Datasets may only be removed, not 

139 added. Nested collections may be of any multi-pass iterable type, 

140 and the order of iteration will set the order of iteration within 

141 `PipelineTask.runQuantum`. 

142 adjusted_outputs : `dict` 

143 Dict of updated output datasets, with the same structure and 

144 interpretation as ``adjusted_inputs``. 

145 

146 Raises 

147 ------ 

148 ScalarError 

149 Raised if any `Input` or `PrerequisiteInput` connection has 

150 ``multiple`` set to `False`, but multiple datasets. 

151 NoWorkFound 

152 Raised to indicate that this quantum should not be run; not enough 

153 datasets were found for a regular `Input` connection, and the 

154 quantum should be pruned or skipped. 

155 FileNotFoundError 

156 Raised to cause QuantumGraph generation to fail (with the message 

157 included in this exception); not enough datasets were found for a 

158 `PrerequisiteInput` connection. 

159 """ 

160 _, refs = inputs["diffIm"] 

161 for ref in refs: 

162 if ref.dataId["band"] not in self.config.validBands: 

163 raise ValueError( 

164 f"Requested '{ref.dataId['band']}' not in " 

165 "DiaPipelineConfig.validBands. To process bands not in " 

166 "the standard Rubin set (ugrizy) you must add the band to " 

167 "the validBands list in DiaPipelineConfig and add the " 

168 "appropriate columns to the Apdb schema.") 

169 return super().adjustQuantum(inputs, outputs, label, dataId) 

170 

171 

172class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

173 pipelineConnections=DiaPipelineConnections): 

174 """Config for DiaPipelineTask. 

175 """ 

176 coaddName = pexConfig.Field( 

177 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

178 dtype=str, 

179 default="deep", 

180 ) 

181 apdb = pexConfig.ConfigurableField( 

182 target=daxApdb.Apdb, 

183 ConfigClass=daxApdb.ApdbConfig, 

184 doc="Database connection for storing associated DiaSources and " 

185 "DiaObjects. Must already be initialized.", 

186 ) 

187 validBands = pexConfig.ListField( 

188 dtype=str, 

189 default=["u", "g", "r", "i", "z", "y"], 

190 doc="List of bands that are valid for AP processing. To process a " 

191 "band not on this list, the appropriate band specific columns " 

192 "must be added to the Apdb schema in dax_apdb.", 

193 ) 

194 diaCatalogLoader = pexConfig.ConfigurableField( 

195 target=LoadDiaCatalogsTask, 

196 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

197 ) 

198 associator = pexConfig.ConfigurableField( 

199 target=AssociationTask, 

200 doc="Task used to associate DiaSources with DiaObjects.", 

201 ) 

202 diaForcedSource = pexConfig.ConfigurableField( 

203 target=DiaForcedSourceTask, 

204 doc="Task used for force photometer DiaObject locations in direct and " 

205 "difference images.", 

206 ) 

207 alertPackager = pexConfig.ConfigurableField( 

208 target=PackageAlertsTask, 

209 doc="Subtask for packaging Ap data into alerts.", 

210 ) 

211 doPackageAlerts = pexConfig.Field( 

212 dtype=bool, 

213 default=False, 

214 doc="Package Dia-data into serialized alerts for distribution and " 

215 "write them to disk.", 

216 ) 

217 doWriteAssociatedSources = pexConfig.Field( 

218 dtype=bool, 

219 default=False, 

220 doc="Write out associated and SDMed DiaSources.", 

221 ) 

222 

223 def setDefaults(self): 

224 self.apdb.dia_object_index = "baseline" 

225 self.apdb.dia_object_columns = [] 

226 self.apdb.extra_schema_file = os.path.join( 

227 "${AP_ASSOCIATION_DIR}", 

228 "data", 

229 "apdb-ap-pipe-schema-extra.yaml") 

230 

231 def validate(self): 

232 pexConfig.Config.validate(self) 

233 if self.diaCatalogLoader.htmLevel != \ 

234 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

235 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

236 "equal to HTMIndexDiaCalculationPlugin index " 

237 "level.") 

238 

239 

240class DiaPipelineTask(pipeBase.PipelineTask): 

241 """Task for loading, associating and storing Difference Image Analysis 

242 (DIA) Objects and Sources. 

243 """ 

244 ConfigClass = DiaPipelineConfig 

245 _DefaultName = "diaPipe" 

246 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

247 

248 def __init__(self, initInputs=None, **kwargs): 

249 super().__init__(**kwargs) 

250 self.apdb = self.config.apdb.apply( 

251 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

252 DiaSource=make_dia_source_schema())) 

253 self.makeSubtask("diaCatalogLoader") 

254 self.makeSubtask("associator") 

255 self.makeSubtask("diaForcedSource") 

256 if self.config.doPackageAlerts: 

257 self.makeSubtask("alertPackager") 

258 

259 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

260 inputs = butlerQC.get(inputRefs) 

261 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

262 returnMaxBits=True) 

263 inputs["ccdExposureIdBits"] = expBits 

264 

265 outputs = self.run(**inputs) 

266 

267 butlerQC.put(outputs, outputRefs) 

268 

269 @pipeBase.timeMethod 

270 def run(self, 

271 diaSourceTable, 

272 diffIm, 

273 exposure, 

274 warpedExposure, 

275 ccdExposureIdBits): 

276 """Process DiaSources and DiaObjects. 

277 

278 Load previous DiaObjects and their DiaSource history. Calibrate the 

279 values in the diaSourceCat. Associate new DiaSources with previous 

280 DiaObjects. Run forced photometry at the updated DiaObject locations. 

281 Store the results in the Alert Production Database (Apdb). 

282 

283 Parameters 

284 ---------- 

285 diaSourceTable : `pandas.DataFrame` 

286 Newly detected DiaSources. 

287 diffIm : `lsst.afw.image.ExposureF` 

288 Difference image exposure in which the sources in ``diaSourceCat`` 

289 were detected. 

290 exposure : `lsst.afw.image.ExposureF` 

291 Calibrated exposure differenced with a template to create 

292 ``diffIm``. 

293 warpedExposure : `lsst.afw.image.ExposureF` 

294 Template exposure used to create diffIm. 

295 ccdExposureIdBits : `int` 

296 Number of bits used for a unique ``ccdVisitId``. 

297 

298 Returns 

299 ------- 

300 results : `lsst.pipe.base.Struct` 

301 Results struct with components. 

302 

303 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

304 that this ccdVisit has completed successfully. 

305 (`lsst.dax.apdb.ApdbConfig`) 

306 """ 

307 self.log.info("Running DiaPipeline...") 

308 # Put the SciencePipelines through a SDMification step and return 

309 # calibrated columns with the expect output database names. 

310 

311 # Load the DiaObjects and DiaSource history. 

312 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

313 

314 # Associate new DiaSources with existing DiaObjects and update 

315 # DiaObject summary statistics using the full DiaSource history. 

316 assocResults = self.associator.run(diaSourceTable, 

317 loaderResult.diaObjects, 

318 loaderResult.diaSources) 

319 

320 # Force photometer on the Difference and Calibrated exposures using 

321 # the new and updated DiaObject locations. 

322 diaForcedSources = self.diaForcedSource.run( 

323 assocResults.diaObjects, 

324 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

325 ccdExposureIdBits, 

326 exposure, 

327 diffIm) 

328 

329 # Store DiaSources and updated DiaObjects in the Apdb. 

330 self.apdb.storeDiaSources(assocResults.diaSources) 

331 self.apdb.storeDiaObjects( 

332 assocResults.updatedDiaObjects, 

333 exposure.getInfo().getVisitInfo().getDate().toPython()) 

334 self.apdb.storeDiaForcedSources(diaForcedSources) 

335 

336 if self.config.doPackageAlerts: 

337 if len(loaderResult.diaForcedSources) > 1: 

338 diaForcedSources = diaForcedSources.append( 

339 loaderResult.diaForcedSources, 

340 sort=True) 

341 if diaForcedSources.index.has_duplicates: 

342 self.log.warn( 

343 "Duplicate DiaForcedSources created after merge with " 

344 "history and new sources. This may cause downstream " 

345 "problems. Dropping duplicates.") 

346 # Drop duplicates via index and keep the first appearance. 

347 # Reset due to the index shape being slight different than 

348 # expected. 

349 diaForcedSources = diaForcedSources.groupby( 

350 diaForcedSources.index).first() 

351 diaForcedSources.reset_index(drop=True, inplace=True) 

352 diaForcedSources.set_index( 

353 ["diaObjectId", "diaForcedSourceId"], 

354 drop=False, 

355 inplace=True) 

356 self.alertPackager.run(assocResults.diaSources, 

357 assocResults.diaObjects, 

358 loaderResult.diaSources, 

359 diaForcedSources, 

360 diffIm, 

361 warpedExposure, 

362 ccdExposureIdBits) 

363 

364 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

365 associatedDiaSources=assocResults.diaSources)