Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 make_dia_object_schema, 

44 make_dia_source_schema, 

45 PackageAlertsTask) 

46 

47__all__ = ("DiaPipelineConfig", 

48 "DiaPipelineTask", 

49 "DiaPipelineConnections") 

50 

51 

52class DiaPipelineConnections( 

53 pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceTable = connTypes.Input( 

59 doc="Catalog of calibrated DiaSources.", 

60 name="{fakesType}{coaddName}Diff_diaSrcTable", 

61 storageClass="DataFrame", 

62 dimensions=("instrument", "visit", "detector"), 

63 ) 

64 diffIm = connTypes.Input( 

65 doc="Difference image on which the DiaSources were detected.", 

66 name="{fakesType}{coaddName}Diff_differenceExp", 

67 storageClass="ExposureF", 

68 dimensions=("instrument", "visit", "detector"), 

69 ) 

70 exposure = connTypes.Input( 

71 doc="Calibrated exposure differenced with a template image during " 

72 "image differencing.", 

73 name="calexp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 warpedExposure = connTypes.Input( 

78 doc="Warped template used to create `subtractedExposure`. Not PSF " 

79 "matched.", 

80 dimensions=("instrument", "visit", "detector"), 

81 storageClass="ExposureF", 

82 name="{fakesType}{coaddName}Diff_warpedExp", 

83 ) 

84 apdbMarker = connTypes.Output( 

85 doc="Marker dataset storing the configuration of the Apdb for each " 

86 "visit/detector. Used to signal the completion of the pipeline.", 

87 name="apdb_marker", 

88 storageClass="Config", 

89 dimensions=("instrument", "visit", "detector"), 

90 ) 

91 associatedDiaSources = connTypes.Output( 

92 doc="Optional output storing the DiaSource catalog after matching, " 

93 "calibration, and standardization for insertation into the Apdb.", 

94 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

95 storageClass="DataFrame", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 

99 def __init__(self, *, config=None): 

100 super().__init__(config=config) 

101 

102 if not config.doWriteAssociatedSources: 

103 self.outputs.remove("associatedDiaSources") 

104 

105 def adjustQuantum(self, inputs, outputs, label, dataId): 

106 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

107 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

108 of the activator. 

109 

110 This implementation checks to make sure that the filters in the dataset 

111 are compatible with AP processing as set by the Apdb/DPDD schema. 

112 

113 Parameters 

114 ---------- 

115 inputs : `dict` 

116 Dictionary whose keys are an input (regular or prerequisite) 

117 connection name and whose values are a tuple of the connection 

118 instance and a collection of associated `DatasetRef` objects. 

119 The exact type of the nested collections is unspecified; it can be 

120 assumed to be multi-pass iterable and support `len` and ``in``, but 

121 it should not be mutated in place. In contrast, the outer 

122 dictionaries are guaranteed to be temporary copies that are true 

123 `dict` instances, and hence may be modified and even returned; this 

124 is especially useful for delegating to `super` (see notes below). 

125 outputs : `dict` 

126 Dict of output datasets, with the same structure as ``inputs``. 

127 label : `str` 

128 Label for this task in the pipeline (should be used in all 

129 diagnostic messages). 

130 data_id : `lsst.daf.butler.DataCoordinate` 

131 Data ID for this quantum in the pipeline (should be used in all 

132 diagnostic messages). 

133 

134 Returns 

135 ------- 

136 adjusted_inputs : `dict` 

137 Dict of the same form as ``inputs`` with updated containers of 

138 input `DatasetRef` objects. Connections that are not changed 

139 should not be returned at all. Datasets may only be removed, not 

140 added. Nested collections may be of any multi-pass iterable type, 

141 and the order of iteration will set the order of iteration within 

142 `PipelineTask.runQuantum`. 

143 adjusted_outputs : `dict` 

144 Dict of updated output datasets, with the same structure and 

145 interpretation as ``adjusted_inputs``. 

146 

147 Raises 

148 ------ 

149 ScalarError 

150 Raised if any `Input` or `PrerequisiteInput` connection has 

151 ``multiple`` set to `False`, but multiple datasets. 

152 NoWorkFound 

153 Raised to indicate that this quantum should not be run; not enough 

154 datasets were found for a regular `Input` connection, and the 

155 quantum should be pruned or skipped. 

156 FileNotFoundError 

157 Raised to cause QuantumGraph generation to fail (with the message 

158 included in this exception); not enough datasets were found for a 

159 `PrerequisiteInput` connection. 

160 """ 

161 _, refs = inputs["diffIm"] 

162 for ref in refs: 

163 if ref.dataId["band"] not in self.config.validBands: 

164 raise ValueError( 

165 f"Requested '{ref.dataId['band']}' not in " 

166 "DiaPipelineConfig.validBands. To process bands not in " 

167 "the standard Rubin set (ugrizy) you must add the band to " 

168 "the validBands list in DiaPipelineConfig and add the " 

169 "appropriate columns to the Apdb schema.") 

170 return super().adjustQuantum(inputs, outputs, label, dataId) 

171 

172 

173class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

174 pipelineConnections=DiaPipelineConnections): 

175 """Config for DiaPipelineTask. 

176 """ 

177 coaddName = pexConfig.Field( 

178 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

179 dtype=str, 

180 default="deep", 

181 ) 

182 apdb = pexConfig.ConfigurableField( 

183 target=daxApdb.Apdb, 

184 ConfigClass=daxApdb.ApdbConfig, 

185 doc="Database connection for storing associated DiaSources and " 

186 "DiaObjects. Must already be initialized.", 

187 ) 

188 validBands = pexConfig.ListField( 

189 dtype=str, 

190 default=["u", "g", "r", "i", "z", "y"], 

191 doc="List of bands that are valid for AP processing. To process a " 

192 "band not on this list, the appropriate band specific columns " 

193 "must be added to the Apdb schema in dax_apdb.", 

194 ) 

195 diaCatalogLoader = pexConfig.ConfigurableField( 

196 target=LoadDiaCatalogsTask, 

197 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

198 ) 

199 associator = pexConfig.ConfigurableField( 

200 target=AssociationTask, 

201 doc="Task used to associate DiaSources with DiaObjects.", 

202 ) 

203 diaForcedSource = pexConfig.ConfigurableField( 

204 target=DiaForcedSourceTask, 

205 doc="Task used for force photometer DiaObject locations in direct and " 

206 "difference images.", 

207 ) 

208 alertPackager = pexConfig.ConfigurableField( 

209 target=PackageAlertsTask, 

210 doc="Subtask for packaging Ap data into alerts.", 

211 ) 

212 doPackageAlerts = pexConfig.Field( 

213 dtype=bool, 

214 default=False, 

215 doc="Package Dia-data into serialized alerts for distribution and " 

216 "write them to disk.", 

217 ) 

218 doWriteAssociatedSources = pexConfig.Field( 

219 dtype=bool, 

220 default=False, 

221 doc="Write out associated and SDMed DiaSources.", 

222 ) 

223 

224 def setDefaults(self): 

225 self.apdb.dia_object_index = "baseline" 

226 self.apdb.dia_object_columns = [] 

227 self.apdb.extra_schema_file = os.path.join( 

228 getPackageDir("ap_association"), 

229 "data", 

230 "apdb-ap-pipe-schema-extra.yaml") 

231 

232 def validate(self): 

233 pexConfig.Config.validate(self) 

234 if self.diaCatalogLoader.htmLevel != \ 

235 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

236 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

237 "equal to HTMIndexDiaCalculationPlugin index " 

238 "level.") 

239 

240 

241class DiaPipelineTask(pipeBase.PipelineTask): 

242 """Task for loading, associating and storing Difference Image Analysis 

243 (DIA) Objects and Sources. 

244 """ 

245 ConfigClass = DiaPipelineConfig 

246 _DefaultName = "diaPipe" 

247 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

248 

249 def __init__(self, initInputs=None, **kwargs): 

250 super().__init__(**kwargs) 

251 self.apdb = self.config.apdb.apply( 

252 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

253 DiaSource=make_dia_source_schema())) 

254 self.makeSubtask("diaCatalogLoader") 

255 self.makeSubtask("associator") 

256 self.makeSubtask("diaForcedSource") 

257 if self.config.doPackageAlerts: 

258 self.makeSubtask("alertPackager") 

259 

260 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

261 inputs = butlerQC.get(inputRefs) 

262 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

263 returnMaxBits=True) 

264 inputs["ccdExposureIdBits"] = expBits 

265 

266 outputs = self.run(**inputs) 

267 

268 butlerQC.put(outputs, outputRefs) 

269 

270 @pipeBase.timeMethod 

271 def run(self, 

272 diaSourceTable, 

273 diffIm, 

274 exposure, 

275 warpedExposure, 

276 ccdExposureIdBits): 

277 """Process DiaSources and DiaObjects. 

278 

279 Load previous DiaObjects and their DiaSource history. Calibrate the 

280 values in the diaSourceCat. Associate new DiaSources with previous 

281 DiaObjects. Run forced photometry at the updated DiaObject locations. 

282 Store the results in the Alert Production Database (Apdb). 

283 

284 Parameters 

285 ---------- 

286 diaSourceTable : `pandas.DataFrame` 

287 Newly detected DiaSources. 

288 diffIm : `lsst.afw.image.ExposureF` 

289 Difference image exposure in which the sources in ``diaSourceCat`` 

290 were detected. 

291 exposure : `lsst.afw.image.ExposureF` 

292 Calibrated exposure differenced with a template to create 

293 ``diffIm``. 

294 warpedExposure : `lsst.afw.image.ExposureF` 

295 Template exposure used to create diffIm. 

296 ccdExposureIdBits : `int` 

297 Number of bits used for a unique ``ccdVisitId``. 

298 

299 Returns 

300 ------- 

301 results : `lsst.pipe.base.Struct` 

302 Results struct with components. 

303 

304 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

305 that this ccdVisit has completed successfully. 

306 (`lsst.dax.apdb.ApdbConfig`) 

307 """ 

308 self.log.info("Running DiaPipeline...") 

309 # Put the SciencePipelines through a SDMification step and return 

310 # calibrated columns with the expect output database names. 

311 

312 # Load the DiaObjects and DiaSource history. 

313 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

314 

315 # Associate new DiaSources with existing DiaObjects and update 

316 # DiaObject summary statistics using the full DiaSource history. 

317 assocResults = self.associator.run(diaSourceTable, 

318 loaderResult.diaObjects, 

319 loaderResult.diaSources) 

320 

321 # Force photometer on the Difference and Calibrated exposures using 

322 # the new and updated DiaObject locations. 

323 diaForcedSources = self.diaForcedSource.run( 

324 assocResults.diaObjects, 

325 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

326 ccdExposureIdBits, 

327 exposure, 

328 diffIm) 

329 

330 # Store DiaSources and updated DiaObjects in the Apdb. 

331 self.apdb.storeDiaSources(assocResults.diaSources) 

332 self.apdb.storeDiaObjects( 

333 assocResults.updatedDiaObjects, 

334 exposure.getInfo().getVisitInfo().getDate().toPython()) 

335 self.apdb.storeDiaForcedSources(diaForcedSources) 

336 

337 if self.config.doPackageAlerts: 

338 if len(loaderResult.diaForcedSources) > 1: 

339 diaForcedSources = diaForcedSources.append( 

340 loaderResult.diaForcedSources, 

341 sort=True) 

342 if diaForcedSources.index.has_duplicates: 

343 self.log.warn( 

344 "Duplicate DiaForcedSources created after merge with " 

345 "history and new sources. This may cause downstream " 

346 "problems. Dropping duplicates.") 

347 # Drop duplicates via index and keep the first appearance. 

348 # Reset due to the index shape being slight different than 

349 # expected. 

350 diaForcedSources = diaForcedSources.groupby( 

351 diaForcedSources.index).first() 

352 diaForcedSources.reset_index(drop=True, inplace=True) 

353 diaForcedSources.set_index( 

354 ["diaObjectId", "diaForcedSourceId"], 

355 drop=False, 

356 inplace=True) 

357 self.alertPackager.run(assocResults.diaSources, 

358 assocResults.diaObjects, 

359 loaderResult.diaSources, 

360 diaForcedSources, 

361 diffIm, 

362 warpedExposure, 

363 ccdExposureIdBits) 

364 

365 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

366 associatedDiaSources=assocResults.diaSources)