Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 make_dia_object_schema, 

44 make_dia_source_schema, 

45 PackageAlertsTask) 

46 

47__all__ = ("DiaPipelineConfig", 

48 "DiaPipelineTask", 

49 "DiaPipelineConnections") 

50 

51 

52class DiaPipelineConnections( 

53 pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceTable = connTypes.Input( 

59 doc="Catalog of calibrated DiaSources.", 

60 name="{fakesType}{coaddName}Diff_diaSrcTable", 

61 storageClass="DataFrame", 

62 dimensions=("instrument", "visit", "detector"), 

63 ) 

64 diffIm = connTypes.Input( 

65 doc="Difference image on which the DiaSources were detected.", 

66 name="{fakesType}{coaddName}Diff_differenceExp", 

67 storageClass="ExposureF", 

68 dimensions=("instrument", "visit", "detector"), 

69 ) 

70 exposure = connTypes.Input( 

71 doc="Calibrated exposure differenced with a template image during " 

72 "image differencing.", 

73 name="calexp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 warpedExposure = connTypes.Input( 

78 doc="Warped template used to create `subtractedExposure`. Not PSF " 

79 "matched.", 

80 dimensions=("instrument", "visit", "detector"), 

81 storageClass="ExposureF", 

82 name="{fakesType}{coaddName}Diff_warpedExp", 

83 ) 

84 apdbMarker = connTypes.Output( 

85 doc="Marker dataset storing the configuration of the Apdb for each " 

86 "visit/detector. Used to signal the completion of the pipeline.", 

87 name="apdb_marker", 

88 storageClass="Config", 

89 dimensions=("instrument", "visit", "detector"), 

90 ) 

91 associatedDiaSources = connTypes.Output( 

92 doc="Optional output storing the DiaSource catalog after matching, " 

93 "calibration, and standardization for insertation into the Apdb.", 

94 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

95 storageClass="DataFrame", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 

99 def __init__(self, *, config=None): 

100 super().__init__(config=config) 

101 

102 if not config.doWriteAssociatedSources: 

103 self.outputs.remove("associatedDiaSources") 

104 

105 def adjustQuantum(self, datasetRefMap: pipeBase.InputQuantizedConnection): 

106 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

107 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

108 of the activator. 

109 

110 This implementation checks to make sure that the filters in the dataset 

111 are compatible with AP processing as set by the Apdb/DPDD schema. 

112 

113 Parameters 

114 ---------- 

115 datasetRefMap : `NamedKeyDict` 

116 Mapping from dataset type to a `set` of 

117 `lsst.daf.butler.DatasetRef` objects 

118 

119 Returns 

120 ------- 

121 datasetRefMap : `NamedKeyDict` 

122 Mapping of input with assurances that bands incompatible with the 

123 Apdb are present. 

124 

125 Raises 

126 ------ 

127 ValueError 

128 Raises if a data ref in the quantum has a band not available in the 

129 Apdb. 

130 """ 

131 refs = datasetRefMap[self.diffIm.name] 

132 for ref in refs: 

133 if ref.dataId["band"] not in self.config.validBands: 

134 raise ValueError( 

135 f"Requested '{ref.dataId['band']}' not in " 

136 "DiaPipelineConfig.validBands. To process bands not in " 

137 "the standard Rubin set (ugrizy) you must add the band to " 

138 "the validBands list in DiaPipelineConfig and add the " 

139 "appropriate columns to the Apdb schema.") 

140 return super().adjustQuantum(datasetRefMap) 

141 

142 

143class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

144 pipelineConnections=DiaPipelineConnections): 

145 """Config for DiaPipelineTask. 

146 """ 

147 coaddName = pexConfig.Field( 

148 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

149 dtype=str, 

150 default="deep", 

151 ) 

152 apdb = pexConfig.ConfigurableField( 

153 target=daxApdb.Apdb, 

154 ConfigClass=daxApdb.ApdbConfig, 

155 doc="Database connection for storing associated DiaSources and " 

156 "DiaObjects. Must already be initialized.", 

157 ) 

158 validBands = pexConfig.ListField( 

159 dtype=str, 

160 default=["u", "g", "r", "i", "z", "y"], 

161 doc="List of bands that are valid for AP processing. To process a " 

162 "band not on this list, the appropriate band specific columns " 

163 "must be added to the Apdb schema in dax_apdb.", 

164 ) 

165 diaCatalogLoader = pexConfig.ConfigurableField( 

166 target=LoadDiaCatalogsTask, 

167 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

168 ) 

169 associator = pexConfig.ConfigurableField( 

170 target=AssociationTask, 

171 doc="Task used to associate DiaSources with DiaObjects.", 

172 ) 

173 diaForcedSource = pexConfig.ConfigurableField( 

174 target=DiaForcedSourceTask, 

175 doc="Task used for force photometer DiaObject locations in direct and " 

176 "difference images.", 

177 ) 

178 alertPackager = pexConfig.ConfigurableField( 

179 target=PackageAlertsTask, 

180 doc="Subtask for packaging Ap data into alerts.", 

181 ) 

182 doPackageAlerts = pexConfig.Field( 

183 dtype=bool, 

184 default=False, 

185 doc="Package Dia-data into serialized alerts for distribution and " 

186 "write them to disk.", 

187 ) 

188 doWriteAssociatedSources = pexConfig.Field( 

189 dtype=bool, 

190 default=False, 

191 doc="Write out associated and SDMed DiaSources.", 

192 ) 

193 

194 def setDefaults(self): 

195 self.apdb.dia_object_index = "baseline" 

196 self.apdb.dia_object_columns = [] 

197 self.apdb.extra_schema_file = os.path.join( 

198 getPackageDir("ap_association"), 

199 "data", 

200 "apdb-ap-pipe-schema-extra.yaml") 

201 

202 def validate(self): 

203 pexConfig.Config.validate(self) 

204 if self.diaCatalogLoader.htmLevel != \ 

205 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

206 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

207 "equal to HTMIndexDiaCalculationPlugin index " 

208 "level.") 

209 

210 

211class DiaPipelineTask(pipeBase.PipelineTask): 

212 """Task for loading, associating and storing Difference Image Analysis 

213 (DIA) Objects and Sources. 

214 """ 

215 ConfigClass = DiaPipelineConfig 

216 _DefaultName = "diaPipe" 

217 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

218 

219 def __init__(self, initInputs=None, **kwargs): 

220 super().__init__(**kwargs) 

221 self.apdb = self.config.apdb.apply( 

222 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

223 DiaSource=make_dia_source_schema())) 

224 self.makeSubtask("diaCatalogLoader") 

225 self.makeSubtask("associator") 

226 self.makeSubtask("diaForcedSource") 

227 if self.config.doPackageAlerts: 

228 self.makeSubtask("alertPackager") 

229 

230 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

231 inputs = butlerQC.get(inputRefs) 

232 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

233 returnMaxBits=True) 

234 inputs["ccdExposureIdBits"] = expBits 

235 

236 outputs = self.run(**inputs) 

237 

238 butlerQC.put(outputs, outputRefs) 

239 

240 @pipeBase.timeMethod 

241 def run(self, 

242 diaSourceTable, 

243 diffIm, 

244 exposure, 

245 warpedExposure, 

246 ccdExposureIdBits): 

247 """Process DiaSources and DiaObjects. 

248 

249 Load previous DiaObjects and their DiaSource history. Calibrate the 

250 values in the diaSourceCat. Associate new DiaSources with previous 

251 DiaObjects. Run forced photometry at the updated DiaObject locations. 

252 Store the results in the Alert Production Database (Apdb). 

253 

254 Parameters 

255 ---------- 

256 diaSourceTable : `pandas.DataFrame` 

257 Newly detected DiaSources. 

258 diffIm : `lsst.afw.image.ExposureF` 

259 Difference image exposure in which the sources in ``diaSourceCat`` 

260 were detected. 

261 exposure : `lsst.afw.image.ExposureF` 

262 Calibrated exposure differenced with a template to create 

263 ``diffIm``. 

264 warpedExposure : `lsst.afw.image.ExposureF` 

265 Template exposure used to create diffIm. 

266 ccdExposureIdBits : `int` 

267 Number of bits used for a unique ``ccdVisitId``. 

268 

269 Returns 

270 ------- 

271 results : `lsst.pipe.base.Struct` 

272 Results struct with components. 

273 

274 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

275 that this ccdVisit has completed successfully. 

276 (`lsst.dax.apdb.ApdbConfig`) 

277 """ 

278 self.log.info("Running DiaPipeline...") 

279 # Put the SciencePipelines through a SDMification step and return 

280 # calibrated columns with the expect output database names. 

281 

282 # Load the DiaObjects and DiaSource history. 

283 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

284 

285 # Associate new DiaSources with existing DiaObjects and update 

286 # DiaObject summary statistics using the full DiaSource history. 

287 assocResults = self.associator.run(diaSourceTable, 

288 loaderResult.diaObjects, 

289 loaderResult.diaSources) 

290 

291 # Force photometer on the Difference and Calibrated exposures using 

292 # the new and updated DiaObject locations. 

293 diaForcedSources = self.diaForcedSource.run( 

294 assocResults.diaObjects, 

295 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

296 ccdExposureIdBits, 

297 exposure, 

298 diffIm) 

299 

300 # Store DiaSources and updated DiaObjects in the Apdb. 

301 self.apdb.storeDiaSources(assocResults.diaSources) 

302 self.apdb.storeDiaObjects( 

303 assocResults.updatedDiaObjects, 

304 exposure.getInfo().getVisitInfo().getDate().toPython()) 

305 self.apdb.storeDiaForcedSources(diaForcedSources) 

306 

307 if self.config.doPackageAlerts: 

308 if len(loaderResult.diaForcedSources) > 1: 

309 diaForcedSources = diaForcedSources.append( 

310 loaderResult.diaForcedSources, 

311 sort=True) 

312 if diaForcedSources.index.has_duplicates: 

313 self.log.warn( 

314 "Duplicate DiaForcedSources created after merge with " 

315 "history and new sources. This may cause downstream " 

316 "problems. Dropping duplicates.") 

317 # Drop duplicates via index and keep the first appearance. 

318 # Reset due to the index shape being slight different than 

319 # expected. 

320 diaForcedSources = diaForcedSources.groupby( 

321 diaForcedSources.index).first() 

322 diaForcedSources.reset_index(drop=True, inplace=True) 

323 diaForcedSources.set_index( 

324 ["diaObjectId", "diaForcedSourceId"], 

325 drop=False, 

326 inplace=True) 

327 self.alertPackager.run(assocResults.diaSources, 

328 assocResults.diaObjects, 

329 loaderResult.diaSources, 

330 diaForcedSources, 

331 diffIm, 

332 warpedExposure, 

333 ccdExposureIdBits) 

334 

335 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

336 associatedDiaSources=assocResults.diaSources)