Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 MapDiaSourceTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47 

48__all__ = ("DiaPipelineConfig", 

49 "DiaPipelineTask", 

50 "DiaPipelineConnections") 

51 

52 

53class DiaPipelineConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceSchema = connTypes.InitInput( 

59 doc="Schema of the DiaSource catalog produced during image " 

60 "differencing", 

61 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

62 storageClass="SourceCatalog", 

63 multiple=True 

64 ) 

65 diaSourceCat = connTypes.Input( 

66 doc="Catalog of DiaSources produced during image differencing.", 

67 name="{fakesType}{coaddName}Diff_diaSrc", 

68 storageClass="SourceCatalog", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 diffIm = connTypes.Input( 

72 doc="Difference image on which the DiaSources were detected.", 

73 name="{fakesType}{coaddName}Diff_differenceExp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 exposure = connTypes.Input( 

78 doc="Calibrated exposure differenced with a template image during " 

79 "image differencing.", 

80 name="calexp", 

81 storageClass="ExposureF", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 warpedExposure = connTypes.Input( 

85 doc="Warped template used to create `subtractedExposure`. Not PSF " 

86 "matched.", 

87 dimensions=("instrument", "visit", "detector"), 

88 storageClass="ExposureF", 

89 name="{fakesType}{coaddName}Diff_warpedExp", 

90 ) 

91 apdbMarker = connTypes.Output( 

92 doc="Marker dataset storing the configuration of the Apdb for each " 

93 "visit/detector. Used to signal the completion of the pipeline.", 

94 name="apdb_marker", 

95 storageClass="Config", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 associatedDiaSources = connTypes.Output( 

99 doc="Optional output storing the DiaSource catalog after matching, " 

100 "calibration, and standardization for insertation into the Apdb.", 

101 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

102 storageClass="DataFrame", 

103 dimensions=("instrument", "visit", "detector"), 

104 ) 

105 

106 def __init__(self, *, config=None): 

107 super().__init__(config=config) 

108 

109 if not config.doWriteAssociatedSources: 

110 self.outputs.remove("associatedDiaSources") 

111 

112 def adjustQuantum(self, datasetRefMap: pipeBase.InputQuantizedConnection): 

113 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

114 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

115 of the activator. 

116 

117 This implementation checks to make sure that the filters in the dataset 

118 are compatible with AP processing as set by the Apdb/DPDD schema. 

119 

120 Parameters 

121 ---------- 

122 datasetRefMap : `NamedKeyDict` 

123 Mapping from dataset type to a `set` of 

124 `lsst.daf.butler.DatasetRef` objects 

125 

126 Returns 

127 ------- 

128 datasetRefMap : `NamedKeyDict` 

129 Mapping of input with assurances that bands incompatible with the 

130 Apdb are present. 

131 

132 Raises 

133 ------ 

134 ValueError 

135 Raises if a data ref in the quantum has a band not available in the 

136 Apdb. 

137 """ 

138 refs = datasetRefMap[self.diffIm.name] 

139 for ref in refs: 

140 if ref.dataId["band"] not in self.config.validBands: 

141 raise ValueError( 

142 f"Requested '{ref.dataId['band']}' not in " 

143 "DiaPipelineConfig.validBands. To process bands not in the " 

144 "standard Rubin set (ugrizy) you must add the band to the " 

145 "validBands list in DiaPipelineConfig and add the " 

146 "appropriate columns to the Apdb schema.") 

147 return super().adjustQuantum(datasetRefMap) 

148 

149 

150class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

151 pipelineConnections=DiaPipelineConnections): 

152 """Config for DiaPipelineTask. 

153 """ 

154 coaddName = pexConfig.Field( 

155 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

156 dtype=str, 

157 default="deep", 

158 ) 

159 apdb = pexConfig.ConfigurableField( 

160 target=daxApdb.Apdb, 

161 ConfigClass=daxApdb.ApdbConfig, 

162 doc="Database connection for storing associated DiaSources and " 

163 "DiaObjects. Must already be initialized.", 

164 ) 

165 validBands = pexConfig.ListField( 

166 dtype=str, 

167 default=["u", "g", "r", "i", "z", "y"], 

168 doc="List of bands that are valid for AP processing. To process a " 

169 "band not on this list, the appropriate band specific columns " 

170 "must be added to the Apdb schema in dax_apdb.", 

171 ) 

172 diaSourceDpddifier = pexConfig.ConfigurableField( 

173 target=MapDiaSourceTask, 

174 doc="Task for assigning columns from the raw output of ip_diffim into " 

175 "a schema that more closely resembles the DPDD.", 

176 ) 

177 diaCatalogLoader = pexConfig.ConfigurableField( 

178 target=LoadDiaCatalogsTask, 

179 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

180 ) 

181 associator = pexConfig.ConfigurableField( 

182 target=AssociationTask, 

183 doc="Task used to associate DiaSources with DiaObjects.", 

184 ) 

185 diaForcedSource = pexConfig.ConfigurableField( 

186 target=DiaForcedSourceTask, 

187 doc="Task used for force photometer DiaObject locations in direct and " 

188 "difference images.", 

189 ) 

190 alertPackager = pexConfig.ConfigurableField( 

191 target=PackageAlertsTask, 

192 doc="Subtask for packaging Ap data into alerts.", 

193 ) 

194 doPackageAlerts = pexConfig.Field( 

195 dtype=bool, 

196 default=False, 

197 doc="Package Dia-data into serialized alerts for distribution and " 

198 "write them to disk.", 

199 ) 

200 doWriteAssociatedSources = pexConfig.Field( 

201 dtype=bool, 

202 default=False, 

203 doc="Write out associated and SDMed DiaSources.", 

204 ) 

205 

206 def setDefaults(self): 

207 self.apdb.dia_object_index = "baseline" 

208 self.apdb.dia_object_columns = [] 

209 self.apdb.extra_schema_file = os.path.join( 

210 getPackageDir("ap_association"), 

211 "data", 

212 "apdb-ap-pipe-schema-extra.yaml") 

213 

214 def validate(self): 

215 pexConfig.Config.validate(self) 

216 if self.diaCatalogLoader.htmLevel != \ 

217 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

218 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

219 "equal to HTMIndexDiaCalculationPlugin index " 

220 "level.") 

221 

222 

223class DiaPipelineTask(pipeBase.PipelineTask): 

224 """Task for loading, associating and storing Difference Image Analysis 

225 (DIA) Objects and Sources. 

226 """ 

227 ConfigClass = DiaPipelineConfig 

228 _DefaultName = "diaPipe" 

229 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

230 

231 def __init__(self, initInputs=None, **kwargs): 

232 super().__init__(**kwargs) 

233 self.apdb = self.config.apdb.apply( 

234 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

235 DiaSource=make_dia_source_schema())) 

236 self.makeSubtask("diaSourceDpddifier", 

237 inputSchema=initInputs["diaSourceSchema"].schema) 

238 self.makeSubtask("diaCatalogLoader") 

239 self.makeSubtask("associator") 

240 self.makeSubtask("diaForcedSource") 

241 if self.config.doPackageAlerts: 

242 self.makeSubtask("alertPackager") 

243 

244 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

245 inputs = butlerQC.get(inputRefs) 

246 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

247 returnMaxBits=True) 

248 inputs["ccdExposureIdBits"] = expBits 

249 

250 outputs = self.run(**inputs) 

251 

252 butlerQC.put(outputs, outputRefs) 

253 

254 @pipeBase.timeMethod 

255 def run(self, diaSourceCat, diffIm, exposure, warpedExposure, ccdExposureIdBits): 

256 """Process DiaSources and DiaObjects. 

257 

258 Load previous DiaObjects and their DiaSource history. Calibrate the 

259 values in the diaSourceCat. Associate new DiaSources with previous 

260 DiaObjects. Run forced photometry at the updated DiaObject locations. 

261 Store the results in the Alert Production Database (Apdb). 

262 

263 Parameters 

264 ---------- 

265 diaSourceCat : `lsst.afw.table.SourceCatalog` 

266 Newly detected DiaSources. 

267 diffIm : `lsst.afw.image.ExposureF` 

268 Difference image exposure in which the sources in ``diaSourceCat`` 

269 were detected. 

270 exposure : `lsst.afw.image.ExposureF` 

271 Calibrated exposure differenced with a template to create 

272 ``diffIm``. 

273 warpedExposure : `lsst.afw.image.ExposureF` 

274 Template exposure used to create diffIm. 

275 ccdExposureIdBits : `int` 

276 Number of bits used for a unique ``ccdVisitId``. 

277 

278 Returns 

279 ------- 

280 results : `lsst.pipe.base.Struct` 

281 Results struct with components. 

282 

283 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

284 that this ccdVisit has completed successfully. 

285 (`lsst.dax.apdb.ApdbConfig`) 

286 """ 

287 self.log.info("Running DiaPipeline...") 

288 # Put the SciencePipelines through a SDMification step and return 

289 # calibrated columns with the expect output database names. 

290 diaSources = self.diaSourceDpddifier.run(diaSourceCat, 

291 diffIm, 

292 return_pandas=True) 

293 

294 # Load the DiaObjects and DiaSource history. 

295 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

296 

297 # Associate new DiaSources with existing DiaObjects and update 

298 # DiaObject summary statistics using the full DiaSource history. 

299 assocResults = self.associator.run(diaSources, 

300 loaderResult.diaObjects, 

301 loaderResult.diaSources) 

302 

303 # Force photometer on the Difference and Calibrated exposures using 

304 # the new and updated DiaObject locations. 

305 diaForcedSources = self.diaForcedSource.run( 

306 assocResults.diaObjects, 

307 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

308 ccdExposureIdBits, 

309 exposure, 

310 diffIm) 

311 

312 # Store DiaSources and updated DiaObjects in the Apdb. 

313 self.apdb.storeDiaSources(assocResults.diaSources) 

314 self.apdb.storeDiaObjects( 

315 assocResults.updatedDiaObjects, 

316 exposure.getInfo().getVisitInfo().getDate().toPython()) 

317 self.apdb.storeDiaForcedSources(diaForcedSources) 

318 if self.config.doPackageAlerts: 

319 if len(loaderResult.diaForcedSources) > 1: 

320 diaForcedSources = diaForcedSources.append( 

321 loaderResult.diaForcedSources, 

322 sort=True) 

323 self.alertPackager.run(assocResults.diaSources, 

324 assocResults.diaObjects, 

325 loaderResult.diaSources, 

326 diaForcedSources, 

327 diffIm, 

328 warpedExposure, 

329 ccdExposureIdBits) 

330 

331 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

332 associatedDiaSources=assocResults.diaSources)