Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 MapDiaSourceTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47 

48__all__ = ("DiaPipelineConfig", 

49 "DiaPipelineTask", 

50 "DiaPipelineConnections") 

51 

52 

53class DiaPipelineConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceSchema = connTypes.InitInput( 

59 doc="Schema of the DiaSource catalog produced during image " 

60 "differencing", 

61 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

62 storageClass="SourceCatalog", 

63 multiple=True 

64 ) 

65 diaSourceCat = connTypes.Input( 

66 doc="Catalog of DiaSources produced during image differencing.", 

67 name="{fakesType}{coaddName}Diff_diaSrc", 

68 storageClass="SourceCatalog", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 diffIm = connTypes.Input( 

72 doc="Difference image on which the DiaSources were detected.", 

73 name="{fakesType}{coaddName}Diff_differenceExp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 exposure = connTypes.Input( 

78 doc="Calibrated exposure differenced with a template image during " 

79 "image differencing.", 

80 name="calexp", 

81 storageClass="ExposureF", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 warpedExposure = connTypes.Input( 

85 doc="Warped template used to create `subtractedExposure`. Not PSF " 

86 "matched.", 

87 dimensions=("instrument", "visit", "detector"), 

88 storageClass="ExposureF", 

89 name="{fakesType}{coaddName}Diff_warpedExp", 

90 ) 

91 apdbMarker = connTypes.Output( 

92 doc="Marker dataset storing the configuration of the Apdb for each " 

93 "visit/detector. Used to signal the completion of the pipeline.", 

94 name="apdb_marker", 

95 storageClass="Config", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 associatedDiaSources = connTypes.Output( 

99 doc="Optional output storing the DiaSource catalog after matching, " 

100 "calibration, and standardization for insertation into the Apdb.", 

101 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

102 storageClass="DataFrame", 

103 dimensions=("instrument", "visit", "detector"), 

104 ) 

105 

106 def __init__(self, *, config=None): 

107 super().__init__(config=config) 

108 

109 if not config.doWriteAssociatedSources: 

110 self.outputs.remove("associatedDiaSources") 

111 

112 

113class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

114 pipelineConnections=DiaPipelineConnections): 

115 """Config for DiaPipelineTask. 

116 """ 

117 coaddName = pexConfig.Field( 

118 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

119 dtype=str, 

120 default="deep", 

121 ) 

122 apdb = pexConfig.ConfigurableField( 

123 target=daxApdb.Apdb, 

124 ConfigClass=daxApdb.ApdbConfig, 

125 doc="Database connection for storing associated DiaSources and " 

126 "DiaObjects. Must already be initialized.", 

127 ) 

128 diaSourceDpddifier = pexConfig.ConfigurableField( 

129 target=MapDiaSourceTask, 

130 doc="Task for assigning columns from the raw output of ip_diffim into " 

131 "a schema that more closely resembles the DPDD.", 

132 ) 

133 diaCatalogLoader = pexConfig.ConfigurableField( 

134 target=LoadDiaCatalogsTask, 

135 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

136 ) 

137 associator = pexConfig.ConfigurableField( 

138 target=AssociationTask, 

139 doc="Task used to associate DiaSources with DiaObjects.", 

140 ) 

141 diaForcedSource = pexConfig.ConfigurableField( 

142 target=DiaForcedSourceTask, 

143 doc="Task used for force photometer DiaObject locations in direct and " 

144 "difference images.", 

145 ) 

146 alertPackager = pexConfig.ConfigurableField( 

147 target=PackageAlertsTask, 

148 doc="Subtask for packaging Ap data into alerts.", 

149 ) 

150 doPackageAlerts = pexConfig.Field( 

151 dtype=bool, 

152 default=False, 

153 doc="Package Dia-data into serialized alerts for distribution and " 

154 "write them to disk.", 

155 ) 

156 doWriteAssociatedSources = pexConfig.Field( 

157 dtype=bool, 

158 default=False, 

159 doc="Write out associated and SDMed DiaSources.", 

160 ) 

161 

162 def setDefaults(self): 

163 self.apdb.dia_object_index = "baseline" 

164 self.apdb.dia_object_columns = [] 

165 self.apdb.extra_schema_file = os.path.join( 

166 getPackageDir("ap_association"), 

167 "data", 

168 "apdb-ap-pipe-schema-extra.yaml") 

169 

170 def validate(self): 

171 pexConfig.Config.validate(self) 

172 if self.diaCatalogLoader.htmLevel != \ 

173 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

174 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

175 "equal to HTMIndexDiaCalculationPlugin index " 

176 "level.") 

177 

178 

179class DiaPipelineTask(pipeBase.PipelineTask): 

180 """Task for loading, associating and storing Difference Image Analysis 

181 (DIA) Objects and Sources. 

182 """ 

183 ConfigClass = DiaPipelineConfig 

184 _DefaultName = "diaPipe" 

185 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

186 

187 def __init__(self, initInputs=None, **kwargs): 

188 super().__init__(**kwargs) 

189 self.apdb = self.config.apdb.apply( 

190 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

191 DiaSource=make_dia_source_schema())) 

192 self.makeSubtask("diaSourceDpddifier", 

193 inputSchema=initInputs["diaSourceSchema"].schema) 

194 self.makeSubtask("diaCatalogLoader") 

195 self.makeSubtask("associator") 

196 self.makeSubtask("diaForcedSource") 

197 if self.config.doPackageAlerts: 

198 self.makeSubtask("alertPackager") 

199 

200 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

201 inputs = butlerQC.get(inputRefs) 

202 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

203 returnMaxBits=True) 

204 inputs["ccdExposureIdBits"] = expBits 

205 

206 outputs = self.run(**inputs) 

207 

208 butlerQC.put(outputs, outputRefs) 

209 

210 @pipeBase.timeMethod 

211 def run(self, diaSourceCat, diffIm, exposure, warpedExposure, ccdExposureIdBits): 

212 """Process DiaSources and DiaObjects. 

213 

214 Load previous DiaObjects and their DiaSource history. Calibrate the 

215 values in the diaSourceCat. Associate new DiaSources with previous 

216 DiaObjects. Run forced photometry at the updated DiaObject locations. 

217 Store the results in the Alert Production Database (Apdb). 

218 

219 Parameters 

220 ---------- 

221 diaSourceCat : `lsst.afw.table.SourceCatalog` 

222 Newly detected DiaSources. 

223 diffIm : `lsst.afw.image.ExposureF` 

224 Difference image exposure in which the sources in ``diaSourceCat`` 

225 were detected. 

226 exposure : `lsst.afw.image.ExposureF` 

227 Calibrated exposure differenced with a template to create 

228 ``diffIm``. 

229 warpedExposure : `lsst.afw.image.ExposureF` 

230 Template exposure used to create diffIm. 

231 ccdExposureIdBits : `int` 

232 Number of bits used for a unique ``ccdVisitId``. 

233 

234 Returns 

235 ------- 

236 results : `lsst.pipe.base.Struct` 

237 Results struct with components. 

238 

239 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

240 that this ccdVisit has completed successfully. 

241 (`lsst.dax.apdb.ApdbConfig`) 

242 """ 

243 self.log.info("Running DiaPipeline...") 

244 # Put the SciencePipelines through a SDMification step and return 

245 # calibrated columns with the expect output database names. 

246 diaSources = self.diaSourceDpddifier.run(diaSourceCat, 

247 diffIm, 

248 return_pandas=True) 

249 

250 # Load the DiaObjects and DiaSource history. 

251 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

252 

253 # Associate new DiaSources with existing DiaObjects and update 

254 # DiaObject summary statistics using the full DiaSource history. 

255 assocResults = self.associator.run(diaSources, 

256 loaderResult.diaObjects, 

257 loaderResult.diaSources) 

258 

259 # Force photometer on the Difference and Calibrated exposures using 

260 # the new and updated DiaObject locations. 

261 diaForcedSources = self.diaForcedSource.run( 

262 assocResults.diaObjects, 

263 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

264 ccdExposureIdBits, 

265 exposure, 

266 diffIm) 

267 

268 # Store DiaSources and updated DiaObjects in the Apdb. 

269 self.apdb.storeDiaSources(assocResults.diaSources) 

270 self.apdb.storeDiaObjects( 

271 assocResults.updatedDiaObjects, 

272 exposure.getInfo().getVisitInfo().getDate().toPython()) 

273 self.apdb.storeDiaForcedSources(diaForcedSources) 

274 if self.config.doPackageAlerts: 

275 if len(loaderResult.diaForcedSources) > 1: 

276 diaForcedSources = diaForcedSources.append( 

277 loaderResult.diaForcedSources, 

278 sort=True) 

279 self.alertPackager.run(assocResults.diaSources, 

280 assocResults.diaObjects, 

281 loaderResult.diaSources, 

282 diaForcedSources, 

283 diffIm, 

284 warpedExposure, 

285 ccdExposureIdBits) 

286 

287 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

288 associatedDiaSources=assocResults.diaSources)