Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 MapDiaSourceTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47 

48__all__ = ("DiaPipelineConfig", 

49 "DiaPipelineTask", 

50 "DiaPipelineConnections") 

51 

52 

53class DiaPipelineConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceSchema = connTypes.InitInput( 

59 doc="Schema of the DiaSource catalog produced during image " 

60 "differencing", 

61 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

62 storageClass="SourceCatalog", 

63 multiple=True 

64 ) 

65 diaSourceCat = connTypes.Input( 

66 doc="Catalog of DiaSources produced during image differencing.", 

67 name="{fakesType}{coaddName}Diff_diaSrc", 

68 storageClass="SourceCatalog", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 diffIm = connTypes.Input( 

72 doc="Difference image on which the DiaSources were detected.", 

73 name="{fakesType}{coaddName}Diff_differenceExp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 exposure = connTypes.Input( 

78 doc="Calibrated exposure differenced with a template image during " 

79 "image differencing.", 

80 name="calexp", 

81 storageClass="ExposureF", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 warpedExposure = connTypes.Input( 

85 doc="Warped template used to create `subtractedExposure`. Not PSF " 

86 "matched.", 

87 dimensions=("instrument", "visit", "detector"), 

88 storageClass="ExposureF", 

89 name="{fakesType}{coaddName}Diff_warpedExp", 

90 ) 

91 apdbMarker = connTypes.Output( 

92 doc="Marker dataset storing the configuration of the Apdb for each " 

93 "visit/detector. Used to signal the completion of the pipeline.", 

94 name="apdb_marker", 

95 storageClass="Config", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 

99 

100class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

101 pipelineConnections=DiaPipelineConnections): 

102 """Config for DiaPipelineTask. 

103 """ 

104 coaddName = pexConfig.Field( 

105 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

106 dtype=str, 

107 default="deep", 

108 ) 

109 apdb = pexConfig.ConfigurableField( 

110 target=daxApdb.Apdb, 

111 ConfigClass=daxApdb.ApdbConfig, 

112 doc="Database connection for storing associated DiaSources and " 

113 "DiaObjects. Must already be initialized.", 

114 ) 

115 diaSourceDpddifier = pexConfig.ConfigurableField( 

116 target=MapDiaSourceTask, 

117 doc="Task for assigning columns from the raw output of ip_diffim into " 

118 "a schema that more closely resembles the DPDD.", 

119 ) 

120 diaCatalogLoader = pexConfig.ConfigurableField( 

121 target=LoadDiaCatalogsTask, 

122 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

123 ) 

124 associator = pexConfig.ConfigurableField( 

125 target=AssociationTask, 

126 doc="Task used to associate DiaSources with DiaObjects.", 

127 ) 

128 diaForcedSource = pexConfig.ConfigurableField( 

129 target=DiaForcedSourceTask, 

130 doc="Task used for force photometer DiaObject locations in direct and " 

131 "difference images.", 

132 ) 

133 alertPackager = pexConfig.ConfigurableField( 

134 target=PackageAlertsTask, 

135 doc="Subtask for packaging Ap data into alerts.", 

136 ) 

137 doPackageAlerts = pexConfig.Field( 

138 dtype=bool, 

139 default=False, 

140 doc="Package Dia-data into serialized alerts for distribution and " 

141 "write them to disk.", 

142 ) 

143 

144 def setDefaults(self): 

145 self.apdb.dia_object_index = "baseline" 

146 self.apdb.dia_object_columns = [] 

147 self.apdb.extra_schema_file = os.path.join( 

148 getPackageDir("ap_association"), 

149 "data", 

150 "apdb-ap-pipe-schema-extra.yaml") 

151 

152 def validate(self): 

153 pexConfig.Config.validate(self) 

154 if self.diaCatalogLoader.htmLevel != \ 

155 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

156 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

157 "equal to HTMIndexDiaCalculationPlugin index " 

158 "level.") 

159 

160 

161class DiaPipelineTask(pipeBase.PipelineTask): 

162 """Task for loading, associating and storing Difference Image Analysis 

163 (DIA) Objects and Sources. 

164 """ 

165 ConfigClass = DiaPipelineConfig 

166 _DefaultName = "diaPipe" 

167 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

168 

169 def __init__(self, initInputs=None, **kwargs): 

170 super().__init__(**kwargs) 

171 self.apdb = self.config.apdb.apply( 

172 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

173 DiaSource=make_dia_source_schema())) 

174 self.makeSubtask("diaSourceDpddifier", 

175 inputSchema=initInputs["diaSourceSchema"].schema) 

176 self.makeSubtask("diaCatalogLoader") 

177 self.makeSubtask("associator") 

178 self.makeSubtask("diaForcedSource") 

179 if self.config.doPackageAlerts: 

180 self.makeSubtask("alertPackager") 

181 

182 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

183 inputs = butlerQC.get(inputRefs) 

184 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

185 returnMaxBits=True) 

186 inputs["ccdExposureIdBits"] = expBits 

187 

188 outputs = self.run(**inputs) 

189 

190 butlerQC.put(outputs, outputRefs) 

191 

192 @pipeBase.timeMethod 

193 def run(self, diaSourceCat, diffIm, exposure, template, ccdExposureIdBits): 

194 """Process DiaSources and DiaObjects. 

195 

196 Load previous DiaObjects and their DiaSource history. Calibrate the 

197 values in the diaSourceCat. Associate new DiaSources with previous 

198 DiaObjects. Run forced photometry at the updated DiaObject locations. 

199 Store the results in the Alert Production Database (Apdb). 

200 

201 Parameters 

202 ---------- 

203 diaSourceCat : `lsst.afw.table.SourceCatalog` 

204 Newly detected DiaSources. 

205 diffIm : `lsst.afw.image.ExposureF` 

206 Difference image exposure in which the sources in ``diaSourceCat`` 

207 were detected. 

208 exposure : `lsst.afw.image.ExposureF` 

209 Calibrated exposure differenced with a template to create 

210 ``diffIm``. 

211 template : `lsst.afw.image.ExposureF` 

212 Template exposure used to create diffIm. 

213 ccdExposureIdBits : `int` 

214 Number of bits used for a unique ``ccdVisitId``. 

215 

216 Returns 

217 ------- 

218 results : `lsst.pipe.base.Struct` 

219 Results struct with components. 

220 

221 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

222 that this ccdVisit has completed successfully. 

223 (`lsst.dax.apdb.ApdbConfig`) 

224 """ 

225 self.log.info("Running DiaPipeline...") 

226 # Put the SciencePipelines through a SDMification step and return 

227 # calibrated columns with the expect output database names. 

228 diaSources = self.diaSourceDpddifier.run(diaSourceCat, 

229 diffIm, 

230 return_pandas=True) 

231 

232 # Load the DiaObjects and DiaSource history. 

233 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

234 

235 # Associate new DiaSources with existing DiaObjects and update 

236 # DiaObject summary statistics using the full DiaSource history. 

237 assocResults = self.associator.run(diaSources, 

238 loaderResult.diaObjects, 

239 loaderResult.diaSources) 

240 

241 # Force photometer on the Difference and Calibrated exposures using 

242 # the new and updated DiaObject locations. 

243 diaForcedSources = self.diaForcedSource.run( 

244 assocResults.diaObjects, 

245 ccdExposureIdBits, 

246 exposure, 

247 diffIm) 

248 

249 # Store DiaSources and updated DiaObjects in the Apdb. 

250 self.apdb.storeDiaSources(assocResults.diaSources) 

251 self.apdb.storeDiaObjects( 

252 assocResults.updatedDiaObjects, 

253 exposure.getInfo().getVisitInfo().getDate().toPython()) 

254 self.apdb.storeDiaForcedSources(diaForcedSources) 

255 if self.config.doPackageAlerts: 

256 if len(loaderResult.diaForcedSources) > 1: 

257 diaForcedSources = diaForcedSources.append( 

258 loaderResult.diaForcedSources, 

259 sort=True) 

260 self.alertPackager.run(assocResults.diaSources, 

261 assocResults.diaObjects, 

262 loaderResult.diaSources, 

263 diaForcedSources, 

264 diffIm, 

265 template, 

266 ccdExposureIdBits) 

267 

268 return pipeBase.Struct(apdbMarker=self.config.apdb.value)