Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 MapDiaSourceTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47 

48__all__ = ("DiaPipelineConfig", 

49 "DiaPipelineTask", 

50 "DiaPipelineConnections") 

51 

52 

53class DiaPipelineConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceSchema = connTypes.InitInput( 

59 doc="Schema of the DiaSource catalog produced during image " 

60 "differencing", 

61 name="{fakesType}{coaddName}Diff_diaSrc_schema", 

62 storageClass="SourceCatalog", 

63 multiple=True 

64 ) 

65 diaSourceCat = connTypes.Input( 

66 doc="Catalog of DiaSources produced during image differencing.", 

67 name="{fakesType}{coaddName}Diff_diaSrc", 

68 storageClass="SourceCatalog", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 diffIm = connTypes.Input( 

72 doc="Difference image on which the DiaSources were detected.", 

73 name="{fakesType}{coaddName}Diff_differenceExp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 exposure = connTypes.Input( 

78 doc="Calibrated exposure differenced with a template image during " 

79 "image differencing.", 

80 name="calexp", 

81 storageClass="ExposureF", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 apdbMarker = connTypes.Output( 

85 doc="Marker dataset storing the configuration of the Apdb for each " 

86 "visit/detector. Used to signal the completion of the pipeline.", 

87 name="apdb_marker", 

88 storageClass="Config", 

89 dimensions=("instrument", "visit", "detector"), 

90 ) 

91 

92 

93class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

94 pipelineConnections=DiaPipelineConnections): 

95 """Config for DiaPipelineTask. 

96 """ 

97 apdb = pexConfig.ConfigurableField( 

98 target=daxApdb.Apdb, 

99 ConfigClass=daxApdb.ApdbConfig, 

100 doc="Database connection for storing associated DiaSources and " 

101 "DiaObjects. Must already be initialized.", 

102 ) 

103 diaSourceDpddifier = pexConfig.ConfigurableField( 

104 target=MapDiaSourceTask, 

105 doc="Task for assigning columns from the raw output of ip_diffim into " 

106 "a schema that more closely resembles the DPDD.", 

107 ) 

108 diaCatalogLoader = pexConfig.ConfigurableField( 

109 target=LoadDiaCatalogsTask, 

110 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

111 ) 

112 associator = pexConfig.ConfigurableField( 

113 target=AssociationTask, 

114 doc="Task used to associate DiaSources with DiaObjects.", 

115 ) 

116 diaForcedSource = pexConfig.ConfigurableField( 

117 target=DiaForcedSourceTask, 

118 doc="Task used for force photometer DiaObject locations in direct and " 

119 "difference images.", 

120 ) 

121 alertPackager = pexConfig.ConfigurableField( 

122 target=PackageAlertsTask, 

123 doc="Subtask for packaging Ap data into alerts.", 

124 ) 

125 doPackageAlerts = pexConfig.Field( 

126 dtype=bool, 

127 default=False, 

128 doc="Package Dia-data into serialized alerts for distribution and " 

129 "write them to disk.", 

130 ) 

131 

132 def setDefaults(self): 

133 self.apdb.dia_object_index = "baseline" 

134 self.apdb.dia_object_columns = [] 

135 self.apdb.extra_schema_file = os.path.join( 

136 getPackageDir("ap_association"), 

137 "data", 

138 "apdb-ap-pipe-schema-extra.yaml") 

139 

140 def validate(self): 

141 pexConfig.Config.validate(self) 

142 if self.diaCatalogLoader.htmLevel != \ 

143 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

144 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

145 "equal to HTMIndexDiaCalculationPlugin index " 

146 "level.") 

147 

148 

149class DiaPipelineTask(pipeBase.PipelineTask): 

150 """Task for loading, associating and storing Difference Image Analysis 

151 (DIA) Objects and Sources. 

152 """ 

153 ConfigClass = DiaPipelineConfig 

154 _DefaultName = "diaPipe" 

155 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

156 

157 def __init__(self, initInputs=None, **kwargs): 

158 super().__init__(**kwargs) 

159 self.apdb = self.config.apdb.apply( 

160 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

161 DiaSource=make_dia_source_schema())) 

162 self.makeSubtask("diaSourceDpddifier", 

163 inputSchema=initInputs["diaSourceSchema"].schema) 

164 self.makeSubtask("diaCatalogLoader") 

165 self.makeSubtask("associator") 

166 self.makeSubtask("diaForcedSource") 

167 if self.config.doPackageAlerts: 

168 self.makeSubtask("alertPackager") 

169 

170 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

171 inputs = butlerQC.get(inputRefs) 

172 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

173 returnMaxBits=True) 

174 inputs["ccdExposureIdBits"] = expBits 

175 

176 outputs = self.run(**inputs) 

177 

178 butlerQC.put(outputs, outputRefs) 

179 

180 @pipeBase.timeMethod 

181 def run(self, diaSourceCat, diffIm, exposure, ccdExposureIdBits): 

182 """Process DiaSources and DiaObjects. 

183 

184 Load previous DiaObjects and their DiaSource history. Calibrate the 

185 values in the diaSourceCat. Associate new DiaSources with previous 

186 DiaObjects. Run forced photometry at the updated DiaObject locations. 

187 Store the results in the Alert Production Database (Apdb). 

188 

189 Parameters 

190 ---------- 

191 diaSourceCat : `lsst.afw.table.SourceCatalog` 

192 Newly detected DiaSources. 

193 diffIm : `lsst.afw.image.Exposure` 

194 Difference image exposure in which the sources in ``diaSourceCat`` 

195 were detected. 

196 exposure : `lsst.afw.image.Exposure` 

197 Calibrated exposure differenced with a template to create 

198 ``diffIm``. 

199 ccdExposureIdBits : `int` 

200 Number of bits used for a unique ``ccdVisitId``. 

201 

202 Returns 

203 ------- 

204 results : `lsst.pipe.base.Struct` 

205 Results struct with components. 

206 

207 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

208 that this ccdVisit has completed successfully. 

209 (`lsst.dax.apdb.ApdbConfig`) 

210 """ 

211 self.log.info("Running DiaPipeline...") 

212 # Put the SciencePipelines through a SDMification step and return 

213 # calibrated columns with the expect output database names. 

214 diaSources = self.diaSourceDpddifier.run(diaSourceCat, 

215 diffIm, 

216 return_pandas=True) 

217 

218 # Load the DiaObjects and DiaSource history. 

219 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

220 

221 # Associate new DiaSources with existing DiaObjects and update 

222 # DiaObject summary statistics using the full DiaSource history. 

223 assocResults = self.associator.run(diaSources, 

224 loaderResult.diaObjects, 

225 loaderResult.diaSources) 

226 

227 # Force photometer on the Difference and Calibrated exposures using 

228 # the new and updated DiaObject locations. 

229 diaForcedSources = self.diaForcedSource.run( 

230 assocResults.diaObjects, 

231 ccdExposureIdBits, 

232 exposure, 

233 diffIm) 

234 

235 # Store DiaSources and updated DiaObjects in the Apdb. 

236 self.apdb.storeDiaSources(assocResults.diaSources) 

237 self.apdb.storeDiaObjects( 

238 assocResults.updatedDiaObjects, 

239 exposure.getInfo().getVisitInfo().getDate().toPython()) 

240 self.apdb.storeDiaForcedSources(diaForcedSources) 

241 if self.config.doPackageAlerts: 

242 if len(loaderResult.diaForcedSources) > 1: 

243 diaForcedSources = diaForcedSources.append( 

244 loaderResult.diaForcedSources, 

245 sort=True) 

246 self.alertPackager.run(assocResults.diaSources, 

247 assocResults.diaObjects, 

248 loaderResult.diaSources, 

249 diaForcedSources, 

250 diffIm, 

251 None, 

252 ccdExposureIdBits) 

253 

254 return pipeBase.Struct(apdbMarker=self.config.apdb.value)