Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32 

33import lsst.dax.apdb as daxApdb 

34import lsst.pex.config as pexConfig 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as connTypes 

37from lsst.utils import getPackageDir 

38 

39from lsst.ap.association import ( 

40 AssociationTask, 

41 DiaForcedSourceTask, 

42 LoadDiaCatalogsTask, 

43 MapDiaSourceTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47 

48__all__ = ("DiaPipelineConfig", 

49 "DiaPipelineTask", 

50 "DiaPipelineConnections") 

51 

52 

53class DiaPipelineConnections(pipeBase.PipelineTaskConnections, 

54 dimensions=("instrument", "visit", "detector"), 

55 defaultTemplates={"coaddName": "deep"}): 

56 """Butler connections for DiaPipelineTask. 

57 """ 

58 diaSourceSchema = connTypes.InitInput( 

59 doc="Schema of the DiaSource catalog produced during image " 

60 "differencing", 

61 name="{coaddName}Diff_diaSrc_schema", 

62 storageClass="SourceCatalog", 

63 multiple=True 

64 ) 

65 diaSourceCat = connTypes.Input( 

66 doc="Catalog of DiaSources produced during image differencing.", 

67 name="{coaddName}Diff_diaSrc", 

68 storageClass="SourceCatalog", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 diffIm = connTypes.Input( 

72 doc="Difference image on which the DiaSources were detected.", 

73 name="{coaddName}Diff_differenceExp", 

74 storageClass="ExposureF", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 exposure = connTypes.Input( 

78 doc="Calibrated exposure differenced with a template image during " 

79 "image differencing.", 

80 name="calexp", 

81 storageClass="ExposureF", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 apdbMarker = connTypes.Output( 

85 doc="Marker dataset storing the configuration of the Apdb for each " 

86 "visit/detector. Used to signal the completion of the pipeline.", 

87 name="apdb_marker", 

88 storageClass="", 

89 dimensions=("instrument", "visit", "detector"), 

90 ) 

91 

92 

93class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

94 pipelineConnections=DiaPipelineConnections): 

95 """Config for DiaPipelineTask. 

96 """ 

97 apdb = pexConfig.ConfigurableField( 

98 target=daxApdb.Apdb, 

99 ConfigClass=daxApdb.ApdbConfig, 

100 doc="Database connection for storing associated DiaSources and " 

101 "DiaObjects. Must already be initialized.", 

102 ) 

103 diaSourceDpddifier = pexConfig.ConfigurableField( 

104 target=MapDiaSourceTask, 

105 doc="Task for assigning columns from the raw output of ip_diffim into " 

106 "a schema that more closely resembles the DPDD.", 

107 ) 

108 diaCatalogLoader = pexConfig.ConfigurableField( 

109 target=LoadDiaCatalogsTask, 

110 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

111 ) 

112 associator = pexConfig.ConfigurableField( 

113 target=AssociationTask, 

114 doc="Task used to associate DiaSources with DiaObjects.", 

115 ) 

116 diaForcedSource = pexConfig.ConfigurableField( 

117 target=DiaForcedSourceTask, 

118 doc="Task used for force photometer DiaObject locations in direct and " 

119 "difference images.", 

120 ) 

121 alertPackager = pexConfig.ConfigurableField( 

122 target=PackageAlertsTask, 

123 doc="Subtask for packaging Ap data into alerts.", 

124 ) 

125 

126 def setDefaults(self): 

127 self.apdb.dia_object_index = "baseline" 

128 self.apdb.dia_object_columns = [] 

129 self.apdb.extra_schema_file = os.path.join( 

130 getPackageDir("ap_association"), 

131 "data", 

132 "apdb-ap-pipe-schema-extra.yaml") 

133 

134 def validate(self): 

135 pexConfig.Config.validate(self) 

136 if self.diaCatalogLoader.htmLevel != \ 

137 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

138 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

139 "equal to HTMIndexDiaCalculationPlugin index " 

140 "level.") 

141 

142 

143class DiaPipelineTask(pipeBase.PipelineTask): 

144 """Task for loading, associating and storing Difference Image Analysis 

145 (DIA) Objects and Sources. 

146 """ 

147 ConfigClass = DiaPipelineConfig 

148 _DefaultName = "diaPipe" 

149 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

150 

151 def __init__(self, initInputs=None, **kwargs): 

152 super().__init__(**kwargs) 

153 self.apdb = self.config.apdb.apply( 

154 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

155 DiaSource=make_dia_source_schema())) 

156 self.makeSubtask("diaSourceDpddifier", 

157 inputSchema=initInputs["diaSourceSchema"]) 

158 self.makeSubtask("diaCatalogLoader") 

159 self.makeSubtask("associator") 

160 self.makeSubtask("diaForcedSource") 

161 self.makeSubtask("alertPackager") 

162 

163 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

164 inputs = butlerQC.get(inputRefs) 

165 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

166 returnMaxBits=True) 

167 inputs["ccdExposureIdBits"] = expBits 

168 

169 outputs = self.run(**inputs) 

170 

171 butlerQC.put(outputs, outputRefs) 

172 

173 @pipeBase.timeMethod 

174 def run(self, diaSourceCat, diffIm, exposure, ccdExposureIdBits): 

175 """Process DiaSources and DiaObjects. 

176 

177 Load previous DiaObjects and their DiaSource history. Calibrate the 

178 values in the diaSourceCat. Associate new DiaSources with previous 

179 DiaObjects. Run forced photometry at the updated DiaObject locations. 

180 Store the results in the Alert Production Database (Apdb). 

181 

182 Parameters 

183 ---------- 

184 diaSourceCat : `lsst.afw.table.SourceCatalog` 

185 Newly detected DiaSources. 

186 diffIm : `lsst.afw.image.Exposure` 

187 Difference image exposure in which the sources in ``diaSourceCat`` 

188 were detected. 

189 exposure : `lsst.afw.image.Exposure` 

190 Calibrated exposure differenced with a template to create 

191 ``diffIm``. 

192 ccdExposureIdBits : `int` 

193 Number of bits used for a unique ``ccdVisitId``. 

194 

195 Returns 

196 ------- 

197 results : `lsst.pipe.base.Struct` 

198 Results struct with components. 

199 

200 - ``apdb_maker`` : Marker dataset to store in the Butler indicating 

201 that this ccdVisit has completed successfully. 

202 (`lsst.dax.apdb.ApdbConfig`) 

203 """ 

204 self.log.info("Running DiaPipeline...") 

205 # Put the SciencePipelines through a SDMification step and return 

206 # calibrated columns with the expect output database names. 

207 diaSources = self.diaSourceDpddifier.run(diaSourceCat, 

208 diffIm, 

209 return_pandas=True) 

210 

211 # Load the DiaObjects and DiaSource history. 

212 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

213 

214 # Associate new DiaSources with existing DiaObjects and update 

215 # DiaObject summary statistics using the full DiaSource history. 

216 assocResults = self.associator.run(diaSources, 

217 loaderResult.diaObjects, 

218 loaderResult.diaSources) 

219 

220 # Force photometer on the Difference and Calibrated exposures using 

221 # the new and updated DiaObject locations. 

222 diaForcedSources = self.diaForcedSource.run( 

223 assocResults.diaObjects, 

224 ccdExposureIdBits, 

225 exposure, 

226 diffIm) 

227 

228 # Store DiaSources and updated DiaObjects in the Apdb. 

229 self.apdb.storeDiaSources(assocResults.diaSources) 

230 self.apdb.storeDiaObjects( 

231 assocResults.updatedDiaObjects, 

232 exposure.getInfo().getVisitInfo().getDate().toPython()) 

233 self.apdb.storeDiaForcedSources(diaForcedSources) 

234 self.alertPackager.run(assocResults.diaSources, 

235 assocResults.diaObjects, 

236 loaderResult.diaSources, 

237 diffIm, 

238 None, 

239 ccdExposureIdBits) 

240 

241 return pipeBase.Struct(apdb_marker=self.config.apdb.value)