Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ("PackageAlertsConfig", "PackageAlertsTask") 

23 

24import os 

25 

26import lsst.afw.fits as afwFits 

27import lsst.alert.packet as alertPack 

28import lsst.geom as geom 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils import getPackageDir 

32 

33 

34"""Methods for packaging Apdb and Pipelines data into Avro alerts. 

35""" 

36 

37 

38class PackageAlertsConfig(pexConfig.Config): 

39 """Config class for AssociationTask. 

40 """ 

41 schemaFile = pexConfig.Field( 

42 dtype=str, 

43 doc="Schema definition file for the avro alerts.", 

44 default=os.path.join(getPackageDir("alert_packet"), 

45 "schema", 

46 *[str(x) for x in alertPack.get_latest_schema_version()], 

47 "lsst.alert.avsc") 

48 ) 

49 minCutoutSize = pexConfig.RangeField( 

50 dtype=int, 

51 min=0, 

52 max=1000, 

53 default=30, 

54 doc="Dimension of the square image cutouts to package in the alert." 

55 ) 

56 alertWriteLocation = pexConfig.Field( 

57 dtype=str, 

58 doc="Location to write alerts to.", 

59 default=os.path.join(os.getcwd(), "alerts"), 

60 ) 

61 

62 

63class PackageAlertsTask(pipeBase.Task): 

64 """Tasks for packaging Dia and Pipelines data into Avro alert packages. 

65 """ 

66 ConfigClass = PackageAlertsConfig 

67 _DefaultName = "packageAlerts" 

68 

69 def __init__(self, **kwargs): 

70 super().__init__(**kwargs) 

71 self.alertSchema = alertPack.Schema.from_file(self.config.schemaFile) 

72 os.makedirs(self.config.alertWriteLocation, exist_ok=True) 

73 

74 def run(self, 

75 diaSourceCat, 

76 diaObjectCat, 

77 diaSrcHistory, 

78 diffIm, 

79 template, 

80 ccdExposureIdBits): 

81 """Package DiaSources/Object and exposure data into Avro alerts. 

82 

83 Writes Avro alerts to a location determined by the 

84 ``alertWriteLocation`` configurable. 

85 

86 Parameters 

87 ---------- 

88 diaSourceCat : `pandas.DataFrame` 

89 New DiaSources to package. DataFrame should be indexed on 

90 ``["diaObjectId", "filterName", "diaSourceId"]`` 

91 diaObjectCat : `pandas.DataFrame` 

92 New and updated DiaObjects matched to the new DiaSources. DataFrame 

93 is indexed on ``["diaObjectId"]`` 

94 diaSrcHistory : `pandas.DataFrame` 

95 12 month history of DiaSources matched to the DiaObjects. Excludes 

96 the newest DiaSource and is indexed on 

97 ``["diaObjectId", "filterName", "diaSourceId"]`` 

98 diffIm : `lsst.afw.image.ExposureF` 

99 Difference image the sources in ``diaSourceCat`` were detected in. 

100 template : `lsst.afw.image.ExposureF` or `None` 

101 Template image used to create the ``diffIm``. 

102 ccdExposureIdBits : `int` 

103 Number of bits used in the ccdVisitId. 

104 """ 

105 alerts = [] 

106 self._patchDiaSources(diaSourceCat) 

107 self._patchDiaSources(diaSrcHistory) 

108 self._patchDiaObjects(diaObjectCat) 

109 ccdVisitId = diffIm.getInfo().getVisitInfo().getExposureId() 

110 for srcIndex, diaSource in diaSourceCat.iterrows(): 

111 # Get all diaSources for the associated diaObject. 

112 diaObject = diaObjectCat.loc[srcIndex[0]] 

113 if diaObject["nDiaSources"] > 1: 

114 objSourceHistory = diaSrcHistory.loc[srcIndex[0]] 

115 else: 

116 objSourceHistory = None 

117 sphPoint = geom.SpherePoint(diaSource["ra"], 

118 diaSource["decl"], 

119 geom.degrees) 

120 cutoutBBox = self.createDiaSourceBBox(diaSource["bboxSize"]) 

121 diffImCutout = diffIm.getCutout(sphPoint, cutoutBBox) 

122 templateCutout = None 

123 # TODO: Create alertIds DM-24858 

124 alertId = diaSource["diaSourceId"] 

125 alerts.append( 

126 self.makeAlertDict(alertId, 

127 diaSource, 

128 diaObject, 

129 objSourceHistory, 

130 diffImCutout, 

131 templateCutout)) 

132 with open(os.path.join(self.config.alertWriteLocation, 

133 f"{ccdVisitId}.avro"), 

134 "wb") as f: 

135 self.alertSchema.store_alerts(f, alerts) 

136 

137 def _patchDiaSources(self, diaSources): 

138 """Add the ``programId`` column to the data and change currently 

139 grouped alert data to ``None``. 

140 

141 TODO: The need to change these column values to ``None`` can be removed 

142 after DM-24696 is merged. 

143 

144 Parameters 

145 ---------- 

146 diaSources : `pandas.DataFrame` 

147 DataFrame of DiaSources to patch. 

148 """ 

149 diaSources["programId"] = 0 

150 diaSources["ra_decl_Cov"] = None 

151 diaSources["x_y_Cov"] = None 

152 diaSources["ps_Cov"] = None 

153 diaSources["trail_Cov"] = None 

154 diaSources["dip_Cov"] = None 

155 diaSources["i_cov"] = None 

156 

157 def _patchDiaObjects(self, diaObjects): 

158 """Change currently grouped alert data to ``None``. 

159 

160 TODO: The need to change these column values to ``None`` can be removed 

161 after DM-24696 is merged. 

162 

163 Parameters 

164 ---------- 

165 diaObjects : `pandas.DataFrame` 

166 DataFrame of DiaObjects to patch. 

167 """ 

168 diaObjects["ra_decl_Cov"] = None 

169 diaObjects["pm_parallax_Cov"] = None 

170 diaObjects["uLcPeriodic"] = None 

171 diaObjects["gLcPeriodic"] = None 

172 diaObjects["rLcPeriodic"] = None 

173 diaObjects["iLcPeriodic"] = None 

174 diaObjects["zLcPeriodic"] = None 

175 diaObjects["yLcPeriodic"] = None 

176 diaObjects["uLcNonPeriodic"] = None 

177 diaObjects["gLcNonPeriodic"] = None 

178 diaObjects["rLcNonPeriodic"] = None 

179 diaObjects["iLcNonPeriodic"] = None 

180 diaObjects["zLcNonPeriodic"] = None 

181 diaObjects["yLcNonPeriodic"] = None 

182 

183 def createDiaSourceBBox(self, bboxSize): 

184 """Create a bounding box for the cutouts given the size of the square 

185 BBox that covers the source footprint. 

186 

187 Parameters 

188 ---------- 

189 bboxSize : `int` 

190 Size of a side of the square bounding box in pixels. 

191 

192 Returns 

193 ------- 

194 bbox : `lsst.geom.Extent2I` 

195 Geom object representing the size of the bounding box. 

196 """ 

197 if bboxSize < self.config.minCutoutSize: 

198 bbox = geom.Extent2I(self.config.minCutoutSize, 

199 self.config.minCutoutSize) 

200 else: 

201 bbox = geom.Extent2I(bboxSize, bboxSize) 

202 return bbox 

203 

204 def makeAlertDict(self, 

205 alertId, 

206 diaSource, 

207 diaObject, 

208 objDiaSrcHistory, 

209 diffImCutout, 

210 templateCutout): 

211 """Convert data and package into a dictionary alert. 

212 

213 Parameters 

214 ---------- 

215 diaSource : `pandas.DataFrame` 

216 New single DiaSource to package. 

217 diaObject : `pandas.DataFrame` 

218 DiaObject that ``diaSource`` is matched to. 

219 objDiaSrcHistory : `pandas.DataFrame` 

220 12 month history of ``diaObject`` excluding the latest DiaSource. 

221 diffImCutout : `lsst.afw.image.ExposureF` 

222 Cutout of the difference image around the location of ``diaSource`` 

223 with a size set by the ``cutoutSize`` configurable. 

224 templateCutout : `lsst.afw.image.ExposureF` 

225 Cutout of the template image around the location of ``diaSource`` 

226 with a size set by the ``cutoutSize`` configurable. 

227 """ 

228 alert = dict() 

229 alert['alertId'] = alertId 

230 alert['diaSource'] = diaSource.to_dict() 

231 

232 if objDiaSrcHistory is None: 

233 alert['prvDiaSources'] = objDiaSrcHistory 

234 else: 

235 alert['prvDiaSources'] = objDiaSrcHistory.to_dict("records") 

236 

237 alert['prvDiaForcedSources'] = None 

238 alert['prvDiaNondetectionLimits'] = None 

239 

240 alert['diaObject'] = diaObject.to_dict() 

241 

242 alert['ssObject'] = None 

243 

244 # TODO: fileName to be removed in future Avro schemas. DM-24696 

245 alert['cutoutDifference'] = { 

246 'fileName': '', 

247 'stampData': self.makeCutoutBytes(diffImCutout), 

248 } 

249 # TODO: add template cutouts in DM-24327 

250 alert["cutoutTemplate"] = None 

251 

252 return alert 

253 

254 def makeCutoutBytes(self, cutout): 

255 """Serialize a cutout into bytes. 

256 

257 Parameters 

258 ---------- 

259 cutout : `lsst.afw.image.ExposureF` 

260 Cutout to serialize. 

261 

262 Returns 

263 ------- 

264 coutputBytes : `bytes` 

265 Input cutout serialized into byte data. 

266 """ 

267 mgr = afwFits.MemFileManager() 

268 cutout.writeFits(mgr) 

269 return mgr.getData()