Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# Developed for the LSST Data Management System. 

3# This product includes software developed by the LSST Project 

4# (http://www.lsst.org). 

5# See the COPYRIGHT file at the top-level directory of this distribution 

6# for details of code ownership. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the GNU General Public License 

19# along with this program. If not, see <http://www.gnu.org/licenses/>. 

20# 

21 

22"""Classes for taking science pipeline outputs and creating data products for 

23use in ap_association and the alert production database (APDB). 

24""" 

25 

26__all__ = ["MapApDataConfig", "MapApDataTask", 

27 "MapDiaSourceConfig", "MapDiaSourceTask", 

28 "UnpackApdbFlags"] 

29 

30import numpy as np 

31import os 

32import yaml 

33 

34import lsst.afw.table as afwTable 

35from lsst.daf.base import DateTime 

36import lsst.pipe.base as pipeBase 

37import lsst.pex.config as pexConfig 

38from lsst.pex.exceptions import RuntimeError 

39from lsst.utils import getPackageDir 

40from .afwUtils import make_dia_source_schema 

41 

42 

43class MapApDataConfig(pexConfig.Config): 

44 """Configuration for the generic MapApDataTask class. 

45 """ 

46 copyColumns = pexConfig.DictField( 

47 keytype=str, 

48 itemtype=str, 

49 doc="Mapping of input SciencePipelines columns to output DPDD " 

50 "columns.", 

51 default={"id": "id", 

52 "parent": "parent", 

53 "coord_ra": "coord_ra", 

54 "coord_dec": "coord_dec"} 

55 ) 

56 

57 

58class MapApDataTask(pipeBase.Task): 

59 """Generic mapper class for copying values from a science pipelines catalog 

60 into a product for use in ap_association or the APDB. 

61 """ 

62 ConfigClass = MapApDataConfig 

63 _DefaultName = "mapApDataTask" 

64 

65 def __init__(self, inputSchema, outputSchema, **kwargs): 

66 pipeBase.Task.__init__(self, **kwargs) 

67 self.inputSchema = inputSchema 

68 self.outputSchema = outputSchema 

69 

70 self.mapper = afwTable.SchemaMapper(inputSchema, outputSchema) 

71 

72 for inputName, outputName in self.config.copyColumns.items(): 

73 self.mapper.addMapping( 

74 self.inputSchema.find(inputName).key, 

75 outputName, 

76 True) 

77 

78 def run(self, inputCatalog, exposure=None): 

79 """Copy data from the inputCatalog into an output catalog with 

80 requested columns. 

81 

82 Parameters 

83 ---------- 

84 inputCatalog: `lsst.afw.table.SourceCatalog` 

85 Input catalog with data to be copied into new output catalog. 

86 

87 Returns 

88 ------- 

89 outputCatalog: `lsst.afw.table.SourceCatalog` 

90 Output catalog with data copied from input and new column names. 

91 """ 

92 outputCatalog = afwTable.SourceCatalog(self.outputSchema) 

93 outputCatalog.extend(inputCatalog, self.mapper) 

94 

95 if not outputCatalog.isContiguous(): 

96 raise RuntimeError("Output catalogs must be contiguous.") 

97 

98 return outputCatalog 

99 

100 

101class MapDiaSourceConfig(pexConfig.Config): 

102 """Config for the DiaSourceMapperTask 

103 """ 

104 copyColumns = pexConfig.DictField( 

105 keytype=str, 

106 itemtype=str, 

107 doc="Mapping of input SciencePipelines columns to output DPDD " 

108 "columns.", 

109 default={"id": "id", 

110 "parent": "parent", 

111 "coord_ra": "coord_ra", 

112 "coord_dec": "coord_dec", 

113 "slot_Centroid_x": "x", 

114 "slot_Centroid_xErr": "xErr", 

115 "slot_Centroid_y": "y", 

116 "slot_Centroid_yErr": "yErr", 

117 "slot_ApFlux_instFlux": "apFlux", 

118 "slot_ApFlux_instFluxErr": "apFluxErr", 

119 "slot_PsfFlux_instFlux": "psFlux", 

120 "slot_PsfFlux_instFluxErr": "psFluxErr", 

121 "ip_diffim_DipoleFit_orientation": "dipAngle", 

122 "ip_diffim_DipoleFit_chi2dof": "dipChi2", 

123 "ip_diffim_forced_PsfFlux_instFlux": "totFlux", 

124 "ip_diffim_forced_PsfFlux_instFluxErr": "totFluxErr", 

125 "ip_diffim_DipoleFit_flag_classification": "isDipole", 

126 "slot_Shape_xx": "ixx", 

127 "slot_Shape_xxErr": "ixxErr", 

128 "slot_Shape_yy": "iyy", 

129 "slot_Shape_yyErr": "iyyErr", 

130 "slot_Shape_xy": "ixy", 

131 "slot_Shape_xyErr": "ixyErr", 

132 "slot_PsfShape_xx": "ixxPSF", 

133 "slot_PsfShape_yy": "iyyPSF", 

134 "slot_PsfShape_xy": "ixyPSF"} 

135 ) 

136 calibrateColumns = pexConfig.ListField( 

137 dtype=str, 

138 doc="Flux columns in the input catalog to calibrate.", 

139 default=["slot_ApFlux", "slot_PsfFlux", "ip_diffim_forced_PsfFlux"] 

140 ) 

141 flagMap = pexConfig.Field( 

142 dtype=str, 

143 doc="Yaml file specifying SciencePipelines flag fields to bit packs.", 

144 default=os.path.join(getPackageDir("ap_association"), 

145 "data", 

146 "association-flag-map.yaml"), 

147 ) 

148 dipFluxPrefix = pexConfig.Field( 

149 dtype=str, 

150 doc="Prefix of the Dipole measurement column containing negative and " 

151 "positive flux lobes.", 

152 default="ip_diffim_DipoleFit", 

153 ) 

154 dipSepColumn = pexConfig.Field( 

155 dtype=str, 

156 doc="Column of the separation of the negative and positive poles of " 

157 "the dipole.", 

158 default="ip_diffim_DipoleFit_separation" 

159 ) 

160 

161 

162class MapDiaSourceTask(MapApDataTask): 

163 """Task specific for copying columns from science pipelines catalogs, 

164 calibrating them, for use in ap_association and the APDB. 

165 

166 This task also copies information from the exposure such as the ExpsoureId 

167 and the exposure date as specified in the DPDD. 

168 """ 

169 

170 ConfigClass = MapDiaSourceConfig 

171 _DefaultName = "mapDiaSourceTask" 

172 

173 def __init__(self, inputSchema, **kwargs): 

174 MapApDataTask.__init__(self, 

175 inputSchema=inputSchema, 

176 outputSchema=make_dia_source_schema(), 

177 **kwargs) 

178 self._create_bit_pack_mappings() 

179 

180 def _create_bit_pack_mappings(self): 

181 """Setup all flag bit packings. 

182 """ 

183 self.bit_pack_columns = [] 

184 with open(self.config.flagMap) as yaml_stream: 

185 table_list = list(yaml.safe_load_all(yaml_stream)) 

186 for table in table_list: 

187 if table['tableName'] == 'DiaSource': 

188 self.bit_pack_columns = table['columns'] 

189 break 

190 

191 # Test that all flags requested are present in both the input and 

192 # output schemas. 

193 for outputFlag in self.bit_pack_columns: 

194 try: 

195 self.outputSchema.find(outputFlag['columnName']) 

196 except KeyError: 

197 raise KeyError( 

198 "Requested column %s not found in MapDiaSourceTask output " 

199 "schema. Please check that the requested output column " 

200 "exists." % outputFlag['columnName']) 

201 bitList = outputFlag['bitList'] 

202 for bit in bitList: 

203 try: 

204 self.inputSchema.find(bit['name']) 

205 except KeyError: 

206 raise KeyError( 

207 "Requested column %s not found in MapDiaSourceTask input " 

208 "schema. Please check that the requested input column " 

209 "exists." % outputFlag['columnName']) 

210 

211 def run(self, inputCatalog, exposure, return_pandas=False): 

212 """Copy data from the inputCatalog into an output catalog with 

213 requested columns. 

214 

215 Parameters 

216 ---------- 

217 inputCatalog : `lsst.afw.table.SourceCatalog` 

218 Input catalog with data to be copied into new output catalog. 

219 exposure: `lsst.afw.image.Exposure` 

220 Exposure with containing the PhotoCalib object relevant to this 

221 catalog. 

222 return_pandas : `bool` 

223 Return `pandas.DataFrame` instead of `lsst.afw.table.SourceCatalog` 

224 

225 Returns 

226 ------- 

227 outputCatalog: `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` 

228 Output catalog with data copied from input and new column names. 

229 """ 

230 visit_info = exposure.getInfo().getVisitInfo() 

231 ccdVisitId = visit_info.getExposureId() 

232 midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD) 

233 filterId = exposure.getFilter().getId() 

234 # canonical name should always be the abstract filter (in Gen 3 sense) 

235 filterName = exposure.getFilter().getCanonicalName() 

236 wcs = exposure.getWcs() 

237 

238 photoCalib = exposure.getPhotoCalib() 

239 

240 outputCatalog = afwTable.SourceCatalog(self.outputSchema) 

241 outputCatalog.reserve(len(inputCatalog)) 

242 

243 for inputRecord in inputCatalog: 

244 outputRecord = outputCatalog.addNew() 

245 outputRecord.assign(inputRecord, self.mapper) 

246 self.calibrateFluxes(inputRecord, outputRecord, photoCalib) 

247 self.computeDipoleFluxes(inputRecord, outputRecord, photoCalib) 

248 self.computeDipoleSep(inputRecord, outputRecord, wcs) 

249 self.bitPackFlags(inputRecord, outputRecord) 

250 self.computeBBoxSize(inputRecord, outputRecord) 

251 outputRecord.set("ccdVisitId", ccdVisitId) 

252 outputRecord.set("midPointTai", midPointTaiMJD) 

253 outputRecord.set("filterId", filterId) 

254 outputRecord.set("filterName", filterName) 

255 

256 if not outputCatalog.isContiguous(): 

257 raise RuntimeError("Output catalogs must be contiguous.") 

258 

259 if return_pandas: 

260 return self._convert_to_pandas(outputCatalog) 

261 return outputCatalog 

262 

263 def calibrateFluxes(self, inputRecord, outputRecord, photoCalib): 

264 """Copy flux values into an output record and calibrate them. 

265 

266 Parameters 

267 ---------- 

268 inputRecord : `lsst.afw.table.SourceRecord` 

269 Record to copy flux values from. 

270 outputRecord : `lsst.afw.table.SourceRecord` 

271 Record to copy and calibrate values into. 

272 photoCalib : `lsst.afw.image.PhotoCalib` 

273 Calibration object from the difference exposure. 

274 """ 

275 for col_name in self.config.calibrateColumns: 

276 meas = photoCalib.instFluxToNanojansky(inputRecord, col_name) 

277 outputRecord.set(self.config.copyColumns[col_name + "_instFlux"], 

278 meas.value) 

279 outputRecord.set( 

280 self.config.copyColumns[col_name + "_instFluxErr"], 

281 meas.error) 

282 

283 def computeDipoleFluxes(self, inputRecord, outputRecord, photoCalib): 

284 """Calibrate and compute dipole mean flux and diff flux. 

285 

286 Parameters 

287 ---------- 

288 inputRecord : `lsst.afw.table.SourceRecord` 

289 Record to copy flux values from. 

290 outputRecord : `lsst.afw.table.SourceRecord` 

291 Record to copy and calibrate values into. 

292 photoCalib `lsst.afw.image.PhotoCalib` 

293 Calibration object from the difference exposure. 

294 """ 

295 

296 neg_meas = photoCalib.instFluxToNanojansky( 

297 inputRecord, self.config.dipFluxPrefix + "_neg") 

298 pos_meas = photoCalib.instFluxToNanojansky( 

299 inputRecord, self.config.dipFluxPrefix + "_pos") 

300 outputRecord.set( 

301 "dipMeanFlux", 

302 0.5 * (np.abs(neg_meas.value) + np.abs(pos_meas.value))) 

303 outputRecord.set( 

304 "dipMeanFluxErr", 

305 0.5 * np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2)) 

306 outputRecord.set( 

307 "dipFluxDiff", 

308 np.abs(pos_meas.value) - np.abs(neg_meas.value)) 

309 outputRecord.set( 

310 "dipFluxDiffErr", 

311 np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2)) 

312 

313 def computeDipoleSep(self, inputRecord, outputRecord, wcs): 

314 """Convert the dipole separation from pixels to arcseconds. 

315 

316 Parameters 

317 ---------- 

318 inputRecord : `lsst.afw.table.SourceRecord` 

319 Record to copy flux values from. 

320 outputRecord : `lsst.afw.table.SourceRecord` 

321 Record to copy and calibrate values into. 

322 wcs : `lsst.afw.geom.SkyWcs` 

323 Wcs of image inputRecords was observed. 

324 """ 

325 pixScale = wcs.getPixelScale(inputRecord.getCentroid()) 

326 dipSep = pixScale * inputRecord.get(self.config.dipSepColumn) 

327 outputRecord.set("dipLength", dipSep.asArcseconds()) 

328 

329 def bitPackFlags(self, inputRecord, outputRecord): 

330 """Pack requested flag columns in inputRecord into single columns in 

331 outputRecord. 

332 

333 Parameters 

334 ---------- 

335 inputRecord : `lsst.afw.table.SourceRecord` 

336 Record to copy flux values from. 

337 outputRecord : `lsst.afw.table.SourceRecord` 

338 Record to copy and calibrate values into. 

339 """ 

340 for outputFlag in self.bit_pack_columns: 

341 bitList = outputFlag['bitList'] 

342 value = 0 

343 for bit in bitList: 

344 value += inputRecord[bit['name']] * 2 ** bit['bit'] 

345 outputRecord.set(outputFlag['columnName'], value) 

346 

347 def computeBBoxSize(self, inputRecord, outputRecord): 

348 """Compute the size of a square bbox that fully contains the detection 

349 footprint. 

350 

351 Parameters 

352 ---------- 

353 inputRecord : `lsst.afw.table.SourceRecord` 

354 Record to copy flux values from. 

355 outputRecord : `lsst.afw.table.SourceRecord` 

356 Record to copy and calibrate values into. 

357 """ 

358 footprintBBox = inputRecord.getFootprint().getBBox() 

359 recX = inputRecord.getCentroid().x 

360 recY = inputRecord.getCentroid().y 

361 bboxSize = int( 

362 np.ceil(2 * np.max(np.fabs([footprintBBox.maxX - recX, 

363 footprintBBox.minX - recX, 

364 footprintBBox.maxY - recY, 

365 footprintBBox.minY - recY])))) 

366 outputRecord.set("bboxSize", bboxSize) 

367 

368 def _convert_to_pandas(self, inputCatalog): 

369 """Convert input afw table to pandas. 

370 

371 Using afwTable.toAstropy().to_pandas() alone is not sufficient to 

372 properly store data in the Apdb. We must also convert the RA/DEC values 

373 from radians to degrees and rename several columns. 

374 

375 Parameters 

376 ---------- 

377 inputCatalog : `lsst.afw.table.SourceCatalog` 

378 Catalog to convert to panads and rename columns. 

379 

380 Returns 

381 ------- 

382 catalog : `pandas.DataFrame` 

383 """ 

384 catalog = inputCatalog.asAstropy().to_pandas() 

385 catalog.rename(columns={"coord_ra": "ra", 

386 "coord_dec": "decl", 

387 "id": "diaSourceId", 

388 "parent": "parentDiaSourceId"}, 

389 inplace=True) 

390 catalog["ra"] = np.degrees(catalog["ra"]) 

391 catalog["decl"] = np.degrees(catalog["decl"]) 

392 

393 return catalog 

394 

395 

396class UnpackApdbFlags: 

397 """Class for unpacking bits from integer flag fields stored in the Apdb. 

398 

399 Attributes 

400 ---------- 

401 flag_map_file : `str` 

402 Absolute or relative path to a yaml file specifiying mappings of flags 

403 to integer bits. 

404 table_name : `str` 

405 Name of the Apdb table the integer bit data are coming from. 

406 """ 

407 

408 def __init__(self, flag_map_file, table_name): 

409 self.bit_pack_columns = [] 

410 with open(flag_map_file) as yaml_stream: 

411 table_list = list(yaml.safe_load_all(yaml_stream)) 

412 for table in table_list: 

413 if table['tableName'] == table_name: 

414 self.bit_pack_columns = table['columns'] 

415 break 

416 

417 self.output_flag_columns = {} 

418 

419 for column in self.bit_pack_columns: 

420 names = [] 

421 for bit in column["bitList"]: 

422 names.append((bit["name"], np.bool)) 

423 self.output_flag_columns[column["columnName"]] = names 

424 

425 def unpack(self, input_flag_values, flag_name): 

426 """Determine individual boolean flags from an input array of unsigned 

427 ints. 

428 

429 Parameters 

430 ---------- 

431 input_flag_values : array-like of type uint 

432 Input integer flags to unpack. 

433 flag_name : `str` 

434 Apdb column name of integer flags to unpack. Names of packed int 

435 flags are given by the flag_map_file. 

436 

437 Returns 

438 ------- 

439 output_flags : `numpy.ndarray` 

440 Numpy named tuple of booleans. 

441 """ 

442 bit_names_types = self.output_flag_columns[flag_name] 

443 output_flags = np.zeros(len(input_flag_values), dtype=bit_names_types) 

444 

445 for bit_idx, (bit_name, dtypes) in enumerate(bit_names_types): 

446 masked_bits = np.bitwise_and(input_flag_values, 2 ** bit_idx) 

447 output_flags[bit_name] = masked_bits 

448 

449 return output_flags