Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# Developed for the LSST Data Management System. 

3# This product includes software developed by the LSST Project 

4# (http://www.lsst.org). 

5# See the COPYRIGHT file at the top-level directory of this distribution 

6# for details of code ownership. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the GNU General Public License 

19# along with this program. If not, see <http://www.gnu.org/licenses/>. 

20# 

21 

22"""Classes for taking science pipeline outputs and creating data products for 

23use in ap_association and the alert production database (APDB). 

24""" 

25 

26__all__ = ["MapApDataConfig", "MapApDataTask", 

27 "MapDiaSourceConfig", "MapDiaSourceTask", 

28 "UnpackApdbFlags"] 

29 

30import numpy as np 

31import os 

32import yaml 

33 

34import lsst.afw.table as afwTable 

35from lsst.daf.base import DateTime 

36import lsst.pipe.base as pipeBase 

37import lsst.pex.config as pexConfig 

38from lsst.pex.exceptions import RuntimeError 

39from lsst.utils import getPackageDir 

40from .afwUtils import make_dia_source_schema 

41 

42 

43class MapApDataConfig(pexConfig.Config): 

44 """Configuration for the generic MapApDataTask class. 

45 """ 

46 copyColumns = pexConfig.DictField( 

47 keytype=str, 

48 itemtype=str, 

49 doc="Mapping of input SciencePipelines columns to output DPDD " 

50 "columns.", 

51 default={"id": "id", 

52 "parent": "parent", 

53 "coord_ra": "coord_ra", 

54 "coord_dec": "coord_dec"} 

55 ) 

56 

57 

58class MapApDataTask(pipeBase.Task): 

59 """Generic mapper class for copying values from a science pipelines catalog 

60 into a product for use in ap_association or the APDB. 

61 """ 

62 ConfigClass = MapApDataConfig 

63 _DefaultName = "mapApDataTask" 

64 

65 def __init__(self, inputSchema, outputSchema, **kwargs): 

66 pipeBase.Task.__init__(self, **kwargs) 

67 self.inputSchema = inputSchema 

68 self.outputSchema = outputSchema 

69 

70 self.mapper = afwTable.SchemaMapper(inputSchema, outputSchema) 

71 

72 for inputName, outputName in self.config.copyColumns.items(): 

73 self.mapper.addMapping( 

74 self.inputSchema.find(inputName).key, 

75 outputName, 

76 True) 

77 

78 @pipeBase.timeMethod 

79 def run(self, inputCatalog, exposure=None): 

80 """Copy data from the inputCatalog into an output catalog with 

81 requested columns. 

82 

83 Parameters 

84 ---------- 

85 inputCatalog: `lsst.afw.table.SourceCatalog` 

86 Input catalog with data to be copied into new output catalog. 

87 

88 Returns 

89 ------- 

90 outputCatalog: `lsst.afw.table.SourceCatalog` 

91 Output catalog with data copied from input and new column names. 

92 """ 

93 outputCatalog = afwTable.SourceCatalog(self.outputSchema) 

94 outputCatalog.extend(inputCatalog, self.mapper) 

95 

96 if not outputCatalog.isContiguous(): 

97 raise RuntimeError("Output catalogs must be contiguous.") 

98 

99 return outputCatalog 

100 

101 

102class MapDiaSourceConfig(pexConfig.Config): 

103 """Config for the DiaSourceMapperTask 

104 """ 

105 copyColumns = pexConfig.DictField( 

106 keytype=str, 

107 itemtype=str, 

108 doc="Mapping of input SciencePipelines columns to output DPDD " 

109 "columns.", 

110 default={"id": "id", 

111 "parent": "parent", 

112 "coord_ra": "coord_ra", 

113 "coord_dec": "coord_dec", 

114 "slot_Centroid_x": "x", 

115 "slot_Centroid_xErr": "xErr", 

116 "slot_Centroid_y": "y", 

117 "slot_Centroid_yErr": "yErr", 

118 "slot_ApFlux_instFlux": "apFlux", 

119 "slot_ApFlux_instFluxErr": "apFluxErr", 

120 "slot_PsfFlux_instFlux": "psFlux", 

121 "slot_PsfFlux_instFluxErr": "psFluxErr", 

122 "ip_diffim_DipoleFit_orientation": "dipAngle", 

123 "ip_diffim_DipoleFit_chi2dof": "dipChi2", 

124 "ip_diffim_forced_PsfFlux_instFlux": "totFlux", 

125 "ip_diffim_forced_PsfFlux_instFluxErr": "totFluxErr", 

126 "ip_diffim_DipoleFit_flag_classification": "isDipole", 

127 "slot_Shape_xx": "ixx", 

128 "slot_Shape_xxErr": "ixxErr", 

129 "slot_Shape_yy": "iyy", 

130 "slot_Shape_yyErr": "iyyErr", 

131 "slot_Shape_xy": "ixy", 

132 "slot_Shape_xyErr": "ixyErr", 

133 "slot_PsfShape_xx": "ixxPSF", 

134 "slot_PsfShape_yy": "iyyPSF", 

135 "slot_PsfShape_xy": "ixyPSF"} 

136 ) 

137 calibrateColumns = pexConfig.ListField( 

138 dtype=str, 

139 doc="Flux columns in the input catalog to calibrate.", 

140 default=["slot_ApFlux", "slot_PsfFlux", "ip_diffim_forced_PsfFlux"] 

141 ) 

142 flagMap = pexConfig.Field( 

143 dtype=str, 

144 doc="Yaml file specifying SciencePipelines flag fields to bit packs.", 

145 default=os.path.join(getPackageDir("ap_association"), 

146 "data", 

147 "association-flag-map.yaml"), 

148 ) 

149 dipFluxPrefix = pexConfig.Field( 

150 dtype=str, 

151 doc="Prefix of the Dipole measurement column containing negative and " 

152 "positive flux lobes.", 

153 default="ip_diffim_DipoleFit", 

154 ) 

155 dipSepColumn = pexConfig.Field( 

156 dtype=str, 

157 doc="Column of the separation of the negative and positive poles of " 

158 "the dipole.", 

159 default="ip_diffim_DipoleFit_separation" 

160 ) 

161 

162 

163class MapDiaSourceTask(MapApDataTask): 

164 """Task specific for copying columns from science pipelines catalogs, 

165 calibrating them, for use in ap_association and the APDB. 

166 

167 This task also copies information from the exposure such as the ExpsoureId 

168 and the exposure date as specified in the DPDD. 

169 """ 

170 

171 ConfigClass = MapDiaSourceConfig 

172 _DefaultName = "mapDiaSourceTask" 

173 

174 def __init__(self, inputSchema, **kwargs): 

175 MapApDataTask.__init__(self, 

176 inputSchema=inputSchema, 

177 outputSchema=make_dia_source_schema(), 

178 **kwargs) 

179 self._create_bit_pack_mappings() 

180 

181 def _create_bit_pack_mappings(self): 

182 """Setup all flag bit packings. 

183 """ 

184 self.bit_pack_columns = [] 

185 with open(self.config.flagMap) as yaml_stream: 

186 table_list = list(yaml.safe_load_all(yaml_stream)) 

187 for table in table_list: 

188 if table['tableName'] == 'DiaSource': 

189 self.bit_pack_columns = table['columns'] 

190 break 

191 

192 # Test that all flags requested are present in both the input and 

193 # output schemas. 

194 for outputFlag in self.bit_pack_columns: 

195 try: 

196 self.outputSchema.find(outputFlag['columnName']) 

197 except KeyError: 

198 raise KeyError( 

199 "Requested column %s not found in MapDiaSourceTask output " 

200 "schema. Please check that the requested output column " 

201 "exists." % outputFlag['columnName']) 

202 bitList = outputFlag['bitList'] 

203 for bit in bitList: 

204 try: 

205 self.inputSchema.find(bit['name']) 

206 except KeyError: 

207 raise KeyError( 

208 "Requested column %s not found in MapDiaSourceTask input " 

209 "schema. Please check that the requested input column " 

210 "exists." % outputFlag['columnName']) 

211 

212 @pipeBase.timeMethod 

213 def run(self, inputCatalog, exposure, return_pandas=False): 

214 """Copy data from the inputCatalog into an output catalog with 

215 requested columns. 

216 

217 Parameters 

218 ---------- 

219 inputCatalog : `lsst.afw.table.SourceCatalog` 

220 Input catalog with data to be copied into new output catalog. 

221 exposure: `lsst.afw.image.Exposure` 

222 Exposure with containing the PhotoCalib object relevant to this 

223 catalog. 

224 return_pandas : `bool` 

225 Return `pandas.DataFrame` instead of `lsst.afw.table.SourceCatalog` 

226 

227 Returns 

228 ------- 

229 outputCatalog: `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` 

230 Output catalog with data copied from input and new column names. 

231 """ 

232 visit_info = exposure.getInfo().getVisitInfo() 

233 ccdVisitId = visit_info.getExposureId() 

234 midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD) 

235 filterId = exposure.getFilter().getId() 

236 # TODO DM-27170: fix this [0] workaround which gets a single character 

237 # representation of the band. 

238 filterName = exposure.getFilter().getCanonicalName()[0] 

239 wcs = exposure.getWcs() 

240 

241 photoCalib = exposure.getPhotoCalib() 

242 

243 outputCatalog = afwTable.SourceCatalog(self.outputSchema) 

244 outputCatalog.reserve(len(inputCatalog)) 

245 

246 for inputRecord in inputCatalog: 

247 outputRecord = outputCatalog.addNew() 

248 outputRecord.assign(inputRecord, self.mapper) 

249 self.calibrateFluxes(inputRecord, outputRecord, photoCalib) 

250 self.computeDipoleFluxes(inputRecord, outputRecord, photoCalib) 

251 self.computeDipoleSep(inputRecord, outputRecord, wcs) 

252 self.bitPackFlags(inputRecord, outputRecord) 

253 self.computeBBoxSize(inputRecord, outputRecord) 

254 outputRecord.set("ccdVisitId", ccdVisitId) 

255 outputRecord.set("midPointTai", midPointTaiMJD) 

256 outputRecord.set("filterId", filterId) 

257 outputRecord.set("filterName", filterName) 

258 

259 if not outputCatalog.isContiguous(): 

260 raise RuntimeError("Output catalogs must be contiguous.") 

261 

262 if return_pandas: 

263 return self._convert_to_pandas(outputCatalog) 

264 return outputCatalog 

265 

266 def calibrateFluxes(self, inputRecord, outputRecord, photoCalib): 

267 """Copy flux values into an output record and calibrate them. 

268 

269 Parameters 

270 ---------- 

271 inputRecord : `lsst.afw.table.SourceRecord` 

272 Record to copy flux values from. 

273 outputRecord : `lsst.afw.table.SourceRecord` 

274 Record to copy and calibrate values into. 

275 photoCalib : `lsst.afw.image.PhotoCalib` 

276 Calibration object from the difference exposure. 

277 """ 

278 for col_name in self.config.calibrateColumns: 

279 meas = photoCalib.instFluxToNanojansky(inputRecord, col_name) 

280 outputRecord.set(self.config.copyColumns[col_name + "_instFlux"], 

281 meas.value) 

282 outputRecord.set( 

283 self.config.copyColumns[col_name + "_instFluxErr"], 

284 meas.error) 

285 

286 def computeDipoleFluxes(self, inputRecord, outputRecord, photoCalib): 

287 """Calibrate and compute dipole mean flux and diff flux. 

288 

289 Parameters 

290 ---------- 

291 inputRecord : `lsst.afw.table.SourceRecord` 

292 Record to copy flux values from. 

293 outputRecord : `lsst.afw.table.SourceRecord` 

294 Record to copy and calibrate values into. 

295 photoCalib `lsst.afw.image.PhotoCalib` 

296 Calibration object from the difference exposure. 

297 """ 

298 

299 neg_meas = photoCalib.instFluxToNanojansky( 

300 inputRecord, self.config.dipFluxPrefix + "_neg") 

301 pos_meas = photoCalib.instFluxToNanojansky( 

302 inputRecord, self.config.dipFluxPrefix + "_pos") 

303 outputRecord.set( 

304 "dipMeanFlux", 

305 0.5 * (np.abs(neg_meas.value) + np.abs(pos_meas.value))) 

306 outputRecord.set( 

307 "dipMeanFluxErr", 

308 0.5 * np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2)) 

309 outputRecord.set( 

310 "dipFluxDiff", 

311 np.abs(pos_meas.value) - np.abs(neg_meas.value)) 

312 outputRecord.set( 

313 "dipFluxDiffErr", 

314 np.sqrt(neg_meas.error ** 2 + pos_meas.error ** 2)) 

315 

316 def computeDipoleSep(self, inputRecord, outputRecord, wcs): 

317 """Convert the dipole separation from pixels to arcseconds. 

318 

319 Parameters 

320 ---------- 

321 inputRecord : `lsst.afw.table.SourceRecord` 

322 Record to copy flux values from. 

323 outputRecord : `lsst.afw.table.SourceRecord` 

324 Record to copy and calibrate values into. 

325 wcs : `lsst.afw.geom.SkyWcs` 

326 Wcs of image inputRecords was observed. 

327 """ 

328 pixScale = wcs.getPixelScale(inputRecord.getCentroid()) 

329 dipSep = pixScale * inputRecord.get(self.config.dipSepColumn) 

330 outputRecord.set("dipLength", dipSep.asArcseconds()) 

331 

332 def bitPackFlags(self, inputRecord, outputRecord): 

333 """Pack requested flag columns in inputRecord into single columns in 

334 outputRecord. 

335 

336 Parameters 

337 ---------- 

338 inputRecord : `lsst.afw.table.SourceRecord` 

339 Record to copy flux values from. 

340 outputRecord : `lsst.afw.table.SourceRecord` 

341 Record to copy and calibrate values into. 

342 """ 

343 for outputFlag in self.bit_pack_columns: 

344 bitList = outputFlag['bitList'] 

345 value = 0 

346 for bit in bitList: 

347 value += inputRecord[bit['name']] * 2 ** bit['bit'] 

348 outputRecord.set(outputFlag['columnName'], value) 

349 

350 def computeBBoxSize(self, inputRecord, outputRecord): 

351 """Compute the size of a square bbox that fully contains the detection 

352 footprint. 

353 

354 Parameters 

355 ---------- 

356 inputRecord : `lsst.afw.table.SourceRecord` 

357 Record to copy flux values from. 

358 outputRecord : `lsst.afw.table.SourceRecord` 

359 Record to copy and calibrate values into. 

360 """ 

361 footprintBBox = inputRecord.getFootprint().getBBox() 

362 # Compute twice the size of the largest dimension of the footprint 

363 # bounding box. This is the largest footprint we should need to cover 

364 # the complete DiaSource assuming the centroid is withing the bounding 

365 # box. 

366 maxSize = 2 * np.max([footprintBBox.getWidth(), 

367 footprintBBox.getHeight()]) 

368 recX = inputRecord.getCentroid().x 

369 recY = inputRecord.getCentroid().y 

370 bboxSize = int( 

371 np.ceil(2 * np.max(np.fabs([footprintBBox.maxX - recX, 

372 footprintBBox.minX - recX, 

373 footprintBBox.maxY - recY, 

374 footprintBBox.minY - recY])))) 

375 if bboxSize > maxSize: 

376 bboxSize = maxSize 

377 outputRecord.set("bboxSize", bboxSize) 

378 

379 def _convert_to_pandas(self, inputCatalog): 

380 """Convert input afw table to pandas. 

381 

382 Using afwTable.toAstropy().to_pandas() alone is not sufficient to 

383 properly store data in the Apdb. We must also convert the RA/DEC values 

384 from radians to degrees and rename several columns. 

385 

386 Parameters 

387 ---------- 

388 inputCatalog : `lsst.afw.table.SourceCatalog` 

389 Catalog to convert to panads and rename columns. 

390 

391 Returns 

392 ------- 

393 catalog : `pandas.DataFrame` 

394 """ 

395 catalog = inputCatalog.asAstropy().to_pandas() 

396 catalog.rename(columns={"coord_ra": "ra", 

397 "coord_dec": "decl", 

398 "id": "diaSourceId", 

399 "parent": "parentDiaSourceId"}, 

400 inplace=True) 

401 catalog["ra"] = np.degrees(catalog["ra"]) 

402 catalog["decl"] = np.degrees(catalog["decl"]) 

403 

404 return catalog 

405 

406 

407class UnpackApdbFlags: 

408 """Class for unpacking bits from integer flag fields stored in the Apdb. 

409 

410 Attributes 

411 ---------- 

412 flag_map_file : `str` 

413 Absolute or relative path to a yaml file specifiying mappings of flags 

414 to integer bits. 

415 table_name : `str` 

416 Name of the Apdb table the integer bit data are coming from. 

417 """ 

418 

419 def __init__(self, flag_map_file, table_name): 

420 self.bit_pack_columns = [] 

421 with open(flag_map_file) as yaml_stream: 

422 table_list = list(yaml.safe_load_all(yaml_stream)) 

423 for table in table_list: 

424 if table['tableName'] == table_name: 

425 self.bit_pack_columns = table['columns'] 

426 break 

427 

428 self.output_flag_columns = {} 

429 

430 for column in self.bit_pack_columns: 

431 names = [] 

432 for bit in column["bitList"]: 

433 names.append((bit["name"], np.bool)) 

434 self.output_flag_columns[column["columnName"]] = names 

435 

436 def unpack(self, input_flag_values, flag_name): 

437 """Determine individual boolean flags from an input array of unsigned 

438 ints. 

439 

440 Parameters 

441 ---------- 

442 input_flag_values : array-like of type uint 

443 Input integer flags to unpack. 

444 flag_name : `str` 

445 Apdb column name of integer flags to unpack. Names of packed int 

446 flags are given by the flag_map_file. 

447 

448 Returns 

449 ------- 

450 output_flags : `numpy.ndarray` 

451 Numpy named tuple of booleans. 

452 """ 

453 bit_names_types = self.output_flag_columns[flag_name] 

454 output_flags = np.zeros(len(input_flag_values), dtype=bit_names_types) 

455 

456 for bit_idx, (bit_name, dtypes) in enumerate(bit_names_types): 

457 masked_bits = np.bitwise_and(input_flag_values, 2 ** bit_idx) 

458 output_flags[bit_name] = masked_bits 

459 

460 return output_flags