Coverage for python/lsst/ap/association/diaPipe.py: 31%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

164 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31import os 

32import pandas as pd 

33 

34import lsst.dax.apdb as daxApdb 

35from lsst.meas.base import DiaObjectCalculationTask 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38import lsst.pipe.base.connectionTypes as connTypes 

39 

40from lsst.ap.association import ( 

41 AssociationTask, 

42 DiaForcedSourceTask, 

43 LoadDiaCatalogsTask, 

44 make_dia_object_schema, 

45 make_dia_source_schema, 

46 PackageAlertsTask) 

47from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask 

48 

49__all__ = ("DiaPipelineConfig", 

50 "DiaPipelineTask", 

51 "DiaPipelineConnections") 

52 

53 

54class DiaPipelineConnections( 

55 pipeBase.PipelineTaskConnections, 

56 dimensions=("instrument", "visit", "detector"), 

57 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

58 """Butler connections for DiaPipelineTask. 

59 """ 

60 diaSourceTable = connTypes.Input( 

61 doc="Catalog of calibrated DiaSources.", 

62 name="{fakesType}{coaddName}Diff_diaSrcTable", 

63 storageClass="DataFrame", 

64 dimensions=("instrument", "visit", "detector"), 

65 ) 

66 diffIm = connTypes.Input( 

67 doc="Difference image on which the DiaSources were detected.", 

68 name="{fakesType}{coaddName}Diff_differenceExp", 

69 storageClass="ExposureF", 

70 dimensions=("instrument", "visit", "detector"), 

71 ) 

72 exposure = connTypes.Input( 

73 doc="Calibrated exposure differenced with a template image during " 

74 "image differencing.", 

75 name="calexp", 

76 storageClass="ExposureF", 

77 dimensions=("instrument", "visit", "detector"), 

78 ) 

79 warpedExposure = connTypes.Input( 

80 doc="Warped template used to create `subtractedExposure`. Not PSF " 

81 "matched.", 

82 dimensions=("instrument", "visit", "detector"), 

83 storageClass="ExposureF", 

84 name="{fakesType}{coaddName}Diff_warpedExp", 

85 ) 

86 apdbMarker = connTypes.Output( 

87 doc="Marker dataset storing the configuration of the Apdb for each " 

88 "visit/detector. Used to signal the completion of the pipeline.", 

89 name="apdb_marker", 

90 storageClass="Config", 

91 dimensions=("instrument", "visit", "detector"), 

92 ) 

93 associatedDiaSources = connTypes.Output( 

94 doc="Optional output storing the DiaSource catalog after matching, " 

95 "calibration, and standardization for insertation into the Apdb.", 

96 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

97 storageClass="DataFrame", 

98 dimensions=("instrument", "visit", "detector"), 

99 ) 

100 

101 def __init__(self, *, config=None): 

102 super().__init__(config=config) 

103 

104 if not config.doWriteAssociatedSources: 

105 self.outputs.remove("associatedDiaSources") 

106 

107 def adjustQuantum(self, inputs, outputs, label, dataId): 

108 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

109 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

110 of the activator. 

111 

112 This implementation checks to make sure that the filters in the dataset 

113 are compatible with AP processing as set by the Apdb/DPDD schema. 

114 

115 Parameters 

116 ---------- 

117 inputs : `dict` 

118 Dictionary whose keys are an input (regular or prerequisite) 

119 connection name and whose values are a tuple of the connection 

120 instance and a collection of associated `DatasetRef` objects. 

121 The exact type of the nested collections is unspecified; it can be 

122 assumed to be multi-pass iterable and support `len` and ``in``, but 

123 it should not be mutated in place. In contrast, the outer 

124 dictionaries are guaranteed to be temporary copies that are true 

125 `dict` instances, and hence may be modified and even returned; this 

126 is especially useful for delegating to `super` (see notes below). 

127 outputs : `dict` 

128 Dict of output datasets, with the same structure as ``inputs``. 

129 label : `str` 

130 Label for this task in the pipeline (should be used in all 

131 diagnostic messages). 

132 data_id : `lsst.daf.butler.DataCoordinate` 

133 Data ID for this quantum in the pipeline (should be used in all 

134 diagnostic messages). 

135 

136 Returns 

137 ------- 

138 adjusted_inputs : `dict` 

139 Dict of the same form as ``inputs`` with updated containers of 

140 input `DatasetRef` objects. Connections that are not changed 

141 should not be returned at all. Datasets may only be removed, not 

142 added. Nested collections may be of any multi-pass iterable type, 

143 and the order of iteration will set the order of iteration within 

144 `PipelineTask.runQuantum`. 

145 adjusted_outputs : `dict` 

146 Dict of updated output datasets, with the same structure and 

147 interpretation as ``adjusted_inputs``. 

148 

149 Raises 

150 ------ 

151 ScalarError 

152 Raised if any `Input` or `PrerequisiteInput` connection has 

153 ``multiple`` set to `False`, but multiple datasets. 

154 NoWorkFound 

155 Raised to indicate that this quantum should not be run; not enough 

156 datasets were found for a regular `Input` connection, and the 

157 quantum should be pruned or skipped. 

158 FileNotFoundError 

159 Raised to cause QuantumGraph generation to fail (with the message 

160 included in this exception); not enough datasets were found for a 

161 `PrerequisiteInput` connection. 

162 """ 

163 _, refs = inputs["diffIm"] 

164 for ref in refs: 

165 if ref.dataId["band"] not in self.config.validBands: 

166 raise ValueError( 

167 f"Requested '{ref.dataId['band']}' not in " 

168 "DiaPipelineConfig.validBands. To process bands not in " 

169 "the standard Rubin set (ugrizy) you must add the band to " 

170 "the validBands list in DiaPipelineConfig and add the " 

171 "appropriate columns to the Apdb schema.") 

172 return super().adjustQuantum(inputs, outputs, label, dataId) 

173 

174 

175class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

176 pipelineConnections=DiaPipelineConnections): 

177 """Config for DiaPipelineTask. 

178 """ 

179 coaddName = pexConfig.Field( 

180 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

181 dtype=str, 

182 default="deep", 

183 ) 

184 apdb = pexConfig.ConfigurableField( 

185 target=daxApdb.Apdb, 

186 ConfigClass=daxApdb.ApdbConfig, 

187 doc="Database connection for storing associated DiaSources and " 

188 "DiaObjects. Must already be initialized.", 

189 ) 

190 validBands = pexConfig.ListField( 

191 dtype=str, 

192 default=["u", "g", "r", "i", "z", "y"], 

193 doc="List of bands that are valid for AP processing. To process a " 

194 "band not on this list, the appropriate band specific columns " 

195 "must be added to the Apdb schema in dax_apdb.", 

196 ) 

197 diaCatalogLoader = pexConfig.ConfigurableField( 

198 target=LoadDiaCatalogsTask, 

199 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

200 ) 

201 associator = pexConfig.ConfigurableField( 

202 target=AssociationTask, 

203 doc="Task used to associate DiaSources with DiaObjects.", 

204 ) 

205 diaCalculation = pexConfig.ConfigurableField( 

206 target=DiaObjectCalculationTask, 

207 doc="Task to compute summary statistics for DiaObjects.", 

208 ) 

209 diaForcedSource = pexConfig.ConfigurableField( 

210 target=DiaForcedSourceTask, 

211 doc="Task used for force photometer DiaObject locations in direct and " 

212 "difference images.", 

213 ) 

214 alertPackager = pexConfig.ConfigurableField( 

215 target=PackageAlertsTask, 

216 doc="Subtask for packaging Ap data into alerts.", 

217 ) 

218 doPackageAlerts = pexConfig.Field( 

219 dtype=bool, 

220 default=False, 

221 doc="Package Dia-data into serialized alerts for distribution and " 

222 "write them to disk.", 

223 ) 

224 doWriteAssociatedSources = pexConfig.Field( 

225 dtype=bool, 

226 default=False, 

227 doc="Write out associated and SDMed DiaSources.", 

228 ) 

229 

230 def setDefaults(self): 

231 self.apdb.dia_object_index = "baseline" 

232 self.apdb.dia_object_columns = [] 

233 self.apdb.extra_schema_file = os.path.join( 

234 "${AP_ASSOCIATION_DIR}", 

235 "data", 

236 "apdb-ap-pipe-schema-extra.yaml") 

237 self.diaCalculation.plugins = ["ap_meanPosition", 

238 "ap_HTMIndex", 

239 "ap_nDiaSources", 

240 "ap_diaObjectFlag", 

241 "ap_meanFlux", 

242 "ap_percentileFlux", 

243 "ap_sigmaFlux", 

244 "ap_chi2Flux", 

245 "ap_madFlux", 

246 "ap_skewFlux", 

247 "ap_minMaxFlux", 

248 "ap_maxSlopeFlux", 

249 "ap_meanErrFlux", 

250 "ap_linearFit", 

251 "ap_stetsonJ", 

252 "ap_meanTotFlux", 

253 "ap_sigmaTotFlux"] 

254 

255 def validate(self): 

256 pexConfig.Config.validate(self) 

257 if self.diaCatalogLoader.htmLevel != \ 

258 self.diaCalculation.plugins["ap_HTMIndex"].htmLevel: 

259 raise ValueError("HTM index level in LoadDiaCatalogsTask must be " 

260 "equal to HTMIndexDiaCalculationPlugin index " 

261 "level.") 

262 if "ap_HTMIndex" not in self.diaCalculation.plugins: 

263 raise ValueError("DiaPipe requires the ap_HTMIndex plugin " 

264 "be enabled for proper insertion into the Apdb.") 

265 

266 

267class DiaPipelineTask(pipeBase.PipelineTask): 

268 """Task for loading, associating and storing Difference Image Analysis 

269 (DIA) Objects and Sources. 

270 """ 

271 ConfigClass = DiaPipelineConfig 

272 _DefaultName = "diaPipe" 

273 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

274 

275 def __init__(self, initInputs=None, **kwargs): 

276 super().__init__(**kwargs) 

277 self.apdb = self.config.apdb.apply( 

278 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

279 DiaSource=make_dia_source_schema())) 

280 self.makeSubtask("diaCatalogLoader") 

281 self.makeSubtask("associator") 

282 self.makeSubtask("diaCalculation") 

283 self.makeSubtask("diaForcedSource") 

284 if self.config.doPackageAlerts: 

285 self.makeSubtask("alertPackager") 

286 

287 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

288 inputs = butlerQC.get(inputRefs) 

289 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

290 returnMaxBits=True) 

291 inputs["ccdExposureIdBits"] = expBits 

292 inputs["band"] = butlerQC.quantum.dataId["band"] 

293 

294 outputs = self.run(**inputs) 

295 

296 butlerQC.put(outputs, outputRefs) 

297 

298 @pipeBase.timeMethod 

299 def run(self, 

300 diaSourceTable, 

301 diffIm, 

302 exposure, 

303 warpedExposure, 

304 ccdExposureIdBits, 

305 band): 

306 """Process DiaSources and DiaObjects. 

307 

308 Load previous DiaObjects and their DiaSource history. Calibrate the 

309 values in the diaSourceCat. Associate new DiaSources with previous 

310 DiaObjects. Run forced photometry at the updated DiaObject locations. 

311 Store the results in the Alert Production Database (Apdb). 

312 

313 Parameters 

314 ---------- 

315 diaSourceTable : `pandas.DataFrame` 

316 Newly detected DiaSources. 

317 diffIm : `lsst.afw.image.ExposureF` 

318 Difference image exposure in which the sources in ``diaSourceCat`` 

319 were detected. 

320 exposure : `lsst.afw.image.ExposureF` 

321 Calibrated exposure differenced with a template to create 

322 ``diffIm``. 

323 warpedExposure : `lsst.afw.image.ExposureF` 

324 Template exposure used to create diffIm. 

325 ccdExposureIdBits : `int` 

326 Number of bits used for a unique ``ccdVisitId``. 

327 band : `str` 

328 The band in which the new DiaSources were detected. 

329 

330 Returns 

331 ------- 

332 results : `lsst.pipe.base.Struct` 

333 Results struct with components. 

334 

335 - ``apdbMaker`` : Marker dataset to store in the Butler indicating 

336 that this ccdVisit has completed successfully. 

337 (`lsst.dax.apdb.ApdbConfig`) 

338 - ``associatedDiaSources`` : Catalog of newly associated 

339 DiaSources. (`pandas.DataFrame`) 

340 """ 

341 # Load the DiaObjects and DiaSource history. 

342 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

343 

344 # Associate new DiaSources with existing DiaObjects. 

345 assocResults = self.associator.run(diaSourceTable, 

346 loaderResult.diaObjects) 

347 

348 # Create new DiaObjects from unassociated diaSources. 

349 createResults = self.createNewDiaObjects(assocResults.diaSources) 

350 self._add_association_meta_data(assocResults.nUpdatedDiaObjects, 

351 assocResults.nUnassociatedDiaObjects, 

352 len(createResults.newDiaObjects)) 

353 

354 # Index the DiaSource catalog for this visit after all associations 

355 # have been made. 

356 updatedDiaObjectIds = createResults.diaSources["diaObjectId"][ 

357 createResults.diaSources["diaObjectId"] != 0].to_numpy() 

358 diaSources = createResults.diaSources.set_index(["diaObjectId", 

359 "filterName", 

360 "diaSourceId"], 

361 drop=False) 

362 

363 # Append new DiaObjects and DiaSources to their previous history. 

364 diaObjects = loaderResult.diaObjects.append( 

365 createResults.newDiaObjects.set_index("diaObjectId", drop=False), 

366 sort=True) 

367 if self.testDataFrameIndex(diaObjects): 

368 raise RuntimeError( 

369 "Duplicate DiaObjects created after association. This is " 

370 "likely due to re-running data with an already populated " 

371 "Apdb. If this was not the case then there was an unexpected " 

372 "failure in Association while matching and creating new " 

373 "DiaObjects and should be reported. Exiting.") 

374 mergedDiaSourceHistory = loaderResult.diaSources.append( 

375 diaSources, 

376 sort=True) 

377 # Test for DiaSource duplication first. If duplicates are found, 

378 # this likely means this is duplicate data being processed and sent 

379 # to the Apdb. 

380 if self.testDataFrameIndex(mergedDiaSourceHistory): 

381 raise RuntimeError( 

382 "Duplicate DiaSources found after association and merging " 

383 "with history. This is likely due to re-running data with an " 

384 "already populated Apdb. If this was not the case then there " 

385 "was an unexpected failure in Association while matching " 

386 "sources to objects, and should be reported. Exiting.") 

387 

388 # Compute DiaObject Summary statistics from their full DiaSource 

389 # history. 

390 diaCalResult = self.diaCalculation.run( 

391 diaObjects, 

392 mergedDiaSourceHistory, 

393 updatedDiaObjectIds, 

394 [band]) 

395 # Test for duplication in the updated DiaObjects. 

396 if self.testDataFrameIndex(diaCalResult.diaObjectCat): 

397 raise RuntimeError( 

398 "Duplicate DiaObjects (loaded + updated) created after " 

399 "DiaCalculation. This is unexpected behavior and should be " 

400 "reported. Existing.") 

401 if self.testDataFrameIndex(diaCalResult.updatedDiaObjects): 

402 raise RuntimeError( 

403 "Duplicate DiaObjects (updated) created after " 

404 "DiaCalculation. This is unexpected behavior and should be " 

405 "reported. Existing.") 

406 

407 # Force photometer on the Difference and Calibrated exposures using 

408 # the new and updated DiaObject locations. 

409 diaForcedSources = self.diaForcedSource.run( 

410 diaCalResult.diaObjectCat, 

411 diaCalResult.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

412 ccdExposureIdBits, 

413 exposure, 

414 diffIm) 

415 

416 # Store DiaSources and updated DiaObjects in the Apdb. 

417 self.apdb.storeDiaSources(diaSources) 

418 self.apdb.storeDiaObjects( 

419 diaCalResult.updatedDiaObjects, 

420 exposure.visitInfo.date.toPython()) 

421 self.apdb.storeDiaForcedSources(diaForcedSources) 

422 

423 if self.config.doPackageAlerts: 

424 if len(loaderResult.diaForcedSources) > 1: 

425 diaForcedSources = diaForcedSources.append( 

426 loaderResult.diaForcedSources, 

427 sort=True) 

428 if self.testDataFrameIndex(diaForcedSources): 

429 self.log.warn( 

430 "Duplicate DiaForcedSources created after merge with " 

431 "history and new sources. This may cause downstream " 

432 "problems. Dropping duplicates.") 

433 # Drop duplicates via index and keep the first appearance. 

434 # Reset due to the index shape being slight different than 

435 # expected. 

436 diaForcedSources = diaForcedSources.groupby( 

437 diaForcedSources.index).first() 

438 diaForcedSources.reset_index(drop=True, inplace=True) 

439 diaForcedSources.set_index( 

440 ["diaObjectId", "diaForcedSourceId"], 

441 drop=False, 

442 inplace=True) 

443 self.alertPackager.run(diaSources, 

444 diaCalResult.diaObjectCat, 

445 loaderResult.diaSources, 

446 diaForcedSources, 

447 diffIm, 

448 warpedExposure, 

449 ccdExposureIdBits) 

450 

451 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

452 associatedDiaSources=diaSources) 

453 

454 def createNewDiaObjects(self, diaSources): 

455 """Loop through the set of DiaSources and create new DiaObjects 

456 for unassociated DiaSources. 

457 

458 Parameters 

459 ---------- 

460 diaSources : `pandas.DataFrame` 

461 Set of DiaSources to create new DiaObjects from. 

462 

463 Returns 

464 ------- 

465 results : `lsst.pipe.base.Struct` 

466 Results struct containing: 

467 

468 - ``diaSources`` : DiaSource catalog with updated DiaObject ids. 

469 (`pandas.DataFrame`) 

470 - ``newDiaObjects`` : Newly created DiaObjects from the 

471 unassociated DiaSources. (`pandas.DataFrame`) 

472 """ 

473 newDiaObjectsList = [] 

474 for idx, diaSource in diaSources.iterrows(): 

475 if diaSource["diaObjectId"] == 0: 

476 newDiaObjectsList.append( 

477 self._initialize_dia_object(diaSource["diaSourceId"])) 

478 diaSources.loc[idx, "diaObjectId"] = diaSource["diaSourceId"] 

479 if len(newDiaObjectsList) > 0: 

480 newDiaObjects = pd.DataFrame(data=newDiaObjectsList) 

481 else: 

482 tmpObj = self._initialize_dia_object(0) 

483 newDiaObjects = pd.DataFrame(data=newDiaObjectsList, 

484 columns=tmpObj.keys()) 

485 return pipeBase.Struct(diaSources=diaSources, 

486 newDiaObjects=pd.DataFrame(data=newDiaObjects)) 

487 

488 def _initialize_dia_object(self, objId): 

489 """Create a new DiaObject with values required to be initialized by the 

490 Ppdb. 

491 

492 Parameters 

493 ---------- 

494 objid : `int` 

495 ``diaObjectId`` value for the of the new DiaObject. 

496 

497 Returns 

498 ------- 

499 diaObject : `dict` 

500 Newly created DiaObject with keys: 

501 

502 ``diaObjectId`` 

503 Unique DiaObjectId (`int`). 

504 ``pmParallaxNdata`` 

505 Number of data points used for parallax calculation (`int`). 

506 ``nearbyObj1`` 

507 Id of the a nearbyObject in the Object table (`int`). 

508 ``nearbyObj2`` 

509 Id of the a nearbyObject in the Object table (`int`). 

510 ``nearbyObj3`` 

511 Id of the a nearbyObject in the Object table (`int`). 

512 ``?PSFluxData`` 

513 Number of data points used to calculate point source flux 

514 summary statistics in each bandpass (`int`). 

515 """ 

516 new_dia_object = {"diaObjectId": objId, 

517 "pmParallaxNdata": 0, 

518 "nearbyObj1": 0, 

519 "nearbyObj2": 0, 

520 "nearbyObj3": 0, 

521 "flags": 0} 

522 for f in ["u", "g", "r", "i", "z", "y"]: 

523 new_dia_object["%sPSFluxNdata" % f] = 0 

524 return new_dia_object 

525 

526 def testDataFrameIndex(self, df): 

527 """Test the sorted DataFrame index for duplicates. 

528 

529 Wrapped as a separate function to allow for mocking of the this task 

530 in unittesting. Default of a mock return for this test is True. 

531 

532 Parameters 

533 ---------- 

534 df : `pandas.DataFrame` 

535 DataFrame to text. 

536 

537 Returns 

538 ------- 

539 `bool` 

540 True if DataFrame contains duplicate rows. 

541 """ 

542 return df.index.has_duplicates 

543 

544 def _add_association_meta_data(self, 

545 nUpdatedDiaObjects, 

546 nUnassociatedDiaObjects, 

547 nNewDiaObjects): 

548 """Store summaries of the association step in the task metadata. 

549 

550 Parameters 

551 ---------- 

552 nUpdatedDiaObjects : `int` 

553 Number of previous DiaObjects associated and updated in this 

554 ccdVisit. 

555 nUnassociatedDiaObjects : `int` 

556 Number of previous DiaObjects that were not associated or updated 

557 in this ccdVisit. 

558 nNewDiaObjects : `int` 

559 Number of newly created DiaObjects for this ccdVisit. 

560 """ 

561 self.metadata.add('numUpdatedDiaObjects', nUpdatedDiaObjects) 

562 self.metadata.add('numUnassociatedDiaObjects', nUnassociatedDiaObjects) 

563 self.metadata.add('numNewDiaObjects', nNewDiaObjects) 

564 

565 

566class DiaPipelineSolarSystemConnections(DiaPipelineConnections): 

567 ssObjects = connTypes.Input( 

568 doc="Solar System Objects observable in this visit.", 

569 name="visitSsObjects", 

570 storageClass="DataFrame", 

571 dimensions=("instrument", "visit"), 

572 ) 

573 ssObjectAssocDiaSources = connTypes.Output( 

574 doc="DiaSources associated with existing Solar System objects..", 

575 name="{fakesType}{coaddName}Diff_ssObjectAssocDiaSrc", 

576 storageClass="DataFrame", 

577 dimensions=("instrument", "visit", "detector"), 

578 ) 

579 

580 

581class DiaPipelineSolarySystemConfig(DiaPipelineConfig, 

582 pipelineConnections=DiaPipelineSolarSystemConnections): 

583 solarSystemAssociation = pexConfig.ConfigurableField( 

584 target=SolarSystemAssociationTask, 

585 doc="Task used to associate DiaSources with Solar System Objects.", 

586 ) 

587 

588 

589class DiaPipelineSolarSystemTask(DiaPipelineTask): 

590 """Task for loading and storing Difference Image Analysis 

591 (DIA) Sources after associating them to previous DiaObjects and 

592 SSObjects. 

593 

594 SSO behavior currently necessitates a separate pipelinetask, however, after 

595 DM-31389 is merged this SSO specific DiaPipe will merge into the default 

596 class. 

597 """ 

598 ConfigClass = DiaPipelineSolarySystemConfig 

599 _DefaultName = "diaPipeSSO" 

600 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

601 

602 def __init__(self, initInputs=None, **kwargs): 

603 super().__init__(**kwargs) 

604 self.makeSubtask("solarSystemAssociation") 

605 

606 @pipeBase.timeMethod 

607 def run(self, 

608 diaSourceTable, 

609 ssObjects, 

610 diffIm, 

611 exposure, 

612 warpedExposure, 

613 ccdExposureIdBits, 

614 band): 

615 """Process DiaSources and DiaObjects. 

616 

617 Load previous DiaObjects and their DiaSource history. Calibrate the 

618 values in the ``diaSourceTable``. Associate new DiaSources with previous 

619 DiaObjects. Run forced photometry at the updated DiaObject locations. 

620 Store the results in the Alert Production Database (Apdb). 

621 

622 Parameters 

623 ---------- 

624 diaSourceTable : `pandas.DataFrame` 

625 Newly detected DiaSources. 

626 diffIm : `lsst.afw.image.ExposureF` 

627 Difference image exposure in which the sources in 

628 ``diaSourceTable`` were detected. 

629 exposure : `lsst.afw.image.ExposureF` 

630 Calibrated exposure differenced with a template to create 

631 ``diffIm``. 

632 warpedExposure : `lsst.afw.image.ExposureF` 

633 Template exposure used to create diffIm. 

634 ccdExposureIdBits : `int` 

635 Number of bits used for a unique ``ccdVisitId``. 

636 band : `str` 

637 The band in which the new DiaSources were detected. 

638 

639 Returns 

640 ------- 

641 results : `lsst.pipe.base.Struct` 

642 Results struct with components. 

643 

644 - ``apdbMaker`` : Marker dataset to store in the Butler indicating 

645 that this ccdVisit has completed successfully. 

646 (`lsst.dax.apdb.ApdbConfig`) 

647 - ``associatedDiaSources`` : Full set of DiaSources associated 

648 to current and new DiaObjects. This is an optional Butler output. 

649 (`pandas.DataFrame`) 

650 - ``ssObjectAssocDiaSources`` : Set of DiaSources associated with 

651 solar system objects. (`pandas.DataFrame`) 

652 """ 

653 # Load the DiaObjects and DiaSource history. 

654 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

655 

656 # Associate new DiaSources with existing DiaObjects and update 

657 # DiaObject summary statistics using the full DiaSource history. 

658 assocResults = self.associator.run(diaSourceTable, 

659 loaderResult.diaObjects, 

660 loaderResult.diaSources) 

661 ssObjectAssocResults = self.solarSystemAssociation.run( 

662 diaSourceTable.reset_index(drop=True), 

663 ssObjects) 

664 

665 mergedDiaSourceHistory = loaderResult.diaSources.append( 

666 assocResults.diaSources, 

667 sort=True) 

668 # Test for DiaSource duplication first. If duplicates are found, 

669 # this likely means this is duplicate data being processed and sent 

670 # to the Apdb. 

671 if self.testDataFrameIndex(mergedDiaSourceHistory): 

672 raise RuntimeError( 

673 "Duplicate DiaSources found after association and merging " 

674 "with history. This is likely due to re-running data with an " 

675 "already populated Apdb. If this was not the case then there " 

676 "was an unexpected failure in Association while matching " 

677 "sources to objects, and should be reported. Exiting.") 

678 

679 diaCalResult = self.diaCalculation.run( 

680 assocResults.diaObjects, 

681 mergedDiaSourceHistory, 

682 assocResults.matchedDiaObjectIds, 

683 [band]) 

684 if self.testDataFrameIndex(diaCalResult.diaObjectCat): 

685 raise RuntimeError( 

686 "Duplicate DiaObjects (loaded + updated) created after " 

687 "DiaCalculation. This is unexpected behavior and should be " 

688 "reported. Existing.") 

689 if self.testDataFrameIndex(diaCalResult.updatedDiaObjects): 

690 raise RuntimeError( 

691 "Duplicate DiaObjects (updated) created after " 

692 "DiaCalculation. This is unexpected behavior and should be " 

693 "reported. Existing.") 

694 

695 # Force photometer on the Difference and Calibrated exposures using 

696 # the new and updated DiaObject locations. 

697 diaForcedSources = self.diaForcedSource.run( 

698 diaCalResult.diaObjectCat, 

699 diaCalResult.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

700 ccdExposureIdBits, 

701 exposure, 

702 diffIm) 

703 

704 # Store DiaSources and updated DiaObjects in the Apdb. 

705 self.apdb.storeDiaSources(assocResults.diaSources) 

706 self.apdb.storeDiaObjects( 

707 diaCalResult.updatedDiaObjects, 

708 exposure.visitInfo.date.toPython()) 

709 self.apdb.storeDiaForcedSources(diaForcedSources) 

710 

711 if self.config.doPackageAlerts: 

712 if len(loaderResult.diaForcedSources) > 1: 

713 diaForcedSources = diaForcedSources.append( 

714 loaderResult.diaForcedSources, 

715 sort=True) 

716 if self.testDataFrameIndex(diaForcedSources): 

717 self.log.warn( 

718 "Duplicate DiaForcedSources created after merge with " 

719 "history and new sources. This may cause downstream " 

720 "problems. Dropping duplicates.") 

721 # Drop duplicates via index and keep the first appearance. 

722 # Reset due to the index shape being slight different than 

723 # expected. 

724 diaForcedSources = diaForcedSources.groupby( 

725 diaForcedSources.index).first() 

726 diaForcedSources.reset_index(drop=True, inplace=True) 

727 diaForcedSources.set_index( 

728 ["diaObjectId", "diaForcedSourceId"], 

729 drop=False, 

730 inplace=True) 

731 self.alertPackager.run(assocResults.diaSources, 

732 diaCalResult.diaObjectCat, 

733 loaderResult.diaSources, 

734 diaForcedSources, 

735 diffIm, 

736 warpedExposure, 

737 ccdExposureIdBits) 

738 

739 return pipeBase.Struct( 

740 apdbMarker=self.config.apdb.value, 

741 associatedDiaSources=assocResults.diaSources, 

742 ssObjectAssocDiaSources=ssObjectAssocResults.ssoAssocDiaSources)