Coverage for python/lsst/ap/association/diaPipe.py: 30%

140 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-24 10:22 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""PipelineTask for associating DiaSources with previous DiaObjects. 

24 

25Additionally performs forced photometry on the calibrated and difference 

26images at the updated locations of DiaObjects. 

27 

28Currently loads directly from the Apdb rather than pre-loading. 

29""" 

30 

31__all__ = ("DiaPipelineConfig", 

32 "DiaPipelineTask", 

33 "DiaPipelineConnections") 

34 

35import pandas as pd 

36 

37from lsst.daf.base import DateTime 

38import lsst.dax.apdb as daxApdb 

39from lsst.meas.base import DetectorVisitIdGeneratorConfig, DiaObjectCalculationTask 

40import lsst.pex.config as pexConfig 

41import lsst.pipe.base as pipeBase 

42import lsst.pipe.base.connectionTypes as connTypes 

43from lsst.utils.timer import timeMethod 

44 

45from lsst.ap.association import ( 

46 AssociationTask, 

47 DiaForcedSourceTask, 

48 LoadDiaCatalogsTask, 

49 PackageAlertsTask) 

50from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask 

51 

52 

53class DiaPipelineConnections( 

54 pipeBase.PipelineTaskConnections, 

55 dimensions=("instrument", "visit", "detector"), 

56 defaultTemplates={"coaddName": "deep", "fakesType": ""}): 

57 """Butler connections for DiaPipelineTask. 

58 """ 

59 diaSourceTable = connTypes.Input( 

60 doc="Catalog of calibrated DiaSources.", 

61 name="{fakesType}{coaddName}Diff_diaSrcTable", 

62 storageClass="DataFrame", 

63 dimensions=("instrument", "visit", "detector"), 

64 ) 

65 solarSystemObjectTable = connTypes.Input( 

66 doc="Catalog of SolarSolarSystem objects expected to be observable in " 

67 "this detectorVisit.", 

68 name="visitSsObjects", 

69 storageClass="DataFrame", 

70 dimensions=("instrument", "visit"), 

71 ) 

72 diffIm = connTypes.Input( 

73 doc="Difference image on which the DiaSources were detected.", 

74 name="{fakesType}{coaddName}Diff_differenceExp", 

75 storageClass="ExposureF", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 exposure = connTypes.Input( 

79 doc="Calibrated exposure differenced with a template image during " 

80 "image differencing.", 

81 name="{fakesType}calexp", 

82 storageClass="ExposureF", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 template = connTypes.Input( 

86 doc="Warped template used to create `subtractedExposure`. Not PSF " 

87 "matched.", 

88 dimensions=("instrument", "visit", "detector"), 

89 storageClass="ExposureF", 

90 name="{fakesType}{coaddName}Diff_templateExp", 

91 ) 

92 apdbMarker = connTypes.Output( 

93 doc="Marker dataset storing the configuration of the Apdb for each " 

94 "visit/detector. Used to signal the completion of the pipeline.", 

95 name="apdb_marker", 

96 storageClass="Config", 

97 dimensions=("instrument", "visit", "detector"), 

98 ) 

99 associatedDiaSources = connTypes.Output( 

100 doc="Optional output storing the DiaSource catalog after matching, " 

101 "calibration, and standardization for insertion into the Apdb.", 

102 name="{fakesType}{coaddName}Diff_assocDiaSrc", 

103 storageClass="DataFrame", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 diaForcedSources = connTypes.Output( 

107 doc="Optional output storing the forced sources computed at the diaObject positions.", 

108 name="{fakesType}{coaddName}Diff_diaForcedSrc", 

109 storageClass="DataFrame", 

110 dimensions=("instrument", "visit", "detector"), 

111 ) 

112 diaObjects = connTypes.Output( 

113 doc="Optional output storing the updated diaObjects associated to these sources.", 

114 name="{fakesType}{coaddName}Diff_diaObject", 

115 storageClass="DataFrame", 

116 dimensions=("instrument", "visit", "detector"), 

117 ) 

118 longTrailedSources = pipeBase.connectionTypes.Output( 

119 doc="Optional output temporarily storing long trailed diaSources.", 

120 dimensions=("instrument", "visit", "detector"), 

121 storageClass="DataFrame", 

122 name="{fakesType}{coaddName}Diff_longTrailedSrc", 

123 ) 

124 

125 def __init__(self, *, config=None): 

126 super().__init__(config=config) 

127 

128 if not config.doWriteAssociatedSources: 

129 self.outputs.remove("associatedDiaSources") 

130 self.outputs.remove("diaForcedSources") 

131 self.outputs.remove("diaObjects") 

132 if not config.doSolarSystemAssociation: 

133 self.inputs.remove("solarSystemObjectTable") 

134 if not config.associator.doTrailedSourceFilter: 

135 self.outputs.remove("longTrailedSources") 

136 

137 def adjustQuantum(self, inputs, outputs, label, dataId): 

138 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

139 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

140 of the activator. 

141 

142 This implementation checks to make sure that the filters in the dataset 

143 are compatible with AP processing as set by the Apdb/DPDD schema. 

144 

145 Parameters 

146 ---------- 

147 inputs : `dict` 

148 Dictionary whose keys are an input (regular or prerequisite) 

149 connection name and whose values are a tuple of the connection 

150 instance and a collection of associated `DatasetRef` objects. 

151 The exact type of the nested collections is unspecified; it can be 

152 assumed to be multi-pass iterable and support `len` and ``in``, but 

153 it should not be mutated in place. In contrast, the outer 

154 dictionaries are guaranteed to be temporary copies that are true 

155 `dict` instances, and hence may be modified and even returned; this 

156 is especially useful for delegating to `super` (see notes below). 

157 outputs : `dict` 

158 Dict of output datasets, with the same structure as ``inputs``. 

159 label : `str` 

160 Label for this task in the pipeline (should be used in all 

161 diagnostic messages). 

162 data_id : `lsst.daf.butler.DataCoordinate` 

163 Data ID for this quantum in the pipeline (should be used in all 

164 diagnostic messages). 

165 

166 Returns 

167 ------- 

168 adjusted_inputs : `dict` 

169 Dict of the same form as ``inputs`` with updated containers of 

170 input `DatasetRef` objects. Connections that are not changed 

171 should not be returned at all. Datasets may only be removed, not 

172 added. Nested collections may be of any multi-pass iterable type, 

173 and the order of iteration will set the order of iteration within 

174 `PipelineTask.runQuantum`. 

175 adjusted_outputs : `dict` 

176 Dict of updated output datasets, with the same structure and 

177 interpretation as ``adjusted_inputs``. 

178 

179 Raises 

180 ------ 

181 ScalarError 

182 Raised if any `Input` or `PrerequisiteInput` connection has 

183 ``multiple`` set to `False`, but multiple datasets. 

184 NoWorkFound 

185 Raised to indicate that this quantum should not be run; not enough 

186 datasets were found for a regular `Input` connection, and the 

187 quantum should be pruned or skipped. 

188 FileNotFoundError 

189 Raised to cause QuantumGraph generation to fail (with the message 

190 included in this exception); not enough datasets were found for a 

191 `PrerequisiteInput` connection. 

192 """ 

193 _, refs = inputs["diffIm"] 

194 for ref in refs: 

195 if ref.dataId["band"] not in self.config.validBands: 

196 raise ValueError( 

197 f"Requested '{ref.dataId['band']}' not in " 

198 "DiaPipelineConfig.validBands. To process bands not in " 

199 "the standard Rubin set (ugrizy) you must add the band to " 

200 "the validBands list in DiaPipelineConfig and add the " 

201 "appropriate columns to the Apdb schema.") 

202 return super().adjustQuantum(inputs, outputs, label, dataId) 

203 

204 

205class DiaPipelineConfig(pipeBase.PipelineTaskConfig, 

206 pipelineConnections=DiaPipelineConnections): 

207 """Config for DiaPipelineTask. 

208 """ 

209 coaddName = pexConfig.Field( 

210 doc="coadd name: typically one of deep, goodSeeing, or dcr", 

211 dtype=str, 

212 default="deep", 

213 ) 

214 apdb = daxApdb.ApdbSql.makeField( 

215 doc="Database connection for storing associated DiaSources and " 

216 "DiaObjects. Must already be initialized.", 

217 ) 

218 validBands = pexConfig.ListField( 

219 dtype=str, 

220 default=["u", "g", "r", "i", "z", "y"], 

221 doc="List of bands that are valid for AP processing. To process a " 

222 "band not on this list, the appropriate band specific columns " 

223 "must be added to the Apdb schema in dax_apdb.", 

224 ) 

225 diaCatalogLoader = pexConfig.ConfigurableField( 

226 target=LoadDiaCatalogsTask, 

227 doc="Task to load DiaObjects and DiaSources from the Apdb.", 

228 ) 

229 associator = pexConfig.ConfigurableField( 

230 target=AssociationTask, 

231 doc="Task used to associate DiaSources with DiaObjects.", 

232 ) 

233 doSolarSystemAssociation = pexConfig.Field( 

234 dtype=bool, 

235 default=False, 

236 doc="Process SolarSystem objects through the pipeline.", 

237 ) 

238 solarSystemAssociator = pexConfig.ConfigurableField( 

239 target=SolarSystemAssociationTask, 

240 doc="Task used to associate DiaSources with SolarSystemObjects.", 

241 ) 

242 diaCalculation = pexConfig.ConfigurableField( 

243 target=DiaObjectCalculationTask, 

244 doc="Task to compute summary statistics for DiaObjects.", 

245 ) 

246 diaForcedSource = pexConfig.ConfigurableField( 

247 target=DiaForcedSourceTask, 

248 doc="Task used for force photometer DiaObject locations in direct and " 

249 "difference images.", 

250 ) 

251 alertPackager = pexConfig.ConfigurableField( 

252 target=PackageAlertsTask, 

253 doc="Subtask for packaging Ap data into alerts.", 

254 ) 

255 doPackageAlerts = pexConfig.Field( 

256 dtype=bool, 

257 default=False, 

258 doc="Package Dia-data into serialized alerts for distribution and " 

259 "write them to disk.", 

260 ) 

261 doWriteAssociatedSources = pexConfig.Field( 

262 dtype=bool, 

263 default=False, 

264 doc="Write out associated DiaSources, DiaForcedSources, and DiaObjects, " 

265 "formatted following the Science Data Model.", 

266 ) 

267 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

268 

269 def setDefaults(self): 

270 self.apdb.dia_object_index = "baseline" 

271 self.apdb.dia_object_columns = [] 

272 self.diaCalculation.plugins = ["ap_meanPosition", 

273 "ap_nDiaSources", 

274 "ap_diaObjectFlag", 

275 "ap_meanFlux", 

276 "ap_percentileFlux", 

277 "ap_sigmaFlux", 

278 "ap_chi2Flux", 

279 "ap_madFlux", 

280 "ap_skewFlux", 

281 "ap_minMaxFlux", 

282 "ap_maxSlopeFlux", 

283 "ap_meanErrFlux", 

284 "ap_linearFit", 

285 "ap_stetsonJ", 

286 "ap_meanTotFlux", 

287 "ap_sigmaTotFlux"] 

288 

289 

290class DiaPipelineTask(pipeBase.PipelineTask): 

291 """Task for loading, associating and storing Difference Image Analysis 

292 (DIA) Objects and Sources. 

293 """ 

294 ConfigClass = DiaPipelineConfig 

295 _DefaultName = "diaPipe" 

296 

297 def __init__(self, initInputs=None, **kwargs): 

298 super().__init__(**kwargs) 

299 self.apdb = self.config.apdb.apply() 

300 self.makeSubtask("diaCatalogLoader") 

301 self.makeSubtask("associator") 

302 self.makeSubtask("diaCalculation") 

303 self.makeSubtask("diaForcedSource") 

304 if self.config.doPackageAlerts: 

305 self.makeSubtask("alertPackager") 

306 if self.config.doSolarSystemAssociation: 

307 self.makeSubtask("solarSystemAssociator") 

308 

309 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

310 inputs = butlerQC.get(inputRefs) 

311 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

312 # Need to set ccdExposureIdBits (now deprecated) to None and pass it, 

313 # since there are non-optional positional arguments after it. 

314 inputs["ccdExposureIdBits"] = None 

315 inputs["band"] = butlerQC.quantum.dataId["band"] 

316 if not self.config.doSolarSystemAssociation: 

317 inputs["solarSystemObjectTable"] = None 

318 

319 outputs = self.run(**inputs) 

320 

321 butlerQC.put(outputs, outputRefs) 

322 

323 @timeMethod 

324 def run(self, 

325 diaSourceTable, 

326 solarSystemObjectTable, 

327 diffIm, 

328 exposure, 

329 template, 

330 ccdExposureIdBits, # TODO: remove on DM-38687. 

331 band, 

332 idGenerator=None): 

333 """Process DiaSources and DiaObjects. 

334 

335 Load previous DiaObjects and their DiaSource history. Calibrate the 

336 values in the diaSourceCat. Associate new DiaSources with previous 

337 DiaObjects. Run forced photometry at the updated DiaObject locations. 

338 Store the results in the Alert Production Database (Apdb). 

339 

340 Parameters 

341 ---------- 

342 diaSourceTable : `pandas.DataFrame` 

343 Newly detected DiaSources. 

344 diffIm : `lsst.afw.image.ExposureF` 

345 Difference image exposure in which the sources in ``diaSourceCat`` 

346 were detected. 

347 exposure : `lsst.afw.image.ExposureF` 

348 Calibrated exposure differenced with a template to create 

349 ``diffIm``. 

350 template : `lsst.afw.image.ExposureF` 

351 Template exposure used to create diffIm. 

352 ccdExposureIdBits : `int` 

353 Number of bits used for a unique ``ccdVisitId``. Deprecated in 

354 favor of ``idGenerator``, and ignored if that is present (will be 

355 removed after v26). Pass `None` explicitly to avoid a deprecation 

356 warning (a default is impossible given that later positional 

357 arguments are not defaulted). 

358 band : `str` 

359 The band in which the new DiaSources were detected. 

360 idGenerator : `lsst.meas.base.IdGenerator`, optional 

361 Object that generates source IDs and random number generator seeds. 

362 Will be required after ``ccdExposureIdBits`` is removed. 

363 

364 Returns 

365 ------- 

366 results : `lsst.pipe.base.Struct` 

367 Results struct with components. 

368 

369 - ``apdbMaker`` : Marker dataset to store in the Butler indicating 

370 that this ccdVisit has completed successfully. 

371 (`lsst.dax.apdb.ApdbConfig`) 

372 - ``associatedDiaSources`` : Catalog of newly associated 

373 DiaSources. (`pandas.DataFrame`) 

374 """ 

375 # Load the DiaObjects and DiaSource history. 

376 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb) 

377 

378 # Associate new DiaSources with existing DiaObjects. 

379 assocResults = self.associator.run(diaSourceTable, loaderResult.diaObjects, 

380 exposure_time=diffIm.visitInfo.exposureTime) 

381 

382 if self.config.doSolarSystemAssociation: 

383 ssoAssocResult = self.solarSystemAssociator.run( 

384 assocResults.unAssocDiaSources, 

385 solarSystemObjectTable, 

386 diffIm) 

387 createResults = self.createNewDiaObjects( 

388 ssoAssocResult.unAssocDiaSources) 

389 associatedDiaSources = pd.concat( 

390 [assocResults.matchedDiaSources, 

391 ssoAssocResult.ssoAssocDiaSources, 

392 createResults.diaSources]) 

393 nTotalSsObjects = ssoAssocResult.nTotalSsObjects 

394 nAssociatedSsObjects = ssoAssocResult.nAssociatedSsObjects 

395 else: 

396 createResults = self.createNewDiaObjects( 

397 assocResults.unAssocDiaSources) 

398 associatedDiaSources = pd.concat( 

399 [assocResults.matchedDiaSources, 

400 createResults.diaSources]) 

401 nTotalSsObjects = 0 

402 nAssociatedSsObjects = 0 

403 

404 # Create new DiaObjects from unassociated diaSources. 

405 self._add_association_meta_data(assocResults.nUpdatedDiaObjects, 

406 assocResults.nUnassociatedDiaObjects, 

407 createResults.nNewDiaObjects, 

408 nTotalSsObjects, 

409 nAssociatedSsObjects) 

410 # Index the DiaSource catalog for this visit after all associations 

411 # have been made. 

412 updatedDiaObjectIds = associatedDiaSources["diaObjectId"][ 

413 associatedDiaSources["diaObjectId"] != 0].to_numpy() 

414 associatedDiaSources.set_index(["diaObjectId", 

415 "band", 

416 "diaSourceId"], 

417 drop=False, 

418 inplace=True) 

419 

420 # Append new DiaObjects and DiaSources to their previous history. 

421 diaObjects = pd.concat( 

422 [loaderResult.diaObjects, 

423 createResults.newDiaObjects.set_index("diaObjectId", drop=False)], 

424 sort=True) 

425 if self.testDataFrameIndex(diaObjects): 

426 raise RuntimeError( 

427 "Duplicate DiaObjects created after association. This is " 

428 "likely due to re-running data with an already populated " 

429 "Apdb. If this was not the case then there was an unexpected " 

430 "failure in Association while matching and creating new " 

431 "DiaObjects and should be reported. Exiting.") 

432 mergedDiaSourceHistory = pd.concat( 

433 [loaderResult.diaSources, associatedDiaSources], 

434 sort=True) 

435 # Test for DiaSource duplication first. If duplicates are found, 

436 # this likely means this is duplicate data being processed and sent 

437 # to the Apdb. 

438 if self.testDataFrameIndex(mergedDiaSourceHistory): 

439 raise RuntimeError( 

440 "Duplicate DiaSources found after association and merging " 

441 "with history. This is likely due to re-running data with an " 

442 "already populated Apdb. If this was not the case then there " 

443 "was an unexpected failure in Association while matching " 

444 "sources to objects, and should be reported. Exiting.") 

445 

446 # Compute DiaObject Summary statistics from their full DiaSource 

447 # history. 

448 diaCalResult = self.diaCalculation.run( 

449 diaObjects, 

450 mergedDiaSourceHistory, 

451 updatedDiaObjectIds, 

452 [band]) 

453 # Test for duplication in the updated DiaObjects. 

454 if self.testDataFrameIndex(diaCalResult.diaObjectCat): 

455 raise RuntimeError( 

456 "Duplicate DiaObjects (loaded + updated) created after " 

457 "DiaCalculation. This is unexpected behavior and should be " 

458 "reported. Exiting.") 

459 if self.testDataFrameIndex(diaCalResult.updatedDiaObjects): 

460 raise RuntimeError( 

461 "Duplicate DiaObjects (updated) created after " 

462 "DiaCalculation. This is unexpected behavior and should be " 

463 "reported. Exiting.") 

464 

465 # Force photometer on the Difference and Calibrated exposures using 

466 # the new and updated DiaObject locations. 

467 diaForcedSources = self.diaForcedSource.run( 

468 diaCalResult.diaObjectCat, 

469 diaCalResult.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(), 

470 # Passing a ccdExposureIdBits here that isn't None will make 

471 # diaForcedSource emit a deprecation warning, so we don't have to 

472 # emit our own. 

473 ccdExposureIdBits, 

474 exposure, 

475 diffIm, 

476 idGenerator=idGenerator) 

477 

478 # Store DiaSources, updated DiaObjects, and DiaForcedSources in the 

479 # Apdb. 

480 self.apdb.store( 

481 DateTime.now(), 

482 diaCalResult.updatedDiaObjects, 

483 associatedDiaSources, 

484 diaForcedSources) 

485 

486 if self.config.doPackageAlerts: 

487 if len(loaderResult.diaForcedSources) > 1: 

488 diaForcedSources = pd.concat( 

489 [diaForcedSources, loaderResult.diaForcedSources], 

490 sort=True) 

491 if self.testDataFrameIndex(diaForcedSources): 

492 self.log.warning( 

493 "Duplicate DiaForcedSources created after merge with " 

494 "history and new sources. This may cause downstream " 

495 "problems. Dropping duplicates.") 

496 # Drop duplicates via index and keep the first appearance. 

497 # Reset due to the index shape being slight different than 

498 # expected. 

499 diaForcedSources = diaForcedSources.groupby( 

500 diaForcedSources.index).first() 

501 diaForcedSources.reset_index(drop=True, inplace=True) 

502 diaForcedSources.set_index( 

503 ["diaObjectId", "diaForcedSourceId"], 

504 drop=False, 

505 inplace=True) 

506 self.alertPackager.run(associatedDiaSources, 

507 diaCalResult.diaObjectCat, 

508 loaderResult.diaSources, 

509 diaForcedSources, 

510 diffIm, 

511 template) 

512 

513 return pipeBase.Struct(apdbMarker=self.config.apdb.value, 

514 associatedDiaSources=associatedDiaSources, 

515 diaForcedSources=diaForcedSources, 

516 diaObjects=diaObjects, 

517 longTrailedSources=assocResults.longTrailedSources 

518 ) 

519 

520 def createNewDiaObjects(self, unAssocDiaSources): 

521 """Loop through the set of DiaSources and create new DiaObjects 

522 for unassociated DiaSources. 

523 

524 Parameters 

525 ---------- 

526 unAssocDiaSources : `pandas.DataFrame` 

527 Set of DiaSources to create new DiaObjects from. 

528 

529 Returns 

530 ------- 

531 results : `lsst.pipe.base.Struct` 

532 Results struct containing: 

533 

534 - ``diaSources`` : DiaSource catalog with updated DiaObject ids. 

535 (`pandas.DataFrame`) 

536 - ``newDiaObjects`` : Newly created DiaObjects from the 

537 unassociated DiaSources. (`pandas.DataFrame`) 

538 - ``nNewDiaObjects`` : Number of newly created diaObjects.(`int`) 

539 """ 

540 if len(unAssocDiaSources) == 0: 

541 tmpObj = self._initialize_dia_object(0) 

542 newDiaObjects = pd.DataFrame(data=[], 

543 columns=tmpObj.keys()) 

544 else: 

545 newDiaObjects = unAssocDiaSources["diaSourceId"].apply( 

546 self._initialize_dia_object) 

547 unAssocDiaSources["diaObjectId"] = unAssocDiaSources["diaSourceId"] 

548 return pipeBase.Struct(diaSources=unAssocDiaSources, 

549 newDiaObjects=newDiaObjects, 

550 nNewDiaObjects=len(newDiaObjects)) 

551 

552 def _initialize_dia_object(self, objId): 

553 """Create a new DiaObject with values required to be initialized by the 

554 Ppdb. 

555 

556 Parameters 

557 ---------- 

558 objid : `int` 

559 ``diaObjectId`` value for the of the new DiaObject. 

560 

561 Returns 

562 ------- 

563 diaObject : `dict` 

564 Newly created DiaObject with keys: 

565 

566 ``diaObjectId`` 

567 Unique DiaObjectId (`int`). 

568 ``pmParallaxNdata`` 

569 Number of data points used for parallax calculation (`int`). 

570 ``nearbyObj1`` 

571 Id of the a nearbyObject in the Object table (`int`). 

572 ``nearbyObj2`` 

573 Id of the a nearbyObject in the Object table (`int`). 

574 ``nearbyObj3`` 

575 Id of the a nearbyObject in the Object table (`int`). 

576 ``?_psfFluxNdata`` 

577 Number of data points used to calculate point source flux 

578 summary statistics in each bandpass (`int`). 

579 """ 

580 new_dia_object = {"diaObjectId": objId, 

581 "pmParallaxNdata": 0, 

582 "nearbyObj1": 0, 

583 "nearbyObj2": 0, 

584 "nearbyObj3": 0, 

585 "flags": 0} 

586 for f in ["u", "g", "r", "i", "z", "y"]: 

587 new_dia_object["%s_psfFluxNdata" % f] = 0 

588 return pd.Series(data=new_dia_object) 

589 

590 def testDataFrameIndex(self, df): 

591 """Test the sorted DataFrame index for duplicates. 

592 

593 Wrapped as a separate function to allow for mocking of the this task 

594 in unittesting. Default of a mock return for this test is True. 

595 

596 Parameters 

597 ---------- 

598 df : `pandas.DataFrame` 

599 DataFrame to text. 

600 

601 Returns 

602 ------- 

603 `bool` 

604 True if DataFrame contains duplicate rows. 

605 """ 

606 return df.index.has_duplicates 

607 

608 def _add_association_meta_data(self, 

609 nUpdatedDiaObjects, 

610 nUnassociatedDiaObjects, 

611 nNewDiaObjects, 

612 nTotalSsObjects, 

613 nAssociatedSsObjects): 

614 """Store summaries of the association step in the task metadata. 

615 

616 Parameters 

617 ---------- 

618 nUpdatedDiaObjects : `int` 

619 Number of previous DiaObjects associated and updated in this 

620 ccdVisit. 

621 nUnassociatedDiaObjects : `int` 

622 Number of previous DiaObjects that were not associated or updated 

623 in this ccdVisit. 

624 nNewDiaObjects : `int` 

625 Number of newly created DiaObjects for this ccdVisit. 

626 nTotalSsObjects : `int` 

627 Number of SolarSystemObjects within the observable detector 

628 area. 

629 nAssociatedSsObjects : `int` 

630 Number of successfully associated SolarSystemObjects. 

631 """ 

632 self.metadata.add('numUpdatedDiaObjects', nUpdatedDiaObjects) 

633 self.metadata.add('numUnassociatedDiaObjects', nUnassociatedDiaObjects) 

634 self.metadata.add('numNewDiaObjects', nNewDiaObjects) 

635 self.metadata.add('numTotalSolarSystemObjects', nTotalSsObjects) 

636 self.metadata.add('numAssociatedSsObjects', nAssociatedSsObjects)