Coverage for python/lsst/ap/association/diaPipe.py : 27%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
32import pandas as pd
34import lsst.dax.apdb as daxApdb
35from lsst.meas.base import DiaObjectCalculationTask
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38import lsst.pipe.base.connectionTypes as connTypes
40from lsst.ap.association import (
41 AssociationTask,
42 DiaForcedSourceTask,
43 LoadDiaCatalogsTask,
44 make_dia_object_schema,
45 make_dia_source_schema,
46 PackageAlertsTask)
47from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask
49__all__ = ("DiaPipelineConfig",
50 "DiaPipelineTask",
51 "DiaPipelineConnections")
54class DiaPipelineConnections(
55 pipeBase.PipelineTaskConnections,
56 dimensions=("instrument", "visit", "detector"),
57 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
58 """Butler connections for DiaPipelineTask.
59 """
60 diaSourceTable = connTypes.Input(
61 doc="Catalog of calibrated DiaSources.",
62 name="{fakesType}{coaddName}Diff_diaSrcTable",
63 storageClass="DataFrame",
64 dimensions=("instrument", "visit", "detector"),
65 )
66 diffIm = connTypes.Input(
67 doc="Difference image on which the DiaSources were detected.",
68 name="{fakesType}{coaddName}Diff_differenceExp",
69 storageClass="ExposureF",
70 dimensions=("instrument", "visit", "detector"),
71 )
72 exposure = connTypes.Input(
73 doc="Calibrated exposure differenced with a template image during "
74 "image differencing.",
75 name="calexp",
76 storageClass="ExposureF",
77 dimensions=("instrument", "visit", "detector"),
78 )
79 warpedExposure = connTypes.Input(
80 doc="Warped template used to create `subtractedExposure`. Not PSF "
81 "matched.",
82 dimensions=("instrument", "visit", "detector"),
83 storageClass="ExposureF",
84 name="{fakesType}{coaddName}Diff_warpedExp",
85 )
86 apdbMarker = connTypes.Output(
87 doc="Marker dataset storing the configuration of the Apdb for each "
88 "visit/detector. Used to signal the completion of the pipeline.",
89 name="apdb_marker",
90 storageClass="Config",
91 dimensions=("instrument", "visit", "detector"),
92 )
93 associatedDiaSources = connTypes.Output(
94 doc="Optional output storing the DiaSource catalog after matching, "
95 "calibration, and standardization for insertation into the Apdb.",
96 name="{fakesType}{coaddName}Diff_assocDiaSrc",
97 storageClass="DataFrame",
98 dimensions=("instrument", "visit", "detector"),
99 )
101 def __init__(self, *, config=None):
102 super().__init__(config=config)
104 if not config.doWriteAssociatedSources:
105 self.outputs.remove("associatedDiaSources")
107 def adjustQuantum(self, inputs, outputs, label, dataId):
108 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects
109 in the `lsst.daf.butler.core.Quantum` during the graph generation stage
110 of the activator.
112 This implementation checks to make sure that the filters in the dataset
113 are compatible with AP processing as set by the Apdb/DPDD schema.
115 Parameters
116 ----------
117 inputs : `dict`
118 Dictionary whose keys are an input (regular or prerequisite)
119 connection name and whose values are a tuple of the connection
120 instance and a collection of associated `DatasetRef` objects.
121 The exact type of the nested collections is unspecified; it can be
122 assumed to be multi-pass iterable and support `len` and ``in``, but
123 it should not be mutated in place. In contrast, the outer
124 dictionaries are guaranteed to be temporary copies that are true
125 `dict` instances, and hence may be modified and even returned; this
126 is especially useful for delegating to `super` (see notes below).
127 outputs : `dict`
128 Dict of output datasets, with the same structure as ``inputs``.
129 label : `str`
130 Label for this task in the pipeline (should be used in all
131 diagnostic messages).
132 data_id : `lsst.daf.butler.DataCoordinate`
133 Data ID for this quantum in the pipeline (should be used in all
134 diagnostic messages).
136 Returns
137 -------
138 adjusted_inputs : `dict`
139 Dict of the same form as ``inputs`` with updated containers of
140 input `DatasetRef` objects. Connections that are not changed
141 should not be returned at all. Datasets may only be removed, not
142 added. Nested collections may be of any multi-pass iterable type,
143 and the order of iteration will set the order of iteration within
144 `PipelineTask.runQuantum`.
145 adjusted_outputs : `dict`
146 Dict of updated output datasets, with the same structure and
147 interpretation as ``adjusted_inputs``.
149 Raises
150 ------
151 ScalarError
152 Raised if any `Input` or `PrerequisiteInput` connection has
153 ``multiple`` set to `False`, but multiple datasets.
154 NoWorkFound
155 Raised to indicate that this quantum should not be run; not enough
156 datasets were found for a regular `Input` connection, and the
157 quantum should be pruned or skipped.
158 FileNotFoundError
159 Raised to cause QuantumGraph generation to fail (with the message
160 included in this exception); not enough datasets were found for a
161 `PrerequisiteInput` connection.
162 """
163 _, refs = inputs["diffIm"]
164 for ref in refs:
165 if ref.dataId["band"] not in self.config.validBands:
166 raise ValueError(
167 f"Requested '{ref.dataId['band']}' not in "
168 "DiaPipelineConfig.validBands. To process bands not in "
169 "the standard Rubin set (ugrizy) you must add the band to "
170 "the validBands list in DiaPipelineConfig and add the "
171 "appropriate columns to the Apdb schema.")
172 return super().adjustQuantum(inputs, outputs, label, dataId)
175class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
176 pipelineConnections=DiaPipelineConnections):
177 """Config for DiaPipelineTask.
178 """
179 coaddName = pexConfig.Field(
180 doc="coadd name: typically one of deep, goodSeeing, or dcr",
181 dtype=str,
182 default="deep",
183 )
184 apdb = pexConfig.ConfigurableField(
185 target=daxApdb.Apdb,
186 ConfigClass=daxApdb.ApdbConfig,
187 doc="Database connection for storing associated DiaSources and "
188 "DiaObjects. Must already be initialized.",
189 )
190 validBands = pexConfig.ListField(
191 dtype=str,
192 default=["u", "g", "r", "i", "z", "y"],
193 doc="List of bands that are valid for AP processing. To process a "
194 "band not on this list, the appropriate band specific columns "
195 "must be added to the Apdb schema in dax_apdb.",
196 )
197 diaCatalogLoader = pexConfig.ConfigurableField(
198 target=LoadDiaCatalogsTask,
199 doc="Task to load DiaObjects and DiaSources from the Apdb.",
200 )
201 associator = pexConfig.ConfigurableField(
202 target=AssociationTask,
203 doc="Task used to associate DiaSources with DiaObjects.",
204 )
205 diaCalculation = pexConfig.ConfigurableField(
206 target=DiaObjectCalculationTask,
207 doc="Task to compute summary statistics for DiaObjects.",
208 )
209 diaForcedSource = pexConfig.ConfigurableField(
210 target=DiaForcedSourceTask,
211 doc="Task used for force photometer DiaObject locations in direct and "
212 "difference images.",
213 )
214 alertPackager = pexConfig.ConfigurableField(
215 target=PackageAlertsTask,
216 doc="Subtask for packaging Ap data into alerts.",
217 )
218 doPackageAlerts = pexConfig.Field(
219 dtype=bool,
220 default=False,
221 doc="Package Dia-data into serialized alerts for distribution and "
222 "write them to disk.",
223 )
224 doWriteAssociatedSources = pexConfig.Field(
225 dtype=bool,
226 default=False,
227 doc="Write out associated and SDMed DiaSources.",
228 )
230 def setDefaults(self):
231 self.apdb.dia_object_index = "baseline"
232 self.apdb.dia_object_columns = []
233 self.apdb.extra_schema_file = os.path.join(
234 "${AP_ASSOCIATION_DIR}",
235 "data",
236 "apdb-ap-pipe-schema-extra.yaml")
237 self.diaCalculation.plugins = ["ap_meanPosition",
238 "ap_HTMIndex",
239 "ap_nDiaSources",
240 "ap_diaObjectFlag",
241 "ap_meanFlux",
242 "ap_percentileFlux",
243 "ap_sigmaFlux",
244 "ap_chi2Flux",
245 "ap_madFlux",
246 "ap_skewFlux",
247 "ap_minMaxFlux",
248 "ap_maxSlopeFlux",
249 "ap_meanErrFlux",
250 "ap_linearFit",
251 "ap_stetsonJ",
252 "ap_meanTotFlux",
253 "ap_sigmaTotFlux"]
255 def validate(self):
256 pexConfig.Config.validate(self)
257 if self.diaCatalogLoader.htmLevel != \
258 self.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
259 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
260 "equal to HTMIndexDiaCalculationPlugin index "
261 "level.")
262 if "ap_HTMIndex" not in self.diaCalculation.plugins:
263 raise ValueError("DiaPipe requires the ap_HTMIndex plugin "
264 "be enabled for proper insertion into the Apdb.")
267class DiaPipelineTask(pipeBase.PipelineTask):
268 """Task for loading, associating and storing Difference Image Analysis
269 (DIA) Objects and Sources.
270 """
271 ConfigClass = DiaPipelineConfig
272 _DefaultName = "diaPipe"
273 RunnerClass = pipeBase.ButlerInitializedTaskRunner
275 def __init__(self, initInputs=None, **kwargs):
276 super().__init__(**kwargs)
277 self.apdb = self.config.apdb.apply(
278 afw_schemas=dict(DiaObject=make_dia_object_schema(),
279 DiaSource=make_dia_source_schema()))
280 self.makeSubtask("diaCatalogLoader")
281 self.makeSubtask("associator")
282 self.makeSubtask("diaCalculation")
283 self.makeSubtask("diaForcedSource")
284 if self.config.doPackageAlerts:
285 self.makeSubtask("alertPackager")
287 def runQuantum(self, butlerQC, inputRefs, outputRefs):
288 inputs = butlerQC.get(inputRefs)
289 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
290 returnMaxBits=True)
291 inputs["ccdExposureIdBits"] = expBits
292 inputs["band"] = butlerQC.quantum.dataId["band"]
294 outputs = self.run(**inputs)
296 butlerQC.put(outputs, outputRefs)
298 @pipeBase.timeMethod
299 def run(self,
300 diaSourceTable,
301 diffIm,
302 exposure,
303 warpedExposure,
304 ccdExposureIdBits,
305 band):
306 """Process DiaSources and DiaObjects.
308 Load previous DiaObjects and their DiaSource history. Calibrate the
309 values in the diaSourceCat. Associate new DiaSources with previous
310 DiaObjects. Run forced photometry at the updated DiaObject locations.
311 Store the results in the Alert Production Database (Apdb).
313 Parameters
314 ----------
315 diaSourceTable : `pandas.DataFrame`
316 Newly detected DiaSources.
317 diffIm : `lsst.afw.image.ExposureF`
318 Difference image exposure in which the sources in ``diaSourceCat``
319 were detected.
320 exposure : `lsst.afw.image.ExposureF`
321 Calibrated exposure differenced with a template to create
322 ``diffIm``.
323 warpedExposure : `lsst.afw.image.ExposureF`
324 Template exposure used to create diffIm.
325 ccdExposureIdBits : `int`
326 Number of bits used for a unique ``ccdVisitId``.
327 band : `str`
328 The band in which the new DiaSources were detected.
330 Returns
331 -------
332 results : `lsst.pipe.base.Struct`
333 Results struct with components.
335 - ``apdbMaker`` : Marker dataset to store in the Butler indicating
336 that this ccdVisit has completed successfully.
337 (`lsst.dax.apdb.ApdbConfig`)
338 - ``associatedDiaSources`` : Catalog of newly associated
339 DiaSources. (`pandas.DataFrame`)
340 """
341 # Load the DiaObjects and DiaSource history.
342 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
344 # Associate new DiaSources with existing DiaObjects.
345 assocResults = self.associator.run(diaSourceTable,
346 loaderResult.diaObjects)
348 # Create new DiaObjects from unassociated diaSources.
349 createResults = self.createNewDiaObjects(assocResults.diaSources)
350 self._add_association_meta_data(assocResults.nUpdatedDiaObjects,
351 assocResults.nUnassociatedDiaObjects,
352 len(createResults.newDiaObjects))
354 # Index the DiaSource catalog for this visit after all associations
355 # have been made.
356 updatedDiaObjectIds = createResults.diaSources["diaObjectId"][
357 createResults.diaSources["diaObjectId"] != 0].to_numpy()
358 diaSources = createResults.diaSources.set_index(["diaObjectId",
359 "filterName",
360 "diaSourceId"],
361 drop=False)
363 # Append new DiaObjects and DiaSources to their previous history.
364 diaObjects = loaderResult.diaObjects.append(
365 createResults.newDiaObjects.set_index("diaObjectId", drop=False),
366 sort=True)
367 if self.testDataFrameIndex(diaObjects):
368 raise RuntimeError(
369 "Duplicate DiaObjects created after association. This is "
370 "likely due to re-running data with an already populated "
371 "Apdb. If this was not the case then there was an unexpected "
372 "failure in Association while matching and creating new "
373 "DiaObjects and should be reported. Exiting.")
374 mergedDiaSourceHistory = loaderResult.diaSources.append(
375 diaSources,
376 sort=True)
377 # Test for DiaSource duplication first. If duplicates are found,
378 # this likely means this is duplicate data being processed and sent
379 # to the Apdb.
380 if self.testDataFrameIndex(mergedDiaSourceHistory):
381 raise RuntimeError(
382 "Duplicate DiaSources found after association and merging "
383 "with history. This is likely due to re-running data with an "
384 "already populated Apdb. If this was not the case then there "
385 "was an unexpected failure in Association while matching "
386 "sources to objects, and should be reported. Exiting.")
388 # Compute DiaObject Summary statistics from their full DiaSource
389 # history.
390 diaCalResult = self.diaCalculation.run(
391 diaObjects,
392 mergedDiaSourceHistory,
393 updatedDiaObjectIds,
394 [band])
395 # Test for duplication in the updated DiaObjects.
396 if self.testDataFrameIndex(diaCalResult.diaObjectCat):
397 raise RuntimeError(
398 "Duplicate DiaObjects (loaded + updated) created after "
399 "DiaCalculation. This is unexpected behavior and should be "
400 "reported. Existing.")
401 if self.testDataFrameIndex(diaCalResult.updatedDiaObjects):
402 raise RuntimeError(
403 "Duplicate DiaObjects (updated) created after "
404 "DiaCalculation. This is unexpected behavior and should be "
405 "reported. Existing.")
407 # Force photometer on the Difference and Calibrated exposures using
408 # the new and updated DiaObject locations.
409 diaForcedSources = self.diaForcedSource.run(
410 diaCalResult.diaObjectCat,
411 diaCalResult.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
412 ccdExposureIdBits,
413 exposure,
414 diffIm)
416 # Store DiaSources and updated DiaObjects in the Apdb.
417 self.apdb.storeDiaSources(diaSources)
418 self.apdb.storeDiaObjects(
419 diaCalResult.updatedDiaObjects,
420 exposure.visitInfo.date.toPython())
421 self.apdb.storeDiaForcedSources(diaForcedSources)
423 if self.config.doPackageAlerts:
424 if len(loaderResult.diaForcedSources) > 1:
425 diaForcedSources = diaForcedSources.append(
426 loaderResult.diaForcedSources,
427 sort=True)
428 if self.testDataFrameIndex(diaForcedSources):
429 self.log.warn(
430 "Duplicate DiaForcedSources created after merge with "
431 "history and new sources. This may cause downstream "
432 "problems. Dropping duplicates.")
433 # Drop duplicates via index and keep the first appearance.
434 # Reset due to the index shape being slight different than
435 # expected.
436 diaForcedSources = diaForcedSources.groupby(
437 diaForcedSources.index).first()
438 diaForcedSources.reset_index(drop=True, inplace=True)
439 diaForcedSources.set_index(
440 ["diaObjectId", "diaForcedSourceId"],
441 drop=False,
442 inplace=True)
443 self.alertPackager.run(diaSources,
444 diaCalResult.diaObjectCat,
445 loaderResult.diaSources,
446 diaForcedSources,
447 diffIm,
448 warpedExposure,
449 ccdExposureIdBits)
451 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
452 associatedDiaSources=diaSources)
454 def createNewDiaObjects(self, diaSources):
455 """Loop through the set of DiaSources and create new DiaObjects
456 for unassociated DiaSources.
458 Parameters
459 ----------
460 diaSources : `pandas.DataFrame`
461 Set of DiaSources to create new DiaObjects from.
463 Returns
464 -------
465 results : `lsst.pipe.base.Struct`
466 Results struct containing:
468 - ``diaSources`` : DiaSource catalog with updated DiaObject ids.
469 (`pandas.DataFrame`)
470 - ``newDiaObjects`` : Newly created DiaObjects from the
471 unassociated DiaSources. (`pandas.DataFrame`)
472 """
473 newDiaObjectsList = []
474 for idx, diaSource in diaSources.iterrows():
475 if diaSource["diaObjectId"] == 0:
476 newDiaObjectsList.append(
477 self._initialize_dia_object(diaSource["diaSourceId"]))
478 diaSources.loc[idx, "diaObjectId"] = diaSource["diaSourceId"]
479 if len(newDiaObjectsList) > 0:
480 newDiaObjects = pd.DataFrame(data=newDiaObjectsList)
481 else:
482 tmpObj = self._initialize_dia_object(0)
483 newDiaObjects = pd.DataFrame(data=newDiaObjectsList,
484 columns=tmpObj.keys())
485 return pipeBase.Struct(diaSources=diaSources,
486 newDiaObjects=pd.DataFrame(data=newDiaObjects))
488 def _initialize_dia_object(self, objId):
489 """Create a new DiaObject with values required to be initialized by the
490 Ppdb.
492 Parameters
493 ----------
494 objid : `int`
495 ``diaObjectId`` value for the of the new DiaObject.
497 Returns
498 -------
499 diaObject : `dict`
500 Newly created DiaObject with keys:
502 ``diaObjectId``
503 Unique DiaObjectId (`int`).
504 ``pmParallaxNdata``
505 Number of data points used for parallax calculation (`int`).
506 ``nearbyObj1``
507 Id of the a nearbyObject in the Object table (`int`).
508 ``nearbyObj2``
509 Id of the a nearbyObject in the Object table (`int`).
510 ``nearbyObj3``
511 Id of the a nearbyObject in the Object table (`int`).
512 ``?PSFluxData``
513 Number of data points used to calculate point source flux
514 summary statistics in each bandpass (`int`).
515 """
516 new_dia_object = {"diaObjectId": objId,
517 "pmParallaxNdata": 0,
518 "nearbyObj1": 0,
519 "nearbyObj2": 0,
520 "nearbyObj3": 0,
521 "flags": 0}
522 for f in ["u", "g", "r", "i", "z", "y"]:
523 new_dia_object["%sPSFluxNdata" % f] = 0
524 return new_dia_object
526 def testDataFrameIndex(self, df):
527 """Test the sorted DataFrame index for duplicates.
529 Wrapped as a separate function to allow for mocking of the this task
530 in unittesting. Default of a mock return for this test is True.
532 Parameters
533 ----------
534 df : `pandas.DataFrame`
535 DataFrame to text.
537 Returns
538 -------
539 `bool`
540 True if DataFrame contains duplicate rows.
541 """
542 return df.index.has_duplicates
544 def _add_association_meta_data(self,
545 nUpdatedDiaObjects,
546 nUnassociatedDiaObjects,
547 nNewDiaObjects):
548 """Store summaries of the association step in the task metadata.
550 Parameters
551 ----------
552 nUpdatedDiaObjects : `int`
553 Number of previous DiaObjects associated and updated in this
554 ccdVisit.
555 nUnassociatedDiaObjects : `int`
556 Number of previous DiaObjects that were not associated or updated
557 in this ccdVisit.
558 nNewDiaObjects : `int`
559 Number of newly created DiaObjects for this ccdVisit.
560 """
561 self.metadata.add('numUpdatedDiaObjects', nUpdatedDiaObjects)
562 self.metadata.add('numUnassociatedDiaObjects', nUnassociatedDiaObjects)
563 self.metadata.add('numNewDiaObjects', nNewDiaObjects)
566class DiaPipelineSolarSystemConnections(DiaPipelineConnections):
567 ssObjects = connTypes.Input(
568 doc="Solar System Objects observable in this visit.",
569 name="visitSsObjects",
570 storageClass="DataFrame",
571 dimensions=("instrument", "visit"),
572 )
573 ssObjectAssocDiaSources = connTypes.Output(
574 doc="DiaSources associated with existing Solar System objects..",
575 name="{fakesType}{coaddName}Diff_ssObjectAssocDiaSrc",
576 storageClass="DataFrame",
577 dimensions=("instrument", "visit", "detector"),
578 )
581class DiaPipelineSolarySystemConfig(DiaPipelineConfig,
582 pipelineConnections=DiaPipelineSolarSystemConnections):
583 solarSystemAssociation = pexConfig.ConfigurableField(
584 target=SolarSystemAssociationTask,
585 doc="Task used to associate DiaSources with Solar System Objects.",
586 )
589class DiaPipelineSolarSystemTask(DiaPipelineTask):
590 """Task for loading and storing Difference Image Analysis
591 (DIA) Sources after associating them to previous DiaObjects and
592 SSObjects.
594 SSO behavior currently necessitates a separate pipelinetask, however, after
595 DM-31389 is merged this SSO specific DiaPipe will merge into the default
596 class.
597 """
598 ConfigClass = DiaPipelineSolarySystemConfig
599 _DefaultName = "diaPipeSSO"
600 RunnerClass = pipeBase.ButlerInitializedTaskRunner
602 def __init__(self, initInputs=None, **kwargs):
603 super().__init__(**kwargs)
604 self.makeSubtask("solarSystemAssociation")
606 @pipeBase.timeMethod
607 def run(self,
608 diaSourceTable,
609 ssObjects,
610 diffIm,
611 exposure,
612 warpedExposure,
613 ccdExposureIdBits,
614 band):
615 """Process DiaSources and DiaObjects.
617 Load previous DiaObjects and their DiaSource history. Calibrate the
618 values in the ``diaSourceTable``. Associate new DiaSources with previous
619 DiaObjects. Run forced photometry at the updated DiaObject locations.
620 Store the results in the Alert Production Database (Apdb).
622 Parameters
623 ----------
624 diaSourceTable : `pandas.DataFrame`
625 Newly detected DiaSources.
626 diffIm : `lsst.afw.image.ExposureF`
627 Difference image exposure in which the sources in
628 ``diaSourceTable`` were detected.
629 exposure : `lsst.afw.image.ExposureF`
630 Calibrated exposure differenced with a template to create
631 ``diffIm``.
632 warpedExposure : `lsst.afw.image.ExposureF`
633 Template exposure used to create diffIm.
634 ccdExposureIdBits : `int`
635 Number of bits used for a unique ``ccdVisitId``.
636 band : `str`
637 The band in which the new DiaSources were detected.
639 Returns
640 -------
641 results : `lsst.pipe.base.Struct`
642 Results struct with components.
644 - ``apdbMaker`` : Marker dataset to store in the Butler indicating
645 that this ccdVisit has completed successfully.
646 (`lsst.dax.apdb.ApdbConfig`)
647 - ``associatedDiaSources`` : Full set of DiaSources associated
648 to current and new DiaObjects. This is an optional Butler output.
649 (`pandas.DataFrame`)
650 - ``ssObjectAssocDiaSources`` : Set of DiaSources associated with
651 solar system objects. (`pandas.DataFrame`)
652 """
653 # Load the DiaObjects and DiaSource history.
654 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
656 # Associate new DiaSources with existing DiaObjects and update
657 # DiaObject summary statistics using the full DiaSource history.
658 assocResults = self.associator.run(diaSourceTable,
659 loaderResult.diaObjects,
660 loaderResult.diaSources)
661 ssObjectAssocResults = self.solarSystemAssociation.run(
662 diaSourceTable.reset_index(drop=True),
663 ssObjects)
665 mergedDiaSourceHistory = loaderResult.diaSources.append(
666 assocResults.diaSources,
667 sort=True)
668 # Test for DiaSource duplication first. If duplicates are found,
669 # this likely means this is duplicate data being processed and sent
670 # to the Apdb.
671 if self.testDataFrameIndex(mergedDiaSourceHistory):
672 raise RuntimeError(
673 "Duplicate DiaSources found after association and merging "
674 "with history. This is likely due to re-running data with an "
675 "already populated Apdb. If this was not the case then there "
676 "was an unexpected failure in Association while matching "
677 "sources to objects, and should be reported. Exiting.")
679 diaCalResult = self.diaCalculation.run(
680 assocResults.diaObjects,
681 mergedDiaSourceHistory,
682 assocResults.matchedDiaObjectIds,
683 [band])
684 if self.testDataFrameIndex(diaCalResult.diaObjectCat):
685 raise RuntimeError(
686 "Duplicate DiaObjects (loaded + updated) created after "
687 "DiaCalculation. This is unexpected behavior and should be "
688 "reported. Existing.")
689 if self.testDataFrameIndex(diaCalResult.updatedDiaObjects):
690 raise RuntimeError(
691 "Duplicate DiaObjects (updated) created after "
692 "DiaCalculation. This is unexpected behavior and should be "
693 "reported. Existing.")
695 # Force photometer on the Difference and Calibrated exposures using
696 # the new and updated DiaObject locations.
697 diaForcedSources = self.diaForcedSource.run(
698 diaCalResult.diaObjectCat,
699 diaCalResult.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
700 ccdExposureIdBits,
701 exposure,
702 diffIm)
704 # Store DiaSources and updated DiaObjects in the Apdb.
705 self.apdb.storeDiaSources(assocResults.diaSources)
706 self.apdb.storeDiaObjects(
707 diaCalResult.updatedDiaObjects,
708 exposure.visitInfo.date.toPython())
709 self.apdb.storeDiaForcedSources(diaForcedSources)
711 if self.config.doPackageAlerts:
712 if len(loaderResult.diaForcedSources) > 1:
713 diaForcedSources = diaForcedSources.append(
714 loaderResult.diaForcedSources,
715 sort=True)
716 if self.testDataFrameIndex(diaForcedSources):
717 self.log.warn(
718 "Duplicate DiaForcedSources created after merge with "
719 "history and new sources. This may cause downstream "
720 "problems. Dropping duplicates.")
721 # Drop duplicates via index and keep the first appearance.
722 # Reset due to the index shape being slight different than
723 # expected.
724 diaForcedSources = diaForcedSources.groupby(
725 diaForcedSources.index).first()
726 diaForcedSources.reset_index(drop=True, inplace=True)
727 diaForcedSources.set_index(
728 ["diaObjectId", "diaForcedSourceId"],
729 drop=False,
730 inplace=True)
731 self.alertPackager.run(assocResults.diaSources,
732 diaCalResult.diaObjectCat,
733 loaderResult.diaSources,
734 diaForcedSources,
735 diffIm,
736 warpedExposure,
737 ccdExposureIdBits)
739 return pipeBase.Struct(
740 apdbMarker=self.config.apdb.value,
741 associatedDiaSources=assocResults.diaSources,
742 ssObjectAssocDiaSources=ssObjectAssocResults.ssoAssocDiaSources)