lsst.pipe.drivers  21.0.0-5-gd00fb1e+55814f50cf
processCcdWithFakesDriver.py
Go to the documentation of this file.
1 from lsst.pipe.base import ArgumentParser
2 from lsst.pipe.tasks.processCcdWithFakes import ProcessCcdWithFakesTask
3 from lsst.pipe.tasks.postprocess import WriteSourceTableTask, TransformSourceTableTask
4 from lsst.pex.config import Config, Field, ConfigurableField, ListField
5 from lsst.ctrl.pool.parallel import BatchParallelTask, BatchTaskRunner
6 from lsst.meas.base import PerTractCcdDataIdContainer
7 
8 
10  processCcdWithFakes = ConfigurableField(
11  target=ProcessCcdWithFakesTask, doc="CCD with fakes processing task")
12  doMakeSourceTable = Field(dtype=bool, default=False,
13  doc="Do postprocessing tasks to write parquet Source Table?")
14  doSaveWideSourceTable = Field(dtype=bool, default=False,
15  doc=("Save the parquet version of the full src catalog?",
16  "Only respected if doMakeSourceTable"))
17  writeSourceTable = ConfigurableField(
18  target=WriteSourceTableTask, doc="Task to make parquet table for full src catalog")
19  transformSourceTable = ConfigurableField(
20  target=TransformSourceTableTask, doc="Transform Source Table to DPDD specification")
21  ignoreCcdList = ListField(dtype=int, default=[],
22  doc="List of CCDs to ignore when processing")
23  ccdKey = Field(dtype=str, default="ccd",
24  doc="DataId key corresponding to a single sensor")
25 
26 
28  """Run batches, and initialize Task"""
29  pass
30 
31 
32 class ProcessCcdWithFakesDriverTask(BatchParallelTask):
33  """Process CCDs in parallel for processCcdWithFakes
34  """
35  ConfigClass = ProcessCcdWithFakesDriverConfig
36  _DefaultName = "processCcdWithFakesDriver"
37  RunnerClass = ProcessCcdWithFakesTaskRunner
38 
39  def __init__(self, *args, **kwargs):
40  """
41  Parameters
42  ----------
43  kwargs : other keyword arguments for lsst.ctrl.pool.BatchParallelTask
44  """
45  BatchParallelTask.__init__(self, *args, **kwargs)
46  self.ignoreCcdsignoreCcds = set(self.config.ignoreCcdList)
47  self.makeSubtask("processCcdWithFakes")
48  if self.config.doMakeSourceTable:
49  self.makeSubtask("writeSourceTable")
50  self.makeSubtask("transformSourceTable")
51 
52  @classmethod
53  def _makeArgumentParser(cls, *args, **kwargs):
54  kwargs.pop("doBatch", False)
55  parser = ArgumentParser(name="processCcdWithFakesDriver", *args, **kwargs)
56  parser.add_id_argument("--id", "fakes_calexp",
57  help="data ID, e.g. --id visit=12345 ccd=67, tract=9813",
58  ContainerClass=PerTractCcdDataIdContainer)
59  return parser
60 
61  def runDataRef(self, sensorRef):
62  """Process a single CCD, with scatter-gather-scatter using MPI.
63  """
64  if sensorRef.dataId[self.config.ccdKey] in self.ignoreCcdsignoreCcds:
65  self.log.warn("Ignoring %s: CCD in ignoreCcdList" %
66  (sensorRef.dataId))
67  return None
68 
69  with self.logOperationlogOperationlogOperation("processing %s" % (sensorRef.dataId,)):
70  result = self.processCcdWithFakes.runDataRef(sensorRef)
71  if self.config.doMakeSourceTable:
72  parquet = self.writeSourceTable.run(result.outputCat,
73  ccdVisitId=sensorRef.get('ccdExposureId'))
74  if self.config.doSaveWideSourceTable:
75  sensorRef.put(parquet.table, 'fakes_source')
76 
77  df = self.transformSourceTable.run(parquet.table,
78  funcs=self.transformSourceTable.getFunctors(),
79  dataId=sensorRef.dataId)
80  self.transformSourceTable.write(df, sensorRef)
81 
82  return result
def logOperation(self, operation, catch=False, trace=True)