lsst.pipe.drivers  13.0-7-g11ca5b2
 All Classes Namespaces Files Functions Variables Pages
constructCalibs.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 import sys
3 import math
4 import time
5 import argparse
6 import traceback
7 
8 import numpy as np
9 from builtins import zip
10 from builtins import range
11 
12 from lsst.pex.config import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base as dafBase
15 import lsst.afw.math as afwMath
16 import lsst.afw.geom as afwGeom
17 import lsst.afw.detection as afwDet
18 import lsst.afw.image as afwImage
19 import lsst.meas.algorithms as measAlg
20 from lsst.pipe.tasks.repair import RepairTask
21 from lsst.ip.isr import IsrTask
22 
23 from lsst.ctrl.pool.parallel import BatchPoolTask
24 from lsst.ctrl.pool.pool import Pool, NODE
25 
26 from .checksum import checksum
27 from .utils import getDataRef
28 
29 
30 class CalibStatsConfig(Config):
31  """Parameters controlling the measurement of background statistics"""
32  stat = Field(doc="Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
33  default=int(afwMath.MEANCLIP))
34  clip = Field(doc="Clipping threshold for background",
35  dtype=float, default=3.0)
36  nIter = Field(doc="Clipping iterations for background",
37  dtype=int, default=3)
38  mask = ListField(doc="Mask planes to reject",
39  dtype=str, default=["DETECTED", "BAD"])
40 
41 
42 class CalibStatsTask(Task):
43  """Measure statistics on the background
44 
45  This can be useful for scaling the background, e.g., for flats and fringe frames.
46  """
47  ConfigClass = CalibStatsConfig
48 
49  def run(self, exposureOrImage):
50  """!Measure a particular statistic on an image (of some sort).
51 
52  @param exposureOrImage Exposure, MaskedImage or Image.
53  @return Value of desired statistic
54  """
55  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
56  afwImage.MaskU.getPlaneBitMask(self.config.mask))
57  try:
58  image = exposureOrImage.getMaskedImage()
59  except:
60  try:
61  image = exposureOrImage.getImage()
62  except:
63  image = exposureOrImage
64 
65  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
66 
67 
68 class CalibCombineConfig(Config):
69  """Configuration for combining calib images"""
70  rows = Field(doc="Number of rows to read at a time",
71  dtype=int, default=512)
72  mask = ListField(doc="Mask planes to respect", dtype=str,
73  default=["SAT", "DETECTED", "INTRP"])
74  combine = Field(doc="Statistic to use for combination (from lsst.afw.math)", dtype=int,
75  default=int(afwMath.MEANCLIP))
76  clip = Field(doc="Clipping threshold for combination",
77  dtype=float, default=3.0)
78  nIter = Field(doc="Clipping iterations for combination",
79  dtype=int, default=3)
80  stats = ConfigurableField(target=CalibStatsTask,
81  doc="Background statistics configuration")
82 
83 
84 class CalibCombineTask(Task):
85  """Task to combine calib images"""
86  ConfigClass = CalibCombineConfig
87 
88  def __init__(self, *args, **kwargs):
89  Task.__init__(self, *args, **kwargs)
90  self.makeSubtask("stats")
91 
92  def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
93  """!Combine calib images for a single sensor
94 
95  @param sensorRefList List of data references to combine (for a single sensor)
96  @param expScales List of scales to apply for each exposure
97  @param finalScale Desired scale for final combined image
98  @param inputName Data set name for inputs
99  @return combined image
100  """
101  width, height = self.getDimensions(sensorRefList)
102  maskVal = 0
103  for mask in self.config.mask:
104  maskVal |= afwImage.MaskU.getPlaneBitMask(mask)
105  stats = afwMath.StatisticsControl(
106  self.config.clip, self.config.nIter, maskVal)
107 
108  # Combine images
109  combined = afwImage.MaskedImageF(width, height)
110  numImages = len(sensorRefList)
111  imageList = [None]*numImages
112  for start in range(0, height, self.config.rows):
113  rows = min(self.config.rows, height - start)
114  box = afwGeom.Box2I(afwGeom.Point2I(0, start),
115  afwGeom.Extent2I(width, rows))
116  subCombined = combined.Factory(combined, box)
117 
118  for i, sensorRef in enumerate(sensorRefList):
119  if sensorRef is None:
120  imageList[i] = None
121  continue
122  exposure = sensorRef.get(inputName + "_sub", bbox=box)
123  if expScales is not None:
124  self.applyScale(exposure, expScales[i])
125  imageList[i] = exposure.getMaskedImage()
126 
127  self.combine(subCombined, imageList, stats)
128 
129  if finalScale is not None:
130  background = self.stats.run(combined)
131  self.log.info("%s: Measured background of stack is %f; adjusting to %f" %
132  (NODE, background, finalScale))
133  combined *= finalScale / background
134 
135  return afwImage.DecoratedImageF(combined.getImage())
136 
137  def getDimensions(self, sensorRefList, inputName="postISRCCD"):
138  """Get dimensions of the inputs"""
139  dimList = []
140  for sensorRef in sensorRefList:
141  if sensorRef is None:
142  continue
143  md = sensorRef.get(inputName + "_md")
144  dimList.append(afwGeom.Extent2I(
145  md.get("NAXIS1"), md.get("NAXIS2")))
146  return getSize(dimList)
147 
148  def applyScale(self, exposure, scale=None):
149  """Apply scale to input exposure
150 
151  This implementation applies a flux scaling: the input exposure is
152  divided by the provided scale.
153  """
154  if scale is not None:
155  mi = exposure.getMaskedImage()
156  mi /= scale
157 
158  def combine(self, target, imageList, stats):
159  """!Combine multiple images
160 
161  @param target Target image to receive the combined pixels
162  @param imageList List of input images
163  @param stats Statistics control
164  """
165  images = [img for img in imageList if img is not None]
166  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
167 
168 
169 def getSize(dimList):
170  """Determine a consistent size, given a list of image sizes"""
171  dim = set((w, h) for w, h in dimList)
172  dim.discard(None)
173  if len(dim) != 1:
174  raise RuntimeError("Inconsistent dimensions: %s" % dim)
175  return dim.pop()
176 
177 
178 def dictToTuple(dict_, keys):
179  """!Return a tuple of specific values from a dict
180 
181  This provides a hashable representation of the dict from certain keywords.
182  This can be useful for creating e.g., a tuple of the values in the DataId
183  that identify the CCD.
184 
185  @param dict_ dict to parse
186  @param keys keys to extract (order is important)
187  @return tuple of values
188  """
189  return tuple(dict_[k] for k in keys)
190 
191 
192 def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"]):
193  """!Determine a list of CCDs from exposure references
194 
195  This essentially inverts the exposure-level references (which
196  provides a list of CCDs for each exposure), by providing
197  a dataId list for each CCD. Consider an input list of exposures
198  [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
199  function returns:
200 
201  {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
202 
203  This is a dict whose keys are tuples of the identifying values of a
204  CCD (usually just the CCD number) and the values are lists of dataIds
205  for that CCD in each exposure. A missing dataId is given the value
206  None.
207 
208  @param expRefList List of data references for exposures
209  @param level Level for the butler to generate CCDs
210  @param ccdKeys DataId keywords that identify a CCD
211  @return dict of data identifier lists for each CCD
212  """
213  expIdList = [[ccdRef.dataId for ccdRef in expRef.subItems(
214  level)] for expRef in expRefList]
215 
216  # Determine what additional keys make a CCD from an exposure
217  ccdKeys = set(ccdKeys) # Set of keywords in the dataId that identify a CCD
218  ccdNames = set() # Set of tuples which are values for each of the CCDs in an exposure
219  for ccdIdList in expIdList:
220  for ccdId in ccdIdList:
221  name = dictToTuple(ccdId, ccdKeys)
222  ccdNames.add(name)
223 
224  # Turn the list of CCDs for each exposure into a list of exposures for
225  # each CCD
226  ccdLists = {}
227  for n, ccdIdList in enumerate(expIdList):
228  for ccdId in ccdIdList:
229  name = dictToTuple(ccdId, ccdKeys)
230  if name not in ccdLists:
231  ccdLists[name] = []
232  ccdLists[name].append(ccdId)
233 
234  return ccdLists
235 
236 
237 class CalibIdAction(argparse.Action):
238  """Split name=value pairs and put the result in a dict"""
239 
240  def __call__(self, parser, namespace, values, option_string):
241  output = getattr(namespace, self.dest, {})
242  for nameValue in values:
243  name, sep, valueStr = nameValue.partition("=")
244  if not valueStr:
245  parser.error("%s value %s must be in form name=value" %
246  (option_string, nameValue))
247  output[name] = valueStr
248  setattr(namespace, self.dest, output)
249 
250 
251 class CalibArgumentParser(ArgumentParser):
252  """ArgumentParser for calibration construction"""
253 
254  def __init__(self, calibName, *args, **kwargs):
255  """Add a --calibId argument to the standard pipe_base argument parser"""
256  ArgumentParser.__init__(self, *args, **kwargs)
257  self.calibName = calibName
258  self.add_id_argument("--id", datasetType="raw",
259  help="input identifiers, e.g., --id visit=123 ccd=4")
260  self.add_argument("--calibId", nargs="*", action=CalibIdAction, default={},
261  help="identifiers for calib, e.g., --calibId version=1",
262  metavar="KEY=VALUE1[^VALUE2[^VALUE3...]")
263 
264  def parse_args(self, *args, **kwargs):
265  """Parse arguments
266 
267  Checks that the "--calibId" provided works.
268  """
269  namespace = ArgumentParser.parse_args(self, *args, **kwargs)
270 
271  keys = namespace.butler.getKeys(self.calibName)
272  parsed = {}
273  for name, value in namespace.calibId.items():
274  if name not in keys:
275  self.error(
276  "%s is not a relevant calib identifier key (%s)" % (name, keys))
277  parsed[name] = keys[name](value)
278  namespace.calibId = parsed
279 
280  return namespace
281 
282 
283 class CalibConfig(Config):
284  """Configuration for constructing calibs"""
285  clobber = Field(dtype=bool, default=True,
286  doc="Clobber existing processed images?")
287  isr = ConfigurableField(target=IsrTask, doc="ISR configuration")
288  dateObs = Field(dtype=str, default="dateObs",
289  doc="Key for observation date in exposure registry")
290  dateCalib = Field(dtype=str, default="calibDate",
291  doc="Key for calib date in calib registry")
292  filter = Field(dtype=str, default="filter",
293  doc="Key for filter name in exposure/calib registries")
294  combination = ConfigurableField(
295  target=CalibCombineTask, doc="Calib combination configuration")
296  ccdKeys = ListField(dtype=str, default=[
297  "ccd"], doc="DataId keywords specifying a CCD")
298  visitKeys = ListField(dtype=str, default=[
299  "visit"], doc="DataId keywords specifying a visit")
300  calibKeys = ListField(dtype=str, default=[],
301  doc="DataId keywords specifying a calibration")
302 
303  def setDefaults(self):
304  self.isr.doWrite = False
305 
306 
307 class CalibTaskRunner(TaskRunner):
308  """Get parsed values into the CalibTask.run"""
309  @staticmethod
310  def getTargetList(parsedCmd, **kwargs):
311  return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
312 
313  def __call__(self, args):
314  """Call the Task with the kwargs from getTargetList"""
315  task = self.TaskClass(config=self.config, log=self.log)
316  if self.doRaise:
317  result = task.run(**args)
318  else:
319  try:
320  result = task.run(**args)
321  except Exception as e:
322  task.log.fatal("Failed: %s" % e)
323  traceback.print_exc(file=sys.stderr)
324 
325  if self.doReturnResults:
326  return Struct(
327  args=args,
328  metadata=task.metadata,
329  result=result,
330  )
331 
332 
333 class CalibTask(BatchPoolTask):
334  """!Base class for constructing calibs.
335 
336  This should be subclassed for each of the required calib types.
337  The subclass should be sure to define the following class variables:
338  * _DefaultName: default name of the task, used by CmdLineTask
339  * calibName: name of the calibration data set in the butler
340  The subclass may optionally set:
341  * filterName: filter name to give the resultant calib
342  """
343  ConfigClass = CalibConfig
344  RunnerClass = CalibTaskRunner
345  filterName = None
346  calibName = None
347 
348  def __init__(self, *args, **kwargs):
349  """Constructor"""
350  BatchPoolTask.__init__(self, *args, **kwargs)
351  self.makeSubtask("isr")
352  self.makeSubtask("combination")
353 
354  @classmethod
355  def batchWallTime(cls, time, parsedCmd, numCores):
356  numCcds = len(parsedCmd.butler.get("camera"))
357  numExps = len(cls.RunnerClass.getTargetList(
358  parsedCmd)[0]['expRefList'])
359  numCycles = int(numCcds/float(numCores) + 0.5)
360  return time*numExps*numCycles
361 
362  @classmethod
363  def _makeArgumentParser(cls, *args, **kwargs):
364  kwargs.pop("doBatch", False)
365  return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
366 
367  def run(self, expRefList, butler, calibId):
368  """!Construct a calib from a list of exposure references
369 
370  This is the entry point, called by the TaskRunner.__call__
371 
372  Only the master node executes this method.
373 
374  @param expRefList List of data references at the exposure level
375  @param butler Data butler
376  @param calibId Identifier dict for calib
377  """
378  outputId = self.getOutputId(expRefList, calibId)
379  ccdIdLists = getCcdIdListFromExposures(
380  expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
381 
382  # Ensure we can generate filenames for each output
383  outputIdItemList = list(outputId.items())
384  for ccdName in ccdIdLists:
385  dataId = dict(outputIdItemList + [(k, ccdName[i])
386  for i, k in enumerate(self.config.ccdKeys)])
387  try:
388  butler.get(self.calibName + "_filename", dataId)
389  except Exception as e:
390  raise RuntimeError(
391  "Unable to determine output filename from %s: %s" % (dataId, e))
392 
393  pool = Pool()
394  pool.storeSet(butler=butler)
395 
396  # Scatter: process CCDs independently
397  data = self.scatterProcess(pool, ccdIdLists)
398 
399  # Gather: determine scalings
400  scales = self.scale(ccdIdLists, data)
401 
402  # Scatter: combine
403  self.scatterCombine(pool, outputId, ccdIdLists, scales)
404 
405  def getOutputId(self, expRefList, calibId):
406  """!Generate the data identifier for the output calib
407 
408  The mean date and the common filter are included, using keywords
409  from the configuration. The CCD-specific part is not included
410  in the data identifier.
411 
412  @param expRefList List of data references at exposure level
413  @param calibId Data identifier elements for the calib provided by the user
414  @return data identifier
415  """
416  midTime = 0
417  filterName = None
418  for expRef in expRefList:
419  butler = expRef.getButler()
420  dataId = expRef.dataId
421 
422  midTime += self.getMjd(butler, dataId)
423  thisFilter = self.getFilter(
424  butler, dataId) if self.filterName is None else self.filterName
425  if filterName is None:
426  filterName = thisFilter
427  elif filterName != thisFilter:
428  raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
429  dataId, thisFilter, filterName))
430 
431  midTime /= len(expRefList)
432  date = str(dafBase.DateTime(
433  midTime, dafBase.DateTime.MJD).toPython().date())
434 
435  outputId = {self.config.filter: filterName,
436  self.config.dateCalib: date}
437  outputId.update(calibId)
438  return outputId
439 
440  def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
441  """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
442  if self.config.dateObs in dataId:
443  dateObs = dataId[self.config.dateObs]
444  else:
445  dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
446  if "T" not in dateObs:
447  dateObs = dateObs + "T12:00:00.0Z"
448  elif not dateObs.endswith("Z"):
449  dateObs += "Z"
450 
451  return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
452 
453  def getFilter(self, butler, dataId):
454  """Determine the filter from a data identifier"""
455  filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
456  return filt
457 
458  def scatterProcess(self, pool, ccdIdLists):
459  """!Scatter the processing among the nodes
460 
461  We scatter each CCD independently (exposures aren't grouped together),
462  to make full use of all available processors. This necessitates piecing
463  everything back together in the same format as ccdIdLists afterwards.
464 
465  Only the master node executes this method.
466 
467  @param pool Process pool
468  @param ccdIdLists Dict of data identifier lists for each CCD name
469  @return Dict of lists of returned data for each CCD name
470  """
471  dataIdList = sum(ccdIdLists.values(), [])
472  self.log.info("Scatter processing")
473 
474  resultList = pool.map(self.process, dataIdList)
475 
476  # Piece everything back together
477  data = dict((ccdName, [None] * len(expList))
478  for ccdName, expList in ccdIdLists.items())
479  indices = dict(sum([[(tuple(dataId.values()) if dataId is not None else None, (ccdName, expNum))
480  for expNum, dataId in enumerate(expList)]
481  for ccdName, expList in ccdIdLists.items()], []))
482  for dataId, result in zip(dataIdList, resultList):
483  if dataId is None:
484  continue
485  ccdName, expNum = indices[tuple(dataId.values())]
486  data[ccdName][expNum] = result
487 
488  return data
489 
490  def process(self, cache, ccdId, outputName="postISRCCD"):
491  """!Process a CCD, specified by a data identifier
492 
493  After processing, optionally returns a result (produced by
494  the 'processResult' method) calculated from the processed
495  exposure. These results will be gathered by the master node,
496  and is a means for coordinated scaling of all CCDs for flats,
497  etc.
498 
499  Only slave nodes execute this method.
500 
501  @param cache Process pool cache
502  @param ccdId Data identifier for CCD
503  @param outputName Output dataset name for butler
504  @return result from 'processResult'
505  """
506  if ccdId is None:
507  self.log.warn("Null identifier received on %s" % NODE)
508  return None
509  sensorRef = getDataRef(cache.butler, ccdId)
510  if self.config.clobber or not sensorRef.datasetExists(outputName):
511  self.log.info("Processing %s on %s" % (ccdId, NODE))
512  try:
513  exposure = self.processSingle(sensorRef)
514  except Exception as e:
515  self.log.warn("Unable to process %s: %s" % (ccdId, e))
516  raise
517  return None
518  self.processWrite(sensorRef, exposure)
519  else:
520  self.log.info(
521  "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
522  exposure = sensorRef.get(outputName, immediate=True)
523  return self.processResult(exposure)
524 
525  def processSingle(self, dataRef):
526  """Process a single CCD, specified by a data reference
527 
528  Generally, this simply means doing ISR.
529 
530  Only slave nodes execute this method.
531  """
532  return self.isr.runDataRef(dataRef).exposure
533 
534  def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
535  """!Write the processed CCD
536 
537  We need to write these out because we can't hold them all in
538  memory at once.
539 
540  Only slave nodes execute this method.
541 
542  @param dataRef Data reference
543  @param exposure CCD exposure to write
544  @param outputName Output dataset name for butler.
545  """
546  dataRef.put(exposure, outputName)
547 
548  def processResult(self, exposure):
549  """Extract processing results from a processed exposure
550 
551  This method generates what is gathered by the master node.
552  This can be a background measurement or similar for scaling
553  flat-fields. It must be picklable!
554 
555  Only slave nodes execute this method.
556  """
557  return None
558 
559  def scale(self, ccdIdLists, data):
560  """!Determine scaling across CCDs and exposures
561 
562  This is necessary mainly for flats, so as to determine a
563  consistent scaling across the entire focal plane. This
564  implementation is simply a placeholder.
565 
566  Only the master node executes this method.
567 
568  @param ccdIdLists Dict of data identifier lists for each CCD tuple
569  @param data Dict of lists of returned data for each CCD tuple
570  @return dict of Struct(ccdScale: scaling for CCD,
571  expScales: scaling for each exposure
572  ) for each CCD tuple
573  """
574  self.log.info("Scale on %s" % NODE)
575  return dict((name, Struct(ccdScale=None, expScales=[None] * len(ccdIdLists[name])))
576  for name in ccdIdLists)
577 
578  def scatterCombine(self, pool, outputId, ccdIdLists, scales):
579  """!Scatter the combination of exposures across multiple nodes
580 
581  In this case, we can only scatter across as many nodes as
582  there are CCDs.
583 
584  Only the master node executes this method.
585 
586  @param pool Process pool
587  @param outputId Output identifier (exposure part only)
588  @param ccdIdLists Dict of data identifier lists for each CCD name
589  @param scales Dict of structs with scales, for each CCD name
590  """
591  self.log.info("Scatter combination")
592  outputIdItemList = outputId.items()
593  data = [Struct(ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName],
594  outputId=dict(outputIdItemList +
595  [(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])) for
596  ccdName in ccdIdLists]
597  pool.map(self.combine, data)
598 
599  def combine(self, cache, struct):
600  """!Combine multiple exposures of a particular CCD and write the output
601 
602  Only the slave nodes execute this method.
603 
604  @param cache Process pool cache
605  @param struct Parameters for the combination, which has the following components:
606  * ccdIdList List of data identifiers for combination
607  * scales Scales to apply (expScales are scalings for each exposure,
608  ccdScale is final scale for combined image)
609  * outputId Data identifier for combined image (fully qualified for this CCD)
610  """
611  dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
612  dataId in struct.ccdIdList]
613  self.log.info("Combining %s on %s" % (struct.outputId, NODE))
614  calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
615  finalScale=struct.scales.ccdScale)
616 
617  self.recordCalibInputs(cache.butler, calib,
618  struct.ccdIdList, struct.outputId)
619 
620  self.interpolateNans(calib)
621 
622  self.write(cache.butler, calib, struct.outputId)
623 
624  def recordCalibInputs(self, butler, calib, dataIdList, outputId):
625  """!Record metadata including the inputs and creation details
626 
627  This metadata will go into the FITS header.
628 
629  @param butler Data butler
630  @param calib Combined calib exposure.
631  @param dataIdList List of data identifiers for calibration inputs
632  @param outputId Data identifier for output
633  """
634  header = calib.getMetadata()
635  header.add("OBSTYPE", self.calibName) # Used by ingestCalibs.py
636 
637  # date, time, host, and root
638  now = time.localtime()
639  header.add("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
640  header.add("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
641 
642  # Inputs
643  visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
644  dataId is not None]
645  for i, v in enumerate(sorted(set(visits))):
646  header.add("CALIB_INPUT_%d" % (i,), v)
647 
648  header.add("CALIB_ID", " ".join("%s=%s" % (key, value)
649  for key, value in outputId.items()))
650  checksum(calib, header)
651 
652  def interpolateNans(self, image):
653  """Interpolate over NANs in the combined image
654 
655  NANs can result from masked areas on the CCD. We don't want them getting
656  into our science images, so we replace them with the median of the image.
657  """
658  if hasattr(image, "getMaskedImage"): # Deal with Exposure vs Image
659  self.interpolateNans(image.getMaskedImage().getVariance())
660  image = image.getMaskedImage().getImage()
661  if hasattr(image, "getImage"): # Deal with DecoratedImage or MaskedImage vs Image
662  image = image.getImage()
663  array = image.getArray()
664  bad = np.isnan(array)
665  array[bad] = np.median(array[np.logical_not(bad)])
666 
667  def write(self, butler, exposure, dataId):
668  """!Write the final combined calib
669 
670  Only the slave nodes execute this method
671 
672  @param butler Data butler
673  @param exposure CCD exposure to write
674  @param dataId Data identifier for output
675  """
676  self.log.info("Writing %s on %s" % (dataId, NODE))
677  butler.put(exposure, self.calibName, dataId)
678 
679 
681  """Configuration for bias construction.
682 
683  No changes required compared to the base class, but
684  subclassed for distinction.
685  """
686  pass
687 
688 
689 class BiasTask(CalibTask):
690  """Bias construction"""
691  ConfigClass = BiasConfig
692  _DefaultName = "bias"
693  calibName = "bias"
694  filterName = "NONE" # Sets this filter name in the output
695 
696  @classmethod
697  def applyOverrides(cls, config):
698  """Overrides to apply for bias construction"""
699  config.isr.doBias = False
700  config.isr.doDark = False
701  config.isr.doFlat = False
702  config.isr.doFringe = False
703 
704 
706  """Task to combine dark images"""
707  def run(*args, **kwargs):
708  combined = CalibCombineTask.run(*args, **kwargs)
709 
710  # Update the metadata
711  visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
712  md = combined.getMetadata()
713  afwImage.setVisitInfoMetadata(md, visitInfo)
714 
715  return combined
716 
717 
719  """Configuration for dark construction"""
720  doRepair = Field(dtype=bool, default=True, doc="Repair artifacts?")
721  psfFwhm = Field(dtype=float, default=3.0, doc="Repair PSF FWHM (pixels)")
722  psfSize = Field(dtype=int, default=21, doc="Repair PSF size (pixels)")
723  crGrow = Field(dtype=int, default=2, doc="Grow radius for CR (pixels)")
724  repair = ConfigurableField(
725  target=RepairTask, doc="Task to repair artifacts")
726 
727  def setDefaults(self):
728  CalibConfig.setDefaults(self)
729  self.combination.retarget(DarkCombineTask)
730  self.combination.mask.append("CR")
731 
732 
734  """Dark construction
735 
736  The only major difference from the base class is a cosmic-ray
737  identification stage, and dividing each image by the dark time
738  to generate images of the dark rate.
739  """
740  ConfigClass = DarkConfig
741  _DefaultName = "dark"
742  calibName = "dark"
743  filterName = "NONE" # Sets this filter name in the output
744 
745  def __init__(self, *args, **kwargs):
746  CalibTask.__init__(self, *args, **kwargs)
747  self.makeSubtask("repair")
748 
749  @classmethod
750  def applyOverrides(cls, config):
751  """Overrides to apply for dark construction"""
752  config.isr.doDark = False
753  config.isr.doFlat = False
754  config.isr.doFringe = False
755 
756  def processSingle(self, sensorRef):
757  """Process a single CCD
758 
759  Besides the regular ISR, also masks cosmic-rays and divides each
760  processed image by the dark time to generate images of the dark rate.
761  The dark time is provided by the 'getDarkTime' method.
762  """
763  exposure = CalibTask.processSingle(self, sensorRef)
764 
765  if self.config.doRepair:
766  psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
767  self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
768  exposure.setPsf(psf)
769  self.repair.run(exposure, keepCRs=False)
770  if self.config.crGrow > 0:
771  mask = exposure.getMaskedImage().getMask().clone()
772  mask &= mask.getPlaneBitMask("CR")
773  fpSet = afwDet.FootprintSet(
774  mask.convertU(), afwDet.Threshold(0.5))
775  fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow, True)
776  fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
777 
778  mi = exposure.getMaskedImage()
779  mi /= self.getDarkTime(exposure)
780  return exposure
781 
782  def getDarkTime(self, exposure):
783  """Retrieve the dark time for an exposure"""
784  darkTime = exposure.getInfo().getVisitInfo().getDarkTime()
785  if not np.isfinite(darkTime):
786  raise RuntimeError("Non-finite darkTime")
787  return darkTime
788 
789 
791  """Configuration for flat construction"""
792  iterations = Field(dtype=int, default=10,
793  doc="Number of iterations for scale determination")
794  stats = ConfigurableField(target=CalibStatsTask,
795  doc="Background statistics configuration")
796 
797 
799  """Flat construction
800 
801  The principal change from the base class involves gathering the background
802  values from each image and using them to determine the scalings for the final
803  combination.
804  """
805  ConfigClass = FlatConfig
806  _DefaultName = "flat"
807  calibName = "flat"
808 
809  @classmethod
810  def applyOverrides(cls, config):
811  """Overrides for flat construction"""
812  config.isr.doFlat = False
813  config.isr.doFringe = False
814 
815  def __init__(self, *args, **kwargs):
816  CalibTask.__init__(self, *args, **kwargs)
817  self.makeSubtask("stats")
818 
819  def processResult(self, exposure):
820  return self.stats.run(exposure)
821 
822  def scale(self, ccdIdLists, data):
823  """Determine the scalings for the final combination
824 
825  We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
826  of one CCD to all the others in an exposure, and E_j is the scaling
827  of the exposure. We convert everything to logarithms so we can work
828  with a linear system. We determine the C_i and E_j from B_ij by iteration,
829  under the additional constraint that the average CCD scale is unity.
830 
831  This algorithm comes from Eugene Magnier and Pan-STARRS.
832  """
833  assert len(ccdIdLists.values()) > 0, "No successful CCDs"
834  lengths = set([len(expList) for expList in ccdIdLists.values()])
835  assert len(
836  lengths) == 1, "Number of successful exposures for each CCD differs"
837  assert tuple(lengths)[0] > 0, "No successful exposures"
838  # Format background measurements into a matrix
839  indices = dict((name, i) for i, name in enumerate(ccdIdLists))
840  bgMatrix = np.array([[0.0] * len(expList)
841  for expList in ccdIdLists.values()])
842  for name in ccdIdLists:
843  i = indices[name]
844  bgMatrix[i] = [
845  d if d is not None else np.nan for d in data[name]]
846 
847  numpyPrint = np.get_printoptions()
848  np.set_printoptions(threshold='nan')
849  self.log.info("Input backgrounds: %s" % bgMatrix)
850 
851  # Flat-field scaling
852  numCcds = len(ccdIdLists)
853  numExps = bgMatrix.shape[1]
854  # log(Background) for each exposure/component
855  bgMatrix = np.log(bgMatrix)
856  bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
857  # Initial guess at log(scale) for each component
858  compScales = np.zeros(numCcds)
859  expScales = np.array(
860  [(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
861 
862  for iterate in range(self.config.iterations):
863  compScales = np.array(
864  [(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
865  expScales = np.array(
866  [(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
867 
868  avgScale = np.average(np.exp(compScales))
869  compScales -= np.log(avgScale)
870  self.log.debug("Iteration %d exposure scales: %s",
871  iterate, np.exp(expScales))
872  self.log.debug("Iteration %d component scales: %s",
873  iterate, np.exp(compScales))
874 
875  expScales = np.array(
876  [(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
877 
878  if np.any(np.isnan(expScales)):
879  raise RuntimeError("Bad exposure scales: %s --> %s" %
880  (bgMatrix, expScales))
881 
882  expScales = np.exp(expScales)
883  compScales = np.exp(compScales)
884 
885  self.log.info("Exposure scales: %s" % expScales)
886  self.log.info("Component relative scaling: %s" % compScales)
887  np.set_printoptions(**numpyPrint)
888 
889  return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
890  for ccdName in ccdIdLists)
891 
892 
894  """Configuration for fringe construction"""
895  stats = ConfigurableField(target=CalibStatsTask,
896  doc="Background statistics configuration")
897  subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
898  doc="Background configuration")
899  detection = ConfigurableField(
900  target=measAlg.SourceDetectionTask, doc="Detection configuration")
901  detectSigma = Field(dtype=float, default=1.0,
902  doc="Detection PSF gaussian sigma")
903 
904 
906  """Fringe construction task
907 
908  The principal change from the base class is that the images are
909  background-subtracted and rescaled by the background.
910 
911  XXX This is probably not right for a straight-up combination, as we
912  are currently doing, since the fringe amplitudes need not scale with
913  the continuum.
914 
915  XXX Would like to have this do PCA and generate multiple images, but
916  that will take a bit of work with the persistence code.
917  """
918  ConfigClass = FringeConfig
919  _DefaultName = "fringe"
920  calibName = "fringe"
921 
922  @classmethod
923  def applyOverrides(cls, config):
924  """Overrides for fringe construction"""
925  config.isr.doFringe = False
926 
927  def __init__(self, *args, **kwargs):
928  CalibTask.__init__(self, *args, **kwargs)
929  self.makeSubtask("detection")
930  self.makeSubtask("stats")
931  self.makeSubtask("subtractBackground")
932 
933  def processSingle(self, sensorRef):
934  """Subtract the background and normalise by the background level"""
935  exposure = CalibTask.processSingle(self, sensorRef)
936  bgLevel = self.stats.run(exposure)
937  self.subtractBackground.run(exposure)
938  mi = exposure.getMaskedImage()
939  mi /= bgLevel
940  footprintSets = self.detection.detectFootprints(
941  exposure, sigma=self.config.detectSigma)
942  mask = exposure.getMaskedImage().getMask()
943  detected = 1 << mask.addMaskPlane("DETECTED")
944  for fpSet in (footprintSets.positive, footprintSets.negative):
945  if fpSet is not None:
946  afwDet.setMaskFromFootprintList(
947  mask, fpSet.getFootprints(), detected)
948  return exposure
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
Definition: checksum.py:29
def recordCalibInputs
Record metadata including the inputs and creation details.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.