lsst.cp.pipe  20.0.0-23-g10eeb28+452d7a9290
utils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24  'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25  'countMaskedPixels', 'checkExpLengthEqual']
26 
27 import re
28 import numpy as np
29 from scipy.optimize import leastsq
30 import numpy.polynomial.polynomial as poly
31 
32 import lsst.pipe.base as pipeBase
33 import lsst.ip.isr as ipIsr
34 from lsst.ip.isr import isrMock
35 import lsst.log
36 
37 import galsim
38 
39 
40 def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
41  """Calculate weighted reduced chi2.
42 
43  Parameters
44  ----------
45 
46  measured : `list`
47  List with measured data.
48 
49  model : `list`
50  List with modeled data.
51 
52  weightsMeasured : `list`
53  List with weights for the measured data.
54 
55  nData : `int`
56  Number of data points.
57 
58  nParsModel : `int`
59  Number of parameters in the model.
60 
61  Returns
62  -------
63 
64  redWeightedChi2 : `float`
65  Reduced weighted chi2.
66  """
67 
68  wRes = (measured - model)*weightsMeasured
69  return ((wRes*wRes).sum())/(nData-nParsModel)
70 
71 
72 def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
73  randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]):
74  """Create a pair or mock flats with isrMock.
75 
76  Parameters
77  ----------
78  expTime : `float`
79  Exposure time of the flats.
80 
81  gain : `float`, optional
82  Gain, in e/ADU.
83 
84  readNoiseElectrons : `float`, optional
85  Read noise rms, in electrons.
86 
87  fluxElectrons : `float`, optional
88  Flux of flats, in electrons per second.
89 
90  randomSeedFlat1 : `int`, optional
91  Random seed for the normal distrubutions for the mean signal and noise (flat1).
92 
93  randomSeedFlat2 : `int`, optional
94  Random seed for the normal distrubutions for the mean signal and noise (flat2).
95 
96  powerLawBfParams : `list`, optional
97  Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
98 
99  Returns
100  -------
101 
102  flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
103  First exposure of flat field pair.
104 
105  flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
106  Second exposure of flat field pair.
107 
108  Notes
109  -----
110  The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
111  information about their meaning, see the Galsim documentation
112  https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
113  and Gruen+15 (1501.02802).
114 
115  Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
116  """
117  flatFlux = fluxElectrons # e/s
118  flatMean = flatFlux*expTime # e
119  readNoise = readNoiseElectrons # e
120 
121  mockImageConfig = isrMock.IsrMock.ConfigClass()
122 
123  mockImageConfig.flatDrop = 0.99999
124  mockImageConfig.isTrimmed = True
125 
126  flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
127  flatExp2 = flatExp1.clone()
128  (shapeY, shapeX) = flatExp1.getDimensions()
129  flatWidth = np.sqrt(flatMean)
130 
131  rng1 = np.random.RandomState(randomSeedFlat1)
132  flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
133  (shapeX, shapeY))
134  rng2 = np.random.RandomState(randomSeedFlat2)
135  flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
136  (shapeX, shapeY))
137  # Simulate BF with power law model in galsim
138  if len(powerLawBfParams):
139  if not len(powerLawBfParams) == 8:
140  raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " +
141  f"Expected 8; passed {len(powerLawBfParams)}.")
142  cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
143  tempFlatData1 = galsim.Image(flatData1)
144  temp2FlatData1 = cd.applyForward(tempFlatData1)
145 
146  tempFlatData2 = galsim.Image(flatData2)
147  temp2FlatData2 = cd.applyForward(tempFlatData2)
148 
149  flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
150  flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
151  else:
152  flatExp1.image.array[:] = flatData1/gain # ADU
153  flatExp2.image.array[:] = flatData2/gain # ADU
154 
155  return flatExp1, flatExp2
156 
157 
158 def countMaskedPixels(maskedIm, maskPlane):
159  """Count the number of pixels in a given mask plane."""
160  maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
161  nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
162  return nPix
163 
164 
165 class PairedVisitListTaskRunner(pipeBase.TaskRunner):
166  """Subclass of TaskRunner for handling intrinsically paired visits.
167 
168  This transforms the processed arguments generated by the ArgumentParser
169  into the arguments expected by tasks which take visit pairs for their
170  run() methods.
171 
172  Such tasks' run() methods tend to take two arguments,
173  one of which is the dataRef (as usual), and the other is the list
174  of visit-pairs, in the form of a list of tuples.
175  This list is supplied on the command line as documented,
176  and this class parses that, and passes the parsed version
177  to the run() method.
178 
179  See pipeBase.TaskRunner for more information.
180  """
181 
182  @staticmethod
183  def getTargetList(parsedCmd, **kwargs):
184  """Parse the visit list and pass through explicitly."""
185  visitPairs = []
186  for visitStringPair in parsedCmd.visitPairs:
187  visitStrings = visitStringPair.split(",")
188  if len(visitStrings) != 2:
189  raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
190  visitStringPair))
191  try:
192  visits = [int(visit) for visit in visitStrings]
193  except Exception:
194  raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
195  visitPairs.append(visits)
196 
197  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
198 
199 
200 def parseCmdlineNumberString(inputString):
201  """Parse command line numerical expression sytax and return as list of int
202 
203  Take an input of the form "'1..5:2^123..126'" as a string, and return
204  a list of ints as [1, 3, 5, 123, 124, 125, 126]
205  """
206  outList = []
207  for subString in inputString.split("^"):
208  mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
209  if mat:
210  v1 = int(mat.group(1))
211  v2 = int(mat.group(2))
212  v3 = mat.group(3)
213  v3 = int(v3) if v3 else 1
214  for v in range(v1, v2 + 1, v3):
215  outList.append(int(v))
216  else:
217  outList.append(int(subString))
218  return outList
219 
220 
221 class SingleVisitListTaskRunner(pipeBase.TaskRunner):
222  """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
223 
224  This transforms the processed arguments generated by the ArgumentParser
225  into the arguments expected by tasks which require a list of visits
226  to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
227 
228  Such tasks' run() methods tend to take two arguments,
229  one of which is the dataRef (as usual), and the other is the list
230  of visits.
231  This list is supplied on the command line as documented,
232  and this class parses that, and passes the parsed version
233  to the run() method.
234 
235  See `lsst.pipe.base.TaskRunner` for more information.
236  """
237 
238  @staticmethod
239  def getTargetList(parsedCmd, **kwargs):
240  """Parse the visit list and pass through explicitly."""
241  # if this has been pre-parsed and therefore doesn't have length of one
242  # then something has gone wrong, so execution should stop here.
243  assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
244  visits = parseCmdlineNumberString(parsedCmd.visitList[0])
245 
246  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
247 
248 
249 class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
250  """A DataIdContainer for the tasks for which the output does
251  not yet exist."""
252 
253  def makeDataRefList(self, namespace):
254  """Compute refList based on idList.
255 
256  This method must be defined as the dataset does not exist before this
257  task is run.
258 
259  Parameters
260  ----------
261  namespace
262  Results of parsing the command-line.
263 
264  Notes
265  -----
266  Not called if ``add_id_argument`` called
267  with ``doMakeDataRefList=False``.
268  Note that this is almost a copy-and-paste of the vanilla
269  implementation, but without checking if the datasets already exist,
270  as this task exists to make them.
271  """
272  if self.datasetType is None:
273  raise RuntimeError("Must call setDatasetType first")
274  butler = namespace.butler
275  for dataId in self.idList:
276  refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
277  # exclude nonexistent data
278  # this is a recursive test, e.g. for the sake of "raw" data
279  if not refList:
280  namespace.log.warn("No data found for dataId=%s", dataId)
281  continue
282  self.refList += refList
283 
284 
285 def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
286  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
287 
288  optimize.leastsq returns the fractional covariance matrix. To estimate the
289  standard deviation of the fit parameters, multiply the entries of this matrix
290  by the unweighted reduced chi squared and take the square root of the diagonal elements.
291 
292  Parameters
293  ----------
294  initialParams : `list` of `float`
295  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
296  determines the degree of the polynomial.
297 
298  dataX : `numpy.array` of `float`
299  Data in the abscissa axis.
300 
301  dataY : `numpy.array` of `float`
302  Data in the ordinate axis.
303 
304  function : callable object (function)
305  Function to fit the data with.
306 
307  weightsY : `numpy.array` of `float`
308  Weights of the data in the ordinate axis.
309 
310  Return
311  ------
312  pFitSingleLeastSquares : `list` of `float`
313  List with fitted parameters.
314 
315  pErrSingleLeastSquares : `list` of `float`
316  List with errors for fitted parameters.
317 
318  reducedChiSqSingleLeastSquares : `float`
319  Reduced chi squared, unweighted if weightsY is not provided.
320  """
321  if weightsY is None:
322  weightsY = np.ones(len(dataX))
323 
324  def errFunc(p, x, y, weightsY=None):
325  if weightsY is None:
326  weightsY = np.ones(len(x))
327  return (function(p, x) - y)*weightsY
328 
329  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
330  args=(dataX, dataY, weightsY), full_output=1,
331  epsfcn=0.0001)
332 
333  if (len(dataY) > len(initialParams)) and pCov is not None:
334  reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY),
335  len(initialParams))
336  pCov *= reducedChiSq
337  else:
338  pCov = np.zeros((len(initialParams), len(initialParams)))
339  pCov[:, :] = np.nan
340  reducedChiSq = np.nan
341 
342  errorVec = []
343  for i in range(len(pFit)):
344  errorVec.append(np.fabs(pCov[i][i])**0.5)
345 
346  pFitSingleLeastSquares = pFit
347  pErrSingleLeastSquares = np.array(errorVec)
348 
349  return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
350 
351 
352 def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
353  """Do a fit using least squares and bootstrap to estimate parameter errors.
354 
355  The bootstrap error bars are calculated by fitting 100 random data sets.
356 
357  Parameters
358  ----------
359  initialParams : `list` of `float`
360  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
361  determines the degree of the polynomial.
362 
363  dataX : `numpy.array` of `float`
364  Data in the abscissa axis.
365 
366  dataY : `numpy.array` of `float`
367  Data in the ordinate axis.
368 
369  function : callable object (function)
370  Function to fit the data with.
371 
372  weightsY : `numpy.array` of `float`, optional.
373  Weights of the data in the ordinate axis.
374 
375  confidenceSigma : `float`, optional.
376  Number of sigmas that determine confidence interval for the bootstrap errors.
377 
378  Return
379  ------
380  pFitBootstrap : `list` of `float`
381  List with fitted parameters.
382 
383  pErrBootstrap : `list` of `float`
384  List with errors for fitted parameters.
385 
386  reducedChiSqBootstrap : `float`
387  Reduced chi squared, unweighted if weightsY is not provided.
388  """
389  if weightsY is None:
390  weightsY = np.ones(len(dataX))
391 
392  def errFunc(p, x, y, weightsY):
393  if weightsY is None:
394  weightsY = np.ones(len(x))
395  return (function(p, x) - y)*weightsY
396 
397  # Fit first time
398  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
399 
400  # Get the stdev of the residuals
401  residuals = errFunc(pFit, dataX, dataY, weightsY)
402  # 100 random data sets are generated and fitted
403  pars = []
404  for i in range(100):
405  randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
406  randomDataY = dataY + randomDelta
407  randomFit, _ = leastsq(errFunc, initialParams,
408  args=(dataX, randomDataY, weightsY), full_output=0)
409  pars.append(randomFit)
410  pars = np.array(pars)
411  meanPfit = np.mean(pars, 0)
412 
413  # confidence interval for parameter estimates
414  errPfit = confidenceSigma*np.std(pars, 0)
415  pFitBootstrap = meanPfit
416  pErrBootstrap = errPfit
417 
418  reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY),
419  len(initialParams))
420  return pFitBootstrap, pErrBootstrap, reducedChiSq
421 
422 
423 def funcPolynomial(pars, x):
424  """Polynomial function definition
425  Parameters
426  ----------
427  params : `list`
428  Polynomial coefficients. Its length determines the polynomial order.
429 
430  x : `numpy.array`
431  Signal mu (ADU).
432 
433  Returns
434  -------
435  C_00 (variance) in ADU^2.
436  """
437  return poly.polyval(x, [*pars]) # C_00
438 
439 
440 def funcAstier(pars, x):
441  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
442 
443  Parameters
444  ----------
445  params : `list`
446  Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
447 
448  x : `numpy.array`
449  Signal mu (ADU).
450 
451  Returns
452  -------
453  C_00 (variance) in ADU^2.
454  """
455  a00, gain, noise = pars
456  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
457 
458 
459 def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
460  """Check the exposure lengths of two exposures are equal.
461 
462  Parameters:
463  -----------
464  exp1 : `lsst.afw.image.exposure.ExposureF`
465  First exposure to check
466  exp2 : `lsst.afw.image.exposure.ExposureF`
467  Second exposure to check
468  v1 : `int` or `str`, optional
469  First visit of the visit pair
470  v2 : `int` or `str`, optional
471  Second visit of the visit pair
472  raiseWithMessage : `bool`
473  If True, instead of returning a bool, raise a RuntimeError if exposure
474  times are not equal, with a message about which visits mismatch if the
475  information is available.
476 
477  Raises:
478  -------
479  RuntimeError
480  Raised if the exposure lengths of the two exposures are not equal
481  """
482  expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
483  expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
484  if expTime1 != expTime2:
485  if raiseWithMessage:
486  msg = "Exposure lengths for visit pairs must be equal. " + \
487  "Found %s and %s" % (expTime1, expTime2)
488  if v1 and v2:
489  msg += " for visit pair %s, %s" % (v1, v2)
490  raise RuntimeError(msg)
491  else:
492  return False
493  return True
494 
495 
496 def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
497  checkTrim=True, logName=None):
498  """Check that appropriate ISR settings have been selected for the task.
499 
500  Note that this checks that the task itself is configured correctly rather
501  than checking a config.
502 
503  Parameters
504  ----------
505  isrTask : `lsst.ip.isr.IsrTask`
506  The task whose config is to be validated
507 
508  mandatory : `iterable` of `str`
509  isr steps that must be set to True. Raises if False or missing
510 
511  forbidden : `iterable` of `str`
512  isr steps that must be set to False. Raises if True, warns if missing
513 
514  desirable : `iterable` of `str`
515  isr steps that should probably be set to True. Warns is False, info if
516  missing
517 
518  undesirable : `iterable` of `str`
519  isr steps that should probably be set to False. Warns is True, info if
520  missing
521 
522  checkTrim : `bool`
523  Check to ensure the isrTask's assembly subtask is trimming the images.
524  This is a separate config as it is very ugly to do this within the
525  normal configuration lists as it is an option of a sub task.
526 
527  Raises
528  ------
529  RuntimeError
530  Raised if ``mandatory`` config parameters are False,
531  or if ``forbidden`` parameters are True.
532 
533  TypeError
534  Raised if parameter ``isrTask`` is an invalid type.
535 
536  Notes
537  -----
538  Logs warnings using an isrValidation logger for desirable/undesirable
539  options that are of the wrong polarity or if keys are missing.
540  """
541  if not isinstance(isrTask, ipIsr.IsrTask):
542  raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
543 
544  configDict = isrTask.config.toDict()
545 
546  if logName and isinstance(logName, str):
547  log = lsst.log.getLogger(logName)
548  else:
549  log = lsst.log.getLogger("isrValidation")
550 
551  if mandatory:
552  for configParam in mandatory:
553  if configParam not in configDict:
554  raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
555  if configDict[configParam] is False:
556  raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
557 
558  if forbidden:
559  for configParam in forbidden:
560  if configParam not in configDict:
561  log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
562  " forbidden list should each have an associated Field in IsrConfig:"
563  " check that there is not a typo in this case.")
564  continue
565  if configDict[configParam] is True:
566  raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
567 
568  if desirable:
569  for configParam in desirable:
570  if configParam not in configDict:
571  log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
572  " to set the equivalent for your obs_package to True.")
573  continue
574  if configDict[configParam] is False:
575  log.warn(f"Found config.isr.{configParam} set to False for this task." +
576  " The cp_pipe Config recommends setting this to True.")
577  if undesirable:
578  for configParam in undesirable:
579  if configParam not in configDict:
580  log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
581  " to set the equivalent for your obs_package to False.")
582  continue
583  if configDict[configParam] is True:
584  log.warn(f"Found config.isr.{configParam} set to True for this task." +
585  " The cp_pipe Config recommends setting this to False.")
586 
587  if checkTrim: # subtask setting, seems non-trivial to combine with above lists
588  if not isrTask.assembleCcd.config.doTrim:
589  raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
lsst.cp.pipe.utils.validateIsrConfig
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:496
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: utils.py:253
lsst.cp.pipe.utils.calculateWeightedReducedChi2
def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel)
Definition: utils.py:40
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:423
lsst.cp.pipe.utils.PairedVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:183
lsst.cp.pipe.utils.countMaskedPixels
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:158
lsst.cp.pipe.utils.SingleVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:239
lsst.cp.pipe.utils.PairedVisitListTaskRunner
Definition: utils.py:165
lsst.cp.pipe.utils.funcAstier
def funcAstier(pars, x)
Definition: utils.py:440
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:285
lsst.cp.pipe.utils.SingleVisitListTaskRunner
Definition: utils.py:221
lsst.cp.pipe.utils.checkExpLengthEqual
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:459
lsst.cp.pipe.utils.parseCmdlineNumberString
def parseCmdlineNumberString(inputString)
Definition: utils.py:200
lsst::ip::isr
lsst.cp.pipe.utils.makeMockFlats
def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[])
Definition: utils.py:72
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:352
lsst::pipe::base
lsst::log
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer
Definition: utils.py:249