lsst.cp.pipe  21.0.0-7-gfd72ab2+cf01990774
utils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24  'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25  'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict']
26 
27 import re
28 import numpy as np
29 from scipy.optimize import leastsq
30 import numpy.polynomial.polynomial as poly
31 
32 import lsst.pipe.base as pipeBase
33 import lsst.ip.isr as ipIsr
34 from lsst.ip.isr import isrMock
35 import lsst.log
36 import lsst.afw.image
37 
38 import galsim
39 
40 
41 def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
42  """Calculate weighted reduced chi2.
43 
44  Parameters
45  ----------
46 
47  measured : `list`
48  List with measured data.
49 
50  model : `list`
51  List with modeled data.
52 
53  weightsMeasured : `list`
54  List with weights for the measured data.
55 
56  nData : `int`
57  Number of data points.
58 
59  nParsModel : `int`
60  Number of parameters in the model.
61 
62  Returns
63  -------
64 
65  redWeightedChi2 : `float`
66  Reduced weighted chi2.
67  """
68 
69  wRes = (measured - model)*weightsMeasured
70  return ((wRes*wRes).sum())/(nData-nParsModel)
71 
72 
73 def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
74  randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[],
75  expId1=0, expId2=1):
76  """Create a pair or mock flats with isrMock.
77 
78  Parameters
79  ----------
80  expTime : `float`
81  Exposure time of the flats.
82 
83  gain : `float`, optional
84  Gain, in e/ADU.
85 
86  readNoiseElectrons : `float`, optional
87  Read noise rms, in electrons.
88 
89  fluxElectrons : `float`, optional
90  Flux of flats, in electrons per second.
91 
92  randomSeedFlat1 : `int`, optional
93  Random seed for the normal distrubutions for the mean signal and noise (flat1).
94 
95  randomSeedFlat2 : `int`, optional
96  Random seed for the normal distrubutions for the mean signal and noise (flat2).
97 
98  powerLawBfParams : `list`, optional
99  Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
100 
101  expId1 : `int`, optional
102  Exposure ID for first flat.
103 
104  expId2 : `int`, optional
105  Exposure ID for second flat.
106 
107  Returns
108  -------
109 
110  flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
111  First exposure of flat field pair.
112 
113  flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
114  Second exposure of flat field pair.
115 
116  Notes
117  -----
118  The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
119  information about their meaning, see the Galsim documentation
120  https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
121  and Gruen+15 (1501.02802).
122 
123  Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
124  """
125  flatFlux = fluxElectrons # e/s
126  flatMean = flatFlux*expTime # e
127  readNoise = readNoiseElectrons # e
128 
129  mockImageConfig = isrMock.IsrMock.ConfigClass()
130 
131  mockImageConfig.flatDrop = 0.99999
132  mockImageConfig.isTrimmed = True
133 
134  flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
135  flatExp2 = flatExp1.clone()
136  (shapeY, shapeX) = flatExp1.getDimensions()
137  flatWidth = np.sqrt(flatMean)
138 
139  rng1 = np.random.RandomState(randomSeedFlat1)
140  flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
141  (shapeX, shapeY))
142  rng2 = np.random.RandomState(randomSeedFlat2)
143  flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
144  (shapeX, shapeY))
145  # Simulate BF with power law model in galsim
146  if len(powerLawBfParams):
147  if not len(powerLawBfParams) == 8:
148  raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. "
149  f"Expected 8; passed {len(powerLawBfParams)}.")
150  cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
151  tempFlatData1 = galsim.Image(flatData1)
152  temp2FlatData1 = cd.applyForward(tempFlatData1)
153 
154  tempFlatData2 = galsim.Image(flatData2)
155  temp2FlatData2 = cd.applyForward(tempFlatData2)
156 
157  flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
158  flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
159  else:
160  flatExp1.image.array[:] = flatData1/gain # ADU
161  flatExp2.image.array[:] = flatData2/gain # ADU
162 
163  visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime)
164  visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime)
165 
166  flatExp1.getInfo().setVisitInfo(visitInfoExp1)
167  flatExp2.getInfo().setVisitInfo(visitInfoExp2)
168 
169  return flatExp1, flatExp2
170 
171 
172 def countMaskedPixels(maskedIm, maskPlane):
173  """Count the number of pixels in a given mask plane."""
174  maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
175  nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
176  return nPix
177 
178 
179 class PairedVisitListTaskRunner(pipeBase.TaskRunner):
180  """Subclass of TaskRunner for handling intrinsically paired visits.
181 
182  This transforms the processed arguments generated by the ArgumentParser
183  into the arguments expected by tasks which take visit pairs for their
184  run() methods.
185 
186  Such tasks' run() methods tend to take two arguments,
187  one of which is the dataRef (as usual), and the other is the list
188  of visit-pairs, in the form of a list of tuples.
189  This list is supplied on the command line as documented,
190  and this class parses that, and passes the parsed version
191  to the run() method.
192 
193  See pipeBase.TaskRunner for more information.
194  """
195 
196  @staticmethod
197  def getTargetList(parsedCmd, **kwargs):
198  """Parse the visit list and pass through explicitly."""
199  visitPairs = []
200  for visitStringPair in parsedCmd.visitPairs:
201  visitStrings = visitStringPair.split(",")
202  if len(visitStrings) != 2:
203  raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
204  visitStringPair))
205  try:
206  visits = [int(visit) for visit in visitStrings]
207  except Exception:
208  raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
209  visitPairs.append(visits)
210 
211  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
212 
213 
214 def parseCmdlineNumberString(inputString):
215  """Parse command line numerical expression sytax and return as list of int
216 
217  Take an input of the form "'1..5:2^123..126'" as a string, and return
218  a list of ints as [1, 3, 5, 123, 124, 125, 126]
219  """
220  outList = []
221  for subString in inputString.split("^"):
222  mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
223  if mat:
224  v1 = int(mat.group(1))
225  v2 = int(mat.group(2))
226  v3 = mat.group(3)
227  v3 = int(v3) if v3 else 1
228  for v in range(v1, v2 + 1, v3):
229  outList.append(int(v))
230  else:
231  outList.append(int(subString))
232  return outList
233 
234 
235 class SingleVisitListTaskRunner(pipeBase.TaskRunner):
236  """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
237 
238  This transforms the processed arguments generated by the ArgumentParser
239  into the arguments expected by tasks which require a list of visits
240  to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
241 
242  Such tasks' run() methods tend to take two arguments,
243  one of which is the dataRef (as usual), and the other is the list
244  of visits.
245  This list is supplied on the command line as documented,
246  and this class parses that, and passes the parsed version
247  to the run() method.
248 
249  See `lsst.pipe.base.TaskRunner` for more information.
250  """
251 
252  @staticmethod
253  def getTargetList(parsedCmd, **kwargs):
254  """Parse the visit list and pass through explicitly."""
255  # if this has been pre-parsed and therefore doesn't have length of one
256  # then something has gone wrong, so execution should stop here.
257  assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
258  visits = parseCmdlineNumberString(parsedCmd.visitList[0])
259 
260  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
261 
262 
263 class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
264  """A DataIdContainer for the tasks for which the output does
265  not yet exist."""
266 
267  def makeDataRefList(self, namespace):
268  """Compute refList based on idList.
269 
270  This method must be defined as the dataset does not exist before this
271  task is run.
272 
273  Parameters
274  ----------
275  namespace
276  Results of parsing the command-line.
277 
278  Notes
279  -----
280  Not called if ``add_id_argument`` called
281  with ``doMakeDataRefList=False``.
282  Note that this is almost a copy-and-paste of the vanilla
283  implementation, but without checking if the datasets already exist,
284  as this task exists to make them.
285  """
286  if self.datasetType is None:
287  raise RuntimeError("Must call setDatasetType first")
288  butler = namespace.butler
289  for dataId in self.idList:
290  refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
291  # exclude nonexistent data
292  # this is a recursive test, e.g. for the sake of "raw" data
293  if not refList:
294  namespace.log.warn("No data found for dataId=%s", dataId)
295  continue
296  self.refList += refList
297 
298 
299 def irlsFit(initialParams, dataX, dataY, function, weightsY=None):
300  """Iteratively reweighted least squares fit.
301 
302  This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies
303  weights based on the Cauchy distribution to the fitter. See
304  e.g. Holland and Welsch, 1977, doi:10.1080/03610927708827533
305 
306  Parameters
307  ----------
308  initialParams : `list` [`float`]
309  Starting parameters.
310  dataX : `numpy.array` [`float`]
311  Abscissa data.
312  dataY : `numpy.array` [`float`]
313  Ordinate data.
314  function : callable
315  Function to fit.
316  weightsY : `numpy.array` [`float`]
317  Weights to apply to the data.
318 
319  Returns
320  -------
321  polyFit : `list` [`float`]
322  Final best fit parameters.
323  polyFitErr : `list` [`float`]
324  Final errors on fit parameters.
325  chiSq : `float`
326  Reduced chi squared.
327  weightsY : `list` [`float`]
328  Final weights used for each point.
329 
330  """
331  if not weightsY:
332  weightsY = np.ones_like(dataX)
333 
334  polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
335  for iteration in range(10):
336  # Use Cauchy weights
337  resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY)
338  weightsY = 1.0 / (1.0 + np.sqrt(resid / 2.385))
339  polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
340 
341  return polyFit, polyFitErr, chiSq, weightsY
342 
343 
344 def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
345  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
346 
347  optimize.leastsq returns the fractional covariance matrix. To estimate the
348  standard deviation of the fit parameters, multiply the entries of this matrix
349  by the unweighted reduced chi squared and take the square root of the diagonal elements.
350 
351  Parameters
352  ----------
353  initialParams : `list` of `float`
354  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
355  determines the degree of the polynomial.
356 
357  dataX : `numpy.array` of `float`
358  Data in the abscissa axis.
359 
360  dataY : `numpy.array` of `float`
361  Data in the ordinate axis.
362 
363  function : callable object (function)
364  Function to fit the data with.
365 
366  weightsY : `numpy.array` of `float`
367  Weights of the data in the ordinate axis.
368 
369  Return
370  ------
371  pFitSingleLeastSquares : `list` of `float`
372  List with fitted parameters.
373 
374  pErrSingleLeastSquares : `list` of `float`
375  List with errors for fitted parameters.
376 
377  reducedChiSqSingleLeastSquares : `float`
378  Reduced chi squared, unweighted if weightsY is not provided.
379  """
380  if weightsY is None:
381  weightsY = np.ones(len(dataX))
382 
383  def errFunc(p, x, y, weightsY=None):
384  if weightsY is None:
385  weightsY = np.ones(len(x))
386  return (function(p, x) - y)*weightsY
387 
388  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
389  args=(dataX, dataY, weightsY), full_output=1,
390  epsfcn=0.0001)
391 
392  if (len(dataY) > len(initialParams)) and pCov is not None:
393  reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY),
394  len(initialParams))
395  pCov *= reducedChiSq
396  else:
397  pCov = np.zeros((len(initialParams), len(initialParams)))
398  pCov[:, :] = np.nan
399  reducedChiSq = np.nan
400 
401  errorVec = []
402  for i in range(len(pFit)):
403  errorVec.append(np.fabs(pCov[i][i])**0.5)
404 
405  pFitSingleLeastSquares = pFit
406  pErrSingleLeastSquares = np.array(errorVec)
407 
408  return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
409 
410 
411 def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
412  """Do a fit using least squares and bootstrap to estimate parameter errors.
413 
414  The bootstrap error bars are calculated by fitting 100 random data sets.
415 
416  Parameters
417  ----------
418  initialParams : `list` of `float`
419  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
420  determines the degree of the polynomial.
421 
422  dataX : `numpy.array` of `float`
423  Data in the abscissa axis.
424 
425  dataY : `numpy.array` of `float`
426  Data in the ordinate axis.
427 
428  function : callable object (function)
429  Function to fit the data with.
430 
431  weightsY : `numpy.array` of `float`, optional.
432  Weights of the data in the ordinate axis.
433 
434  confidenceSigma : `float`, optional.
435  Number of sigmas that determine confidence interval for the bootstrap errors.
436 
437  Return
438  ------
439  pFitBootstrap : `list` of `float`
440  List with fitted parameters.
441 
442  pErrBootstrap : `list` of `float`
443  List with errors for fitted parameters.
444 
445  reducedChiSqBootstrap : `float`
446  Reduced chi squared, unweighted if weightsY is not provided.
447  """
448  if weightsY is None:
449  weightsY = np.ones(len(dataX))
450 
451  def errFunc(p, x, y, weightsY):
452  if weightsY is None:
453  weightsY = np.ones(len(x))
454  return (function(p, x) - y)*weightsY
455 
456  # Fit first time
457  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
458 
459  # Get the stdev of the residuals
460  residuals = errFunc(pFit, dataX, dataY, weightsY)
461  # 100 random data sets are generated and fitted
462  pars = []
463  for i in range(100):
464  randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
465  randomDataY = dataY + randomDelta
466  randomFit, _ = leastsq(errFunc, initialParams,
467  args=(dataX, randomDataY, weightsY), full_output=0)
468  pars.append(randomFit)
469  pars = np.array(pars)
470  meanPfit = np.mean(pars, 0)
471 
472  # confidence interval for parameter estimates
473  errPfit = confidenceSigma*np.std(pars, 0)
474  pFitBootstrap = meanPfit
475  pErrBootstrap = errPfit
476 
477  reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY),
478  len(initialParams))
479  return pFitBootstrap, pErrBootstrap, reducedChiSq
480 
481 
482 def funcPolynomial(pars, x):
483  """Polynomial function definition
484  Parameters
485  ----------
486  params : `list`
487  Polynomial coefficients. Its length determines the polynomial order.
488 
489  x : `numpy.array`
490  Abscisa array.
491 
492  Returns
493  -------
494  Ordinate array after evaluating polynomial of order len(pars)-1 at `x`.
495  """
496  return poly.polyval(x, [*pars])
497 
498 
499 def funcAstier(pars, x):
500  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
501 
502  Parameters
503  ----------
504  params : `list`
505  Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
506 
507  x : `numpy.array`
508  Signal mu (ADU).
509 
510  Returns
511  -------
512  C_00 (variance) in ADU^2.
513  """
514  a00, gain, noise = pars
515  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
516 
517 
518 def arrangeFlatsByExpTime(exposureList):
519  """Arrange exposures by exposure time.
520 
521  Parameters
522  ----------
523  exposureList : `list`[`lsst.afw.image.exposure.exposure.ExposureF`]
524  Input list of exposures.
525 
526  Returns
527  ------
528  flatsAtExpTime : `dict` [`float`,
529  `list`[`lsst.afw.image.exposure.exposure.ExposureF`]]
530  Dictionary that groups flat-field exposures that have the same
531  exposure time (seconds).
532  """
533  flatsAtExpTime = {}
534  for exp in exposureList:
535  tempFlat = exp
536  expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
537  listAtExpTime = flatsAtExpTime.setdefault(expTime, [])
538  listAtExpTime.append(tempFlat)
539 
540  return flatsAtExpTime
541 
542 
543 def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
544  """Check the exposure lengths of two exposures are equal.
545 
546  Parameters:
547  -----------
548  exp1 : `lsst.afw.image.exposure.ExposureF`
549  First exposure to check
550  exp2 : `lsst.afw.image.exposure.ExposureF`
551  Second exposure to check
552  v1 : `int` or `str`, optional
553  First visit of the visit pair
554  v2 : `int` or `str`, optional
555  Second visit of the visit pair
556  raiseWithMessage : `bool`
557  If True, instead of returning a bool, raise a RuntimeError if exposure
558  times are not equal, with a message about which visits mismatch if the
559  information is available.
560 
561  Raises:
562  -------
563  RuntimeError
564  Raised if the exposure lengths of the two exposures are not equal
565  """
566  expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
567  expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
568  if expTime1 != expTime2:
569  if raiseWithMessage:
570  msg = "Exposure lengths for visit pairs must be equal. " + \
571  "Found %s and %s" % (expTime1, expTime2)
572  if v1 and v2:
573  msg += " for visit pair %s, %s" % (v1, v2)
574  raise RuntimeError(msg)
575  else:
576  return False
577  return True
578 
579 
580 def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
581  checkTrim=True, logName=None):
582  """Check that appropriate ISR settings have been selected for the task.
583 
584  Note that this checks that the task itself is configured correctly rather
585  than checking a config.
586 
587  Parameters
588  ----------
589  isrTask : `lsst.ip.isr.IsrTask`
590  The task whose config is to be validated
591 
592  mandatory : `iterable` of `str`
593  isr steps that must be set to True. Raises if False or missing
594 
595  forbidden : `iterable` of `str`
596  isr steps that must be set to False. Raises if True, warns if missing
597 
598  desirable : `iterable` of `str`
599  isr steps that should probably be set to True. Warns is False, info if
600  missing
601 
602  undesirable : `iterable` of `str`
603  isr steps that should probably be set to False. Warns is True, info if
604  missing
605 
606  checkTrim : `bool`
607  Check to ensure the isrTask's assembly subtask is trimming the images.
608  This is a separate config as it is very ugly to do this within the
609  normal configuration lists as it is an option of a sub task.
610 
611  Raises
612  ------
613  RuntimeError
614  Raised if ``mandatory`` config parameters are False,
615  or if ``forbidden`` parameters are True.
616 
617  TypeError
618  Raised if parameter ``isrTask`` is an invalid type.
619 
620  Notes
621  -----
622  Logs warnings using an isrValidation logger for desirable/undesirable
623  options that are of the wrong polarity or if keys are missing.
624  """
625  if not isinstance(isrTask, ipIsr.IsrTask):
626  raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
627 
628  configDict = isrTask.config.toDict()
629 
630  if logName and isinstance(logName, str):
631  log = lsst.log.getLogger(logName)
632  else:
633  log = lsst.log.getLogger("isrValidation")
634 
635  if mandatory:
636  for configParam in mandatory:
637  if configParam not in configDict:
638  raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
639  if configDict[configParam] is False:
640  raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
641 
642  if forbidden:
643  for configParam in forbidden:
644  if configParam not in configDict:
645  log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
646  " forbidden list should each have an associated Field in IsrConfig:"
647  " check that there is not a typo in this case.")
648  continue
649  if configDict[configParam] is True:
650  raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
651 
652  if desirable:
653  for configParam in desirable:
654  if configParam not in configDict:
655  log.info(f"Failed to find key {configParam} in the isr config. You probably want"
656  " to set the equivalent for your obs_package to True.")
657  continue
658  if configDict[configParam] is False:
659  log.warn(f"Found config.isr.{configParam} set to False for this task."
660  " The cp_pipe Config recommends setting this to True.")
661  if undesirable:
662  for configParam in undesirable:
663  if configParam not in configDict:
664  log.info(f"Failed to find key {configParam} in the isr config. You probably want"
665  " to set the equivalent for your obs_package to False.")
666  continue
667  if configDict[configParam] is True:
668  log.warn(f"Found config.isr.{configParam} set to True for this task."
669  " The cp_pipe Config recommends setting this to False.")
670 
671  if checkTrim: # subtask setting, seems non-trivial to combine with above lists
672  if not isrTask.assembleCcd.config.doTrim:
673  raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
674 
675 
676 def ddict2dict(d):
677  """Convert nested default dictionaries to regular dictionaries.
678 
679  This is needed to prevent yaml persistence issues.
680 
681  Parameters
682  ----------
683  d : `defaultdict`
684  A possibly nested set of `defaultdict`.
685 
686  Returns
687  -------
688  dict : `dict`
689  A possibly nested set of `dict`.
690  """
691  for k, v in d.items():
692  if isinstance(v, dict):
693  d[k] = ddict2dict(v)
694  return dict(d)
lsst::afw::image
lsst.cp.pipe.utils.validateIsrConfig
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:580
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer.makeDataRefList
def makeDataRefList(self, namespace)
Definition: utils.py:267
lsst.cp.pipe.utils.arrangeFlatsByExpTime
def arrangeFlatsByExpTime(exposureList)
Definition: utils.py:518
lsst.cp.pipe.utils.irlsFit
def irlsFit(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:299
lsst.cp.pipe.utils.calculateWeightedReducedChi2
def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel)
Definition: utils.py:41
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:482
lsst.cp.pipe.utils.PairedVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:197
lsst.cp.pipe.utils.makeMockFlats
def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[], expId1=0, expId2=1)
Definition: utils.py:73
lsst.cp.pipe.utils.countMaskedPixels
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:172
lsst::afw::image::VisitInfo
lsst.cp.pipe.utils.SingleVisitListTaskRunner.getTargetList
def getTargetList(parsedCmd, **kwargs)
Definition: utils.py:253
lsst.cp.pipe.utils.PairedVisitListTaskRunner
Definition: utils.py:179
lsst.cp.pipe.utils.funcAstier
def funcAstier(pars, x)
Definition: utils.py:499
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:344
lsst.cp.pipe.utils.SingleVisitListTaskRunner
Definition: utils.py:235
lsst.cp.pipe.utils.checkExpLengthEqual
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:543
lsst.cp.pipe.utils.parseCmdlineNumberString
def parseCmdlineNumberString(inputString)
Definition: utils.py:214
lsst::ip::isr
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:411
lsst::pipe::base
lsst::log
lsst.cp.pipe.utils.NonexistentDatasetTaskDataIdContainer
Definition: utils.py:263
lsst.cp.pipe.utils.ddict2dict
def ddict2dict(d)
Definition: utils.py:676