Coverage for python/lsst/cp/pipe/utils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict']
27import re
28import numpy as np
29from scipy.optimize import leastsq
30import numpy.polynomial.polynomial as poly
32import lsst.pipe.base as pipeBase
33import lsst.ip.isr as ipIsr
34from lsst.ip.isr import isrMock
35import lsst.log
36import lsst.afw.image
38import galsim
41def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel):
42 """Calculate weighted reduced chi2.
44 Parameters
45 ----------
47 measured : `list`
48 List with measured data.
50 model : `list`
51 List with modeled data.
53 weightsMeasured : `list`
54 List with weights for the measured data.
56 nData : `int`
57 Number of data points.
59 nParsModel : `int`
60 Number of parameters in the model.
62 Returns
63 -------
65 redWeightedChi2 : `float`
66 Reduced weighted chi2.
67 """
69 wRes = (measured - model)*weightsMeasured
70 return ((wRes*wRes).sum())/(nData-nParsModel)
73def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
74 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[],
75 expId1=0, expId2=1):
76 """Create a pair or mock flats with isrMock.
78 Parameters
79 ----------
80 expTime : `float`
81 Exposure time of the flats.
83 gain : `float`, optional
84 Gain, in e/ADU.
86 readNoiseElectrons : `float`, optional
87 Read noise rms, in electrons.
89 fluxElectrons : `float`, optional
90 Flux of flats, in electrons per second.
92 randomSeedFlat1 : `int`, optional
93 Random seed for the normal distrubutions for the mean signal and noise (flat1).
95 randomSeedFlat2 : `int`, optional
96 Random seed for the normal distrubutions for the mean signal and noise (flat2).
98 powerLawBfParams : `list`, optional
99 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
101 expId1 : `int`, optional
102 Exposure ID for first flat.
104 expId2 : `int`, optional
105 Exposure ID for second flat.
107 Returns
108 -------
110 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
111 First exposure of flat field pair.
113 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
114 Second exposure of flat field pair.
116 Notes
117 -----
118 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
119 information about their meaning, see the Galsim documentation
120 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
121 and Gruen+15 (1501.02802).
123 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
124 """
125 flatFlux = fluxElectrons # e/s
126 flatMean = flatFlux*expTime # e
127 readNoise = readNoiseElectrons # e
129 mockImageConfig = isrMock.IsrMock.ConfigClass()
131 mockImageConfig.flatDrop = 0.99999
132 mockImageConfig.isTrimmed = True
134 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
135 flatExp2 = flatExp1.clone()
136 (shapeY, shapeX) = flatExp1.getDimensions()
137 flatWidth = np.sqrt(flatMean)
139 rng1 = np.random.RandomState(randomSeedFlat1)
140 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
141 (shapeX, shapeY))
142 rng2 = np.random.RandomState(randomSeedFlat2)
143 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
144 (shapeX, shapeY))
145 # Simulate BF with power law model in galsim
146 if len(powerLawBfParams):
147 if not len(powerLawBfParams) == 8:
148 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. "
149 f"Expected 8; passed {len(powerLawBfParams)}.")
150 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
151 tempFlatData1 = galsim.Image(flatData1)
152 temp2FlatData1 = cd.applyForward(tempFlatData1)
154 tempFlatData2 = galsim.Image(flatData2)
155 temp2FlatData2 = cd.applyForward(tempFlatData2)
157 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU
158 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU
159 else:
160 flatExp1.image.array[:] = flatData1/gain # ADU
161 flatExp2.image.array[:] = flatData2/gain # ADU
163 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime)
164 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime)
166 flatExp1.getInfo().setVisitInfo(visitInfoExp1)
167 flatExp2.getInfo().setVisitInfo(visitInfoExp2)
169 return flatExp1, flatExp2
172def countMaskedPixels(maskedIm, maskPlane):
173 """Count the number of pixels in a given mask plane."""
174 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
175 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
176 return nPix
179class PairedVisitListTaskRunner(pipeBase.TaskRunner):
180 """Subclass of TaskRunner for handling intrinsically paired visits.
182 This transforms the processed arguments generated by the ArgumentParser
183 into the arguments expected by tasks which take visit pairs for their
184 run() methods.
186 Such tasks' run() methods tend to take two arguments,
187 one of which is the dataRef (as usual), and the other is the list
188 of visit-pairs, in the form of a list of tuples.
189 This list is supplied on the command line as documented,
190 and this class parses that, and passes the parsed version
191 to the run() method.
193 See pipeBase.TaskRunner for more information.
194 """
196 @staticmethod
197 def getTargetList(parsedCmd, **kwargs):
198 """Parse the visit list and pass through explicitly."""
199 visitPairs = []
200 for visitStringPair in parsedCmd.visitPairs:
201 visitStrings = visitStringPair.split(",")
202 if len(visitStrings) != 2:
203 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
204 visitStringPair))
205 try:
206 visits = [int(visit) for visit in visitStrings]
207 except Exception:
208 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
209 visitPairs.append(visits)
211 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
214def parseCmdlineNumberString(inputString):
215 """Parse command line numerical expression sytax and return as list of int
217 Take an input of the form "'1..5:2^123..126'" as a string, and return
218 a list of ints as [1, 3, 5, 123, 124, 125, 126]
219 """
220 outList = []
221 for subString in inputString.split("^"):
222 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
223 if mat:
224 v1 = int(mat.group(1))
225 v2 = int(mat.group(2))
226 v3 = mat.group(3)
227 v3 = int(v3) if v3 else 1
228 for v in range(v1, v2 + 1, v3):
229 outList.append(int(v))
230 else:
231 outList.append(int(subString))
232 return outList
235class SingleVisitListTaskRunner(pipeBase.TaskRunner):
236 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
238 This transforms the processed arguments generated by the ArgumentParser
239 into the arguments expected by tasks which require a list of visits
240 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
242 Such tasks' run() methods tend to take two arguments,
243 one of which is the dataRef (as usual), and the other is the list
244 of visits.
245 This list is supplied on the command line as documented,
246 and this class parses that, and passes the parsed version
247 to the run() method.
249 See `lsst.pipe.base.TaskRunner` for more information.
250 """
252 @staticmethod
253 def getTargetList(parsedCmd, **kwargs):
254 """Parse the visit list and pass through explicitly."""
255 # if this has been pre-parsed and therefore doesn't have length of one
256 # then something has gone wrong, so execution should stop here.
257 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
258 visits = parseCmdlineNumberString(parsedCmd.visitList[0])
260 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
263class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
264 """A DataIdContainer for the tasks for which the output does
265 not yet exist."""
267 def makeDataRefList(self, namespace):
268 """Compute refList based on idList.
270 This method must be defined as the dataset does not exist before this
271 task is run.
273 Parameters
274 ----------
275 namespace
276 Results of parsing the command-line.
278 Notes
279 -----
280 Not called if ``add_id_argument`` called
281 with ``doMakeDataRefList=False``.
282 Note that this is almost a copy-and-paste of the vanilla
283 implementation, but without checking if the datasets already exist,
284 as this task exists to make them.
285 """
286 if self.datasetType is None:
287 raise RuntimeError("Must call setDatasetType first")
288 butler = namespace.butler
289 for dataId in self.idList:
290 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
291 # exclude nonexistent data
292 # this is a recursive test, e.g. for the sake of "raw" data
293 if not refList:
294 namespace.log.warn("No data found for dataId=%s", dataId)
295 continue
296 self.refList += refList
299def irlsFit(initialParams, dataX, dataY, function, weightsY=None):
300 """Iteratively reweighted least squares fit.
302 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies
303 weights based on the Cauchy distribution to the fitter. See
304 e.g. Holland and Welsch, 1977, doi:10.1080/03610927708827533
306 Parameters
307 ----------
308 initialParams : `list` [`float`]
309 Starting parameters.
310 dataX : `numpy.array` [`float`]
311 Abscissa data.
312 dataY : `numpy.array` [`float`]
313 Ordinate data.
314 function : callable
315 Function to fit.
316 weightsY : `numpy.array` [`float`]
317 Weights to apply to the data.
319 Returns
320 -------
321 polyFit : `list` [`float`]
322 Final best fit parameters.
323 polyFitErr : `list` [`float`]
324 Final errors on fit parameters.
325 chiSq : `float`
326 Reduced chi squared.
327 weightsY : `list` [`float`]
328 Final weights used for each point.
330 """
331 if not weightsY:
332 weightsY = np.ones_like(dataX)
334 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
335 for iteration in range(10):
336 # Use Cauchy weights
337 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY)
338 weightsY = 1.0 / (1.0 + np.sqrt(resid / 2.385))
339 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY)
341 return polyFit, polyFitErr, chiSq, weightsY
344def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
345 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
347 optimize.leastsq returns the fractional covariance matrix. To estimate the
348 standard deviation of the fit parameters, multiply the entries of this matrix
349 by the unweighted reduced chi squared and take the square root of the diagonal elements.
351 Parameters
352 ----------
353 initialParams : `list` of `float`
354 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
355 determines the degree of the polynomial.
357 dataX : `numpy.array` of `float`
358 Data in the abscissa axis.
360 dataY : `numpy.array` of `float`
361 Data in the ordinate axis.
363 function : callable object (function)
364 Function to fit the data with.
366 weightsY : `numpy.array` of `float`
367 Weights of the data in the ordinate axis.
369 Return
370 ------
371 pFitSingleLeastSquares : `list` of `float`
372 List with fitted parameters.
374 pErrSingleLeastSquares : `list` of `float`
375 List with errors for fitted parameters.
377 reducedChiSqSingleLeastSquares : `float`
378 Reduced chi squared, unweighted if weightsY is not provided.
379 """
380 if weightsY is None:
381 weightsY = np.ones(len(dataX))
383 def errFunc(p, x, y, weightsY=None):
384 if weightsY is None:
385 weightsY = np.ones(len(x))
386 return (function(p, x) - y)*weightsY
388 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
389 args=(dataX, dataY, weightsY), full_output=1,
390 epsfcn=0.0001)
392 if (len(dataY) > len(initialParams)) and pCov is not None:
393 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY),
394 len(initialParams))
395 pCov *= reducedChiSq
396 else:
397 pCov = np.zeros((len(initialParams), len(initialParams)))
398 pCov[:, :] = np.nan
399 reducedChiSq = np.nan
401 errorVec = []
402 for i in range(len(pFit)):
403 errorVec.append(np.fabs(pCov[i][i])**0.5)
405 pFitSingleLeastSquares = pFit
406 pErrSingleLeastSquares = np.array(errorVec)
408 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
411def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
412 """Do a fit using least squares and bootstrap to estimate parameter errors.
414 The bootstrap error bars are calculated by fitting 100 random data sets.
416 Parameters
417 ----------
418 initialParams : `list` of `float`
419 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
420 determines the degree of the polynomial.
422 dataX : `numpy.array` of `float`
423 Data in the abscissa axis.
425 dataY : `numpy.array` of `float`
426 Data in the ordinate axis.
428 function : callable object (function)
429 Function to fit the data with.
431 weightsY : `numpy.array` of `float`, optional.
432 Weights of the data in the ordinate axis.
434 confidenceSigma : `float`, optional.
435 Number of sigmas that determine confidence interval for the bootstrap errors.
437 Return
438 ------
439 pFitBootstrap : `list` of `float`
440 List with fitted parameters.
442 pErrBootstrap : `list` of `float`
443 List with errors for fitted parameters.
445 reducedChiSqBootstrap : `float`
446 Reduced chi squared, unweighted if weightsY is not provided.
447 """
448 if weightsY is None:
449 weightsY = np.ones(len(dataX))
451 def errFunc(p, x, y, weightsY):
452 if weightsY is None:
453 weightsY = np.ones(len(x))
454 return (function(p, x) - y)*weightsY
456 # Fit first time
457 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
459 # Get the stdev of the residuals
460 residuals = errFunc(pFit, dataX, dataY, weightsY)
461 # 100 random data sets are generated and fitted
462 pars = []
463 for i in range(100):
464 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
465 randomDataY = dataY + randomDelta
466 randomFit, _ = leastsq(errFunc, initialParams,
467 args=(dataX, randomDataY, weightsY), full_output=0)
468 pars.append(randomFit)
469 pars = np.array(pars)
470 meanPfit = np.mean(pars, 0)
472 # confidence interval for parameter estimates
473 errPfit = confidenceSigma*np.std(pars, 0)
474 pFitBootstrap = meanPfit
475 pErrBootstrap = errPfit
477 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY),
478 len(initialParams))
479 return pFitBootstrap, pErrBootstrap, reducedChiSq
482def funcPolynomial(pars, x):
483 """Polynomial function definition
484 Parameters
485 ----------
486 params : `list`
487 Polynomial coefficients. Its length determines the polynomial order.
489 x : `numpy.array`
490 Abscisa array.
492 Returns
493 -------
494 Ordinate array after evaluating polynomial of order len(pars)-1 at `x`.
495 """
496 return poly.polyval(x, [*pars])
499def funcAstier(pars, x):
500 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
502 Parameters
503 ----------
504 params : `list`
505 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
507 x : `numpy.array`
508 Signal mu (ADU).
510 Returns
511 -------
512 C_00 (variance) in ADU^2.
513 """
514 a00, gain, noise = pars
515 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00
518def arrangeFlatsByExpTime(exposureList):
519 """Arrange exposures by exposure time.
521 Parameters
522 ----------
523 exposureList : `list`[`lsst.afw.image.exposure.exposure.ExposureF`]
524 Input list of exposures.
526 Returns
527 ------
528 flatsAtExpTime : `dict` [`float`,
529 `list`[`lsst.afw.image.exposure.exposure.ExposureF`]]
530 Dictionary that groups flat-field exposures that have the same
531 exposure time (seconds).
532 """
533 flatsAtExpTime = {}
534 for exp in exposureList:
535 tempFlat = exp
536 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
537 listAtExpTime = flatsAtExpTime.setdefault(expTime, [])
538 listAtExpTime.append(tempFlat)
540 return flatsAtExpTime
543def arrangeFlatsByExpId(exposureList):
544 """Arrange exposures by exposure ID.
546 There is no guarantee that this will properly group exposures, but
547 allows a sequence of flats that have different illumination
548 (despite having the same exposure time) to be processed.
550 Parameters
551 ----------
552 exposureList : `list`[`lsst.afw.image.exposure.exposure.ExposureF`]
553 Input list of exposures.
555 Returns
556 ------
557 flatsAtExpId : `dict` [`float`,
558 `list`[`lsst.afw.image.exposure.exposure.ExposureF`]]
559 Dictionary that groups flat-field exposures sequentially by
560 their exposure id.
562 Notes
563 -----
565 This algorithm sorts the input exposures by their exposure id, and
566 then assigns each pair of exposures (exp_j, exp_{j+1}) to pair k,
567 such that 2*k = j, where j is the python index of one of the
568 exposures (starting from zero). By checking for the IndexError
569 while appending, we can ensure that there will only ever be fully
570 populated pairs.
571 """
572 flatsAtExpId = {}
573 sortedExposures = sorted(exposureList, key=lambda exp: exp.getInfo().getVisitInfo().getExposureId())
575 for jPair, exp in enumerate(sortedExposures):
576 if (jPair + 1) % 2:
577 kPair = jPair // 2
578 listAtExpId = flatsAtExpId.setdefault(kPair, [])
579 try:
580 listAtExpId.append(exp)
581 listAtExpId.append(sortedExposures[jPair + 1])
582 except IndexError:
583 pass
585 return flatsAtExpId
588def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
589 """Check the exposure lengths of two exposures are equal.
591 Parameters:
592 -----------
593 exp1 : `lsst.afw.image.exposure.ExposureF`
594 First exposure to check
595 exp2 : `lsst.afw.image.exposure.ExposureF`
596 Second exposure to check
597 v1 : `int` or `str`, optional
598 First visit of the visit pair
599 v2 : `int` or `str`, optional
600 Second visit of the visit pair
601 raiseWithMessage : `bool`
602 If True, instead of returning a bool, raise a RuntimeError if exposure
603 times are not equal, with a message about which visits mismatch if the
604 information is available.
606 Raises:
607 -------
608 RuntimeError
609 Raised if the exposure lengths of the two exposures are not equal
610 """
611 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
612 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
613 if expTime1 != expTime2:
614 if raiseWithMessage:
615 msg = "Exposure lengths for visit pairs must be equal. " + \
616 "Found %s and %s" % (expTime1, expTime2)
617 if v1 and v2:
618 msg += " for visit pair %s, %s" % (v1, v2)
619 raise RuntimeError(msg)
620 else:
621 return False
622 return True
625def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
626 checkTrim=True, logName=None):
627 """Check that appropriate ISR settings have been selected for the task.
629 Note that this checks that the task itself is configured correctly rather
630 than checking a config.
632 Parameters
633 ----------
634 isrTask : `lsst.ip.isr.IsrTask`
635 The task whose config is to be validated
637 mandatory : `iterable` of `str`
638 isr steps that must be set to True. Raises if False or missing
640 forbidden : `iterable` of `str`
641 isr steps that must be set to False. Raises if True, warns if missing
643 desirable : `iterable` of `str`
644 isr steps that should probably be set to True. Warns is False, info if
645 missing
647 undesirable : `iterable` of `str`
648 isr steps that should probably be set to False. Warns is True, info if
649 missing
651 checkTrim : `bool`
652 Check to ensure the isrTask's assembly subtask is trimming the images.
653 This is a separate config as it is very ugly to do this within the
654 normal configuration lists as it is an option of a sub task.
656 Raises
657 ------
658 RuntimeError
659 Raised if ``mandatory`` config parameters are False,
660 or if ``forbidden`` parameters are True.
662 TypeError
663 Raised if parameter ``isrTask`` is an invalid type.
665 Notes
666 -----
667 Logs warnings using an isrValidation logger for desirable/undesirable
668 options that are of the wrong polarity or if keys are missing.
669 """
670 if not isinstance(isrTask, ipIsr.IsrTask):
671 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
673 configDict = isrTask.config.toDict()
675 if logName and isinstance(logName, str):
676 log = lsst.log.getLogger(logName)
677 else:
678 log = lsst.log.getLogger("isrValidation")
680 if mandatory:
681 for configParam in mandatory:
682 if configParam not in configDict:
683 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
684 if configDict[configParam] is False:
685 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
687 if forbidden:
688 for configParam in forbidden:
689 if configParam not in configDict:
690 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
691 " forbidden list should each have an associated Field in IsrConfig:"
692 " check that there is not a typo in this case.")
693 continue
694 if configDict[configParam] is True:
695 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
697 if desirable:
698 for configParam in desirable:
699 if configParam not in configDict:
700 log.info(f"Failed to find key {configParam} in the isr config. You probably want"
701 " to set the equivalent for your obs_package to True.")
702 continue
703 if configDict[configParam] is False:
704 log.warn(f"Found config.isr.{configParam} set to False for this task."
705 " The cp_pipe Config recommends setting this to True.")
706 if undesirable:
707 for configParam in undesirable:
708 if configParam not in configDict:
709 log.info(f"Failed to find key {configParam} in the isr config. You probably want"
710 " to set the equivalent for your obs_package to False.")
711 continue
712 if configDict[configParam] is True:
713 log.warn(f"Found config.isr.{configParam} set to True for this task."
714 " The cp_pipe Config recommends setting this to False.")
716 if checkTrim: # subtask setting, seems non-trivial to combine with above lists
717 if not isrTask.assembleCcd.config.doTrim:
718 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")
721def ddict2dict(d):
722 """Convert nested default dictionaries to regular dictionaries.
724 This is needed to prevent yaml persistence issues.
726 Parameters
727 ----------
728 d : `defaultdict`
729 A possibly nested set of `defaultdict`.
731 Returns
732 -------
733 dict : `dict`
734 A possibly nested set of `dict`.
735 """
736 for k, v in d.items():
737 if isinstance(v, dict):
738 d[k] = ddict2dict(v)
739 return dict(d)