23 __all__ = [
'PairedVisitListTaskRunner',
'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer',
'parseCmdlineNumberString',
25 'countMaskedPixels',
'checkExpLengthEqual']
29 from scipy.optimize
import leastsq
30 import numpy.polynomial.polynomial
as poly
41 """Calculate weighted reduced chi2.
47 List with measured data.
50 List with modeled data.
52 weightsMeasured : `list`
53 List with weights for the measured data.
56 Number of data points.
59 Number of parameters in the model.
64 redWeightedChi2 : `float`
65 Reduced weighted chi2.
68 wRes = (measured - model)*weightsMeasured
69 return ((wRes*wRes).sum())/(nData-nParsModel)
72 def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000,
73 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]):
74 """Create a pair or mock flats with isrMock.
79 Exposure time of the flats.
81 gain : `float`, optional
84 readNoiseElectrons : `float`, optional
85 Read noise rms, in electrons.
87 fluxElectrons : `float`, optional
88 Flux of flats, in electrons per second.
90 randomSeedFlat1 : `int`, optional
91 Random seed for the normal distrubutions for the mean signal and noise (flat1).
93 randomSeedFlat2 : `int`, optional
94 Random seed for the normal distrubutions for the mean signal and noise (flat2).
96 powerLawBfParams : `list`, optional
97 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect.
102 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF`
103 First exposure of flat field pair.
105 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF`
106 Second exposure of flat field pair.
110 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more
111 information about their meaning, see the Galsim documentation
112 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html
113 and Gruen+15 (1501.02802).
115 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0)
117 flatFlux = fluxElectrons
118 flatMean = flatFlux*expTime
119 readNoise = readNoiseElectrons
121 mockImageConfig = isrMock.IsrMock.ConfigClass()
123 mockImageConfig.flatDrop = 0.99999
124 mockImageConfig.isTrimmed =
True
126 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
127 flatExp2 = flatExp1.clone()
128 (shapeY, shapeX) = flatExp1.getDimensions()
129 flatWidth = np.sqrt(flatMean)
131 rng1 = np.random.RandomState(randomSeedFlat1)
132 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise,
134 rng2 = np.random.RandomState(randomSeedFlat2)
135 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise,
138 if len(powerLawBfParams):
139 if not len(powerLawBfParams) == 8:
140 raise RuntimeError(
"Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " +
141 f
"Expected 8; passed {len(powerLawBfParams)}.")
142 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams)
143 tempFlatData1 = galsim.Image(flatData1)
144 temp2FlatData1 = cd.applyForward(tempFlatData1)
146 tempFlatData2 = galsim.Image(flatData2)
147 temp2FlatData2 = cd.applyForward(tempFlatData2)
149 flatExp1.image.array[:] = temp2FlatData1.array/gain
150 flatExp2.image.array[:] = temp2FlatData2.array/gain
152 flatExp1.image.array[:] = flatData1/gain
153 flatExp2.image.array[:] = flatData2/gain
155 return flatExp1, flatExp2
159 """Count the number of pixels in a given mask plane."""
160 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
161 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
166 """Subclass of TaskRunner for handling intrinsically paired visits.
168 This transforms the processed arguments generated by the ArgumentParser
169 into the arguments expected by tasks which take visit pairs for their
172 Such tasks' run() methods tend to take two arguments,
173 one of which is the dataRef (as usual), and the other is the list
174 of visit-pairs, in the form of a list of tuples.
175 This list is supplied on the command line as documented,
176 and this class parses that, and passes the parsed version
179 See pipeBase.TaskRunner for more information.
184 """Parse the visit list and pass through explicitly."""
186 for visitStringPair
in parsedCmd.visitPairs:
187 visitStrings = visitStringPair.split(
",")
188 if len(visitStrings) != 2:
189 raise RuntimeError(
"Found {} visits in {} instead of 2".format(len(visitStrings),
192 visits = [int(visit)
for visit
in visitStrings]
194 raise RuntimeError(
"Could not parse {} as two integer visit numbers".format(visitStringPair))
195 visitPairs.append(visits)
197 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
201 """Parse command line numerical expression sytax and return as list of int
203 Take an input of the form "'1..5:2^123..126'" as a string, and return
204 a list of ints as [1, 3, 5, 123, 124, 125, 126]
207 for subString
in inputString.split(
"^"):
208 mat = re.search(
r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
210 v1 = int(mat.group(1))
211 v2 = int(mat.group(2))
213 v3 = int(v3)
if v3
else 1
214 for v
in range(v1, v2 + 1, v3):
215 outList.append(int(v))
217 outList.append(int(subString))
222 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
224 This transforms the processed arguments generated by the ArgumentParser
225 into the arguments expected by tasks which require a list of visits
226 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
228 Such tasks' run() methods tend to take two arguments,
229 one of which is the dataRef (as usual), and the other is the list
231 This list is supplied on the command line as documented,
232 and this class parses that, and passes the parsed version
235 See `lsst.pipe.base.TaskRunner` for more information.
240 """Parse the visit list and pass through explicitly."""
243 assert len(parsedCmd.visitList) == 1,
'visitList parsing assumptions violated'
246 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
250 """A DataIdContainer for the tasks for which the output does
254 """Compute refList based on idList.
256 This method must be defined as the dataset does not exist before this
262 Results of parsing the command-line.
266 Not called if ``add_id_argument`` called
267 with ``doMakeDataRefList=False``.
268 Note that this is almost a copy-and-paste of the vanilla
269 implementation, but without checking if the datasets already exist,
270 as this task exists to make them.
272 if self.datasetType
is None:
273 raise RuntimeError(
"Must call setDatasetType first")
274 butler = namespace.butler
275 for dataId
in self.idList:
276 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
280 namespace.log.warn(
"No data found for dataId=%s", dataId)
282 self.refList += refList
285 def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None):
286 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
288 optimize.leastsq returns the fractional covariance matrix. To estimate the
289 standard deviation of the fit parameters, multiply the entries of this matrix
290 by the unweighted reduced chi squared and take the square root of the diagonal elements.
294 initialParams : `list` of `float`
295 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
296 determines the degree of the polynomial.
298 dataX : `numpy.array` of `float`
299 Data in the abscissa axis.
301 dataY : `numpy.array` of `float`
302 Data in the ordinate axis.
304 function : callable object (function)
305 Function to fit the data with.
307 weightsY : `numpy.array` of `float`
308 Weights of the data in the ordinate axis.
312 pFitSingleLeastSquares : `list` of `float`
313 List with fitted parameters.
315 pErrSingleLeastSquares : `list` of `float`
316 List with errors for fitted parameters.
318 reducedChiSqSingleLeastSquares : `float`
319 Reduced chi squared, unweighted if weightsY is not provided.
322 weightsY = np.ones(len(dataX))
324 def errFunc(p, x, y, weightsY=None):
326 weightsY = np.ones(len(x))
327 return (function(p, x) - y)*weightsY
329 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
330 args=(dataX, dataY, weightsY), full_output=1,
333 if (len(dataY) > len(initialParams))
and pCov
is not None:
338 pCov = np.zeros((len(initialParams), len(initialParams)))
340 reducedChiSq = np.inf
343 for i
in range(len(pFit)):
344 errorVec.append(np.fabs(pCov[i][i])**0.5)
346 pFitSingleLeastSquares = pFit
347 pErrSingleLeastSquares = np.array(errorVec)
349 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
352 def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.):
353 """Do a fit using least squares and bootstrap to estimate parameter errors.
355 The bootstrap error bars are calculated by fitting 100 random data sets.
359 initialParams : `list` of `float`
360 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
361 determines the degree of the polynomial.
363 dataX : `numpy.array` of `float`
364 Data in the abscissa axis.
366 dataY : `numpy.array` of `float`
367 Data in the ordinate axis.
369 function : callable object (function)
370 Function to fit the data with.
372 weightsY : `numpy.array` of `float`, optional.
373 Weights of the data in the ordinate axis.
375 confidenceSigma : `float`, optional.
376 Number of sigmas that determine confidence interval for the bootstrap errors.
380 pFitBootstrap : `list` of `float`
381 List with fitted parameters.
383 pErrBootstrap : `list` of `float`
384 List with errors for fitted parameters.
386 reducedChiSqBootstrap : `float`
387 Reduced chi squared, unweighted if weightsY is not provided.
390 weightsY = np.ones(len(dataX))
392 def errFunc(p, x, y, weightsY):
394 weightsY = np.ones(len(x))
395 return (function(p, x) - y)*weightsY
398 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0)
401 residuals = errFunc(pFit, dataX, dataY, weightsY)
405 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY))
406 randomDataY = dataY + randomDelta
407 randomFit, _ = leastsq(errFunc, initialParams,
408 args=(dataX, randomDataY, weightsY), full_output=0)
409 pars.append(randomFit)
410 pars = np.array(pars)
411 meanPfit = np.mean(pars, 0)
414 errPfit = confidenceSigma*np.std(pars, 0)
415 pFitBootstrap = meanPfit
416 pErrBootstrap = errPfit
420 return pFitBootstrap, pErrBootstrap, reducedChiSq
424 """Polynomial function definition
428 Polynomial coefficients. Its length determines the polynomial order.
435 C_00 (variance) in ADU^2.
437 return poly.polyval(x, [*pars])
441 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19.
446 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2).
453 C_00 (variance) in ADU^2.
455 a00, gain, noise = pars
456 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
460 """Check the exposure lengths of two exposures are equal.
464 exp1 : `lsst.afw.image.exposure.ExposureF`
465 First exposure to check
466 exp2 : `lsst.afw.image.exposure.ExposureF`
467 Second exposure to check
468 v1 : `int` or `str`, optional
469 First visit of the visit pair
470 v2 : `int` or `str`, optional
471 Second visit of the visit pair
472 raiseWithMessage : `bool`
473 If True, instead of returning a bool, raise a RuntimeError if exposure
474 times are not equal, with a message about which visits mismatch if the
475 information is available.
480 Raised if the exposure lengths of the two exposures are not equal
482 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
483 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
484 if expTime1 != expTime2:
486 msg =
"Exposure lengths for visit pairs must be equal. " + \
487 "Found %s and %s" % (expTime1, expTime2)
489 msg +=
" for visit pair %s, %s" % (v1, v2)
490 raise RuntimeError(msg)
496 def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
497 checkTrim=True, logName=None):
498 """Check that appropriate ISR settings have been selected for the task.
500 Note that this checks that the task itself is configured correctly rather
501 than checking a config.
505 isrTask : `lsst.ip.isr.IsrTask`
506 The task whose config is to be validated
508 mandatory : `iterable` of `str`
509 isr steps that must be set to True. Raises if False or missing
511 forbidden : `iterable` of `str`
512 isr steps that must be set to False. Raises if True, warns if missing
514 desirable : `iterable` of `str`
515 isr steps that should probably be set to True. Warns is False, info if
518 undesirable : `iterable` of `str`
519 isr steps that should probably be set to False. Warns is True, info if
523 Check to ensure the isrTask's assembly subtask is trimming the images.
524 This is a separate config as it is very ugly to do this within the
525 normal configuration lists as it is an option of a sub task.
530 Raised if ``mandatory`` config parameters are False,
531 or if ``forbidden`` parameters are True.
534 Raised if parameter ``isrTask`` is an invalid type.
538 Logs warnings using an isrValidation logger for desirable/undesirable
539 options that are of the wrong polarity or if keys are missing.
541 if not isinstance(isrTask, ipIsr.IsrTask):
542 raise TypeError(f
'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
544 configDict = isrTask.config.toDict()
546 if logName
and isinstance(logName, str):
547 log = lsst.log.getLogger(logName)
549 log = lsst.log.getLogger(
"isrValidation")
552 for configParam
in mandatory:
553 if configParam
not in configDict:
554 raise RuntimeError(f
"Mandatory parameter {configParam} not found in the isr configuration.")
555 if configDict[configParam]
is False:
556 raise RuntimeError(f
"Must set config.isr.{configParam} to True for this task.")
559 for configParam
in forbidden:
560 if configParam
not in configDict:
561 log.warn(f
"Failed to find forbidden key {configParam} in the isr config. The keys in the"
562 " forbidden list should each have an associated Field in IsrConfig:"
563 " check that there is not a typo in this case.")
565 if configDict[configParam]
is True:
566 raise RuntimeError(f
"Must set config.isr.{configParam} to False for this task.")
569 for configParam
in desirable:
570 if configParam
not in configDict:
571 log.info(f
"Failed to find key {configParam} in the isr config. You probably want" +
572 " to set the equivalent for your obs_package to True.")
574 if configDict[configParam]
is False:
575 log.warn(f
"Found config.isr.{configParam} set to False for this task." +
576 " The cp_pipe Config recommends setting this to True.")
578 for configParam
in undesirable:
579 if configParam
not in configDict:
580 log.info(f
"Failed to find key {configParam} in the isr config. You probably want" +
581 " to set the equivalent for your obs_package to False.")
583 if configDict[configParam]
is True:
584 log.warn(f
"Found config.isr.{configParam} set to True for this task." +
585 " The cp_pipe Config recommends setting this to False.")
588 if not isrTask.assembleCcd.config.doTrim:
589 raise RuntimeError(
"Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")