Coverage for python/lsst/cp/pipe/utils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner',
24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString',
25 'countMaskedPixels', 'checkExpLengthEqual']
27import re
28import numpy as np
30import lsst.pipe.base as pipeBase
31import lsst.ip.isr as ipIsr
32import lsst.log
35def countMaskedPixels(maskedIm, maskPlane):
36 """Count the number of pixels in a given mask plane."""
37 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane)
38 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size
39 return nPix
42class PairedVisitListTaskRunner(pipeBase.TaskRunner):
43 """Subclass of TaskRunner for handling intrinsically paired visits.
45 This transforms the processed arguments generated by the ArgumentParser
46 into the arguments expected by tasks which take visit pairs for their
47 run() methods.
49 Such tasks' run() methods tend to take two arguments,
50 one of which is the dataRef (as usual), and the other is the list
51 of visit-pairs, in the form of a list of tuples.
52 This list is supplied on the command line as documented,
53 and this class parses that, and passes the parsed version
54 to the run() method.
56 See pipeBase.TaskRunner for more information.
57 """
59 @staticmethod
60 def getTargetList(parsedCmd, **kwargs):
61 """Parse the visit list and pass through explicitly."""
62 visitPairs = []
63 for visitStringPair in parsedCmd.visitPairs:
64 visitStrings = visitStringPair.split(",")
65 if len(visitStrings) != 2:
66 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
67 visitStringPair))
68 try:
69 visits = [int(visit) for visit in visitStrings]
70 except Exception:
71 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
72 visitPairs.append(visits)
74 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
77def parseCmdlineNumberString(inputString):
78 """Parse command line numerical expression sytax and return as list of int
80 Take an input of the form "'1..5:2^123..126'" as a string, and return
81 a list of ints as [1, 3, 5, 123, 124, 125, 126]
82 """
83 outList = []
84 for subString in inputString.split("^"):
85 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString)
86 if mat:
87 v1 = int(mat.group(1))
88 v2 = int(mat.group(2))
89 v3 = mat.group(3)
90 v3 = int(v3) if v3 else 1
91 for v in range(v1, v2 + 1, v3):
92 outList.append(int(v))
93 else:
94 outList.append(int(subString))
95 return outList
98class SingleVisitListTaskRunner(pipeBase.TaskRunner):
99 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef.
101 This transforms the processed arguments generated by the ArgumentParser
102 into the arguments expected by tasks which require a list of visits
103 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code.
105 Such tasks' run() methods tend to take two arguments,
106 one of which is the dataRef (as usual), and the other is the list
107 of visits.
108 This list is supplied on the command line as documented,
109 and this class parses that, and passes the parsed version
110 to the run() method.
112 See `lsst.pipe.base.TaskRunner` for more information.
113 """
115 @staticmethod
116 def getTargetList(parsedCmd, **kwargs):
117 """Parse the visit list and pass through explicitly."""
118 # if this has been pre-parsed and therefore doesn't have length of one
119 # then something has gone wrong, so execution should stop here.
120 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated'
121 visits = parseCmdlineNumberString(parsedCmd.visitList[0])
123 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs)
126class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer):
127 """A DataIdContainer for the tasks for which the output does
128 not yet exist."""
130 def makeDataRefList(self, namespace):
131 """Compute refList based on idList.
133 This method must be defined as the dataset does not exist before this
134 task is run.
136 Parameters
137 ----------
138 namespace
139 Results of parsing the command-line.
141 Notes
142 -----
143 Not called if ``add_id_argument`` called
144 with ``doMakeDataRefList=False``.
145 Note that this is almost a copy-and-paste of the vanilla
146 implementation, but without checking if the datasets already exist,
147 as this task exists to make them.
148 """
149 if self.datasetType is None:
150 raise RuntimeError("Must call setDatasetType first")
151 butler = namespace.butler
152 for dataId in self.idList:
153 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
154 # exclude nonexistent data
155 # this is a recursive test, e.g. for the sake of "raw" data
156 if not refList:
157 namespace.log.warn("No data found for dataId=%s", dataId)
158 continue
159 self.refList += refList
162def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False):
163 """Check the exposure lengths of two exposures are equal.
165 Parameters:
166 -----------
167 exp1 : `lsst.afw.image.exposure.ExposureF`
168 First exposure to check
169 exp2 : `lsst.afw.image.exposure.ExposureF`
170 Second exposure to check
171 v1 : `int` or `str`, optional
172 First visit of the visit pair
173 v2 : `int` or `str`, optional
174 Second visit of the visit pair
175 raiseWithMessage : `bool`
176 If True, instead of returning a bool, raise a RuntimeError if exposure
177 times are not equal, with a message about which visits mismatch if the
178 information is available.
180 Raises:
181 -------
182 RuntimeError
183 Raised if the exposure lengths of the two exposures are not equal
184 """
185 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
186 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
187 if expTime1 != expTime2:
188 if raiseWithMessage:
189 msg = "Exposure lengths for visit pairs must be equal. " + \
190 "Found %s and %s" % (expTime1, expTime2)
191 if v1 and v2:
192 msg += " for visit pair %s, %s" % (v1, v2)
193 raise RuntimeError(msg)
194 else:
195 return False
196 return True
199def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None,
200 checkTrim=True, logName=None):
201 """Check that appropriate ISR settings have been selected for the task.
203 Note that this checks that the task itself is configured correctly rather
204 than checking a config.
206 Parameters
207 ----------
208 isrTask : `lsst.ip.isr.IsrTask`
209 The task whose config is to be validated
211 mandatory : `iterable` of `str`
212 isr steps that must be set to True. Raises if False or missing
214 forbidden : `iterable` of `str`
215 isr steps that must be set to False. Raises if True, warns if missing
217 desirable : `iterable` of `str`
218 isr steps that should probably be set to True. Warns is False, info if
219 missing
221 undesirable : `iterable` of `str`
222 isr steps that should probably be set to False. Warns is True, info if
223 missing
225 checkTrim : `bool`
226 Check to ensure the isrTask's assembly subtask is trimming the images.
227 This is a separate config as it is very ugly to do this within the
228 normal configuration lists as it is an option of a sub task.
230 Raises
231 ------
232 RuntimeError
233 Raised if ``mandatory`` config parameters are False,
234 or if ``forbidden`` parameters are True.
236 TypeError
237 Raised if parameter ``isrTask`` is an invalid type.
239 Notes
240 -----
241 Logs warnings using an isrValidation logger for desirable/undesirable
242 options that are of the wrong polarity or if keys are missing.
243 """
244 if not isinstance(isrTask, ipIsr.IsrTask):
245 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}')
247 configDict = isrTask.config.toDict()
249 if logName and isinstance(logName, str):
250 log = lsst.log.getLogger(logName)
251 else:
252 log = lsst.log.getLogger("isrValidation")
254 if mandatory:
255 for configParam in mandatory:
256 if configParam not in configDict:
257 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.")
258 if configDict[configParam] is False:
259 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.")
261 if forbidden:
262 for configParam in forbidden:
263 if configParam not in configDict:
264 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the"
265 " forbidden list should each have an associated Field in IsrConfig:"
266 " check that there is not a typo in this case.")
267 continue
268 if configDict[configParam] is True:
269 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.")
271 if desirable:
272 for configParam in desirable:
273 if configParam not in configDict:
274 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
275 " to set the equivalent for your obs_package to True.")
276 continue
277 if configDict[configParam] is False:
278 log.warn(f"Found config.isr.{configParam} set to False for this task." +
279 " The cp_pipe Config recommends setting this to True.")
280 if undesirable:
281 for configParam in undesirable:
282 if configParam not in configDict:
283 log.info(f"Failed to find key {configParam} in the isr config. You probably want" +
284 " to set the equivalent for your obs_package to False.")
285 continue
286 if configDict[configParam] is True:
287 log.warn(f"Found config.isr.{configParam} set to True for this task." +
288 " The cp_pipe Config recommends setting this to False.")
290 if checkTrim: # subtask setting, seems non-trivial to combine with above lists
291 if not isrTask.assembleCcd.config.doTrim:
292 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")