23 """Build star observations for input to FGCM using sourceTable_visit.
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
37 import lsst.pex.config
as pexConfig
38 import lsst.pipe.base
as pipeBase
39 import lsst.afw.table
as afwTable
41 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities
import computeApproxPixelAreaFields
44 __all__ = [
'FgcmBuildStarsTableConfig',
'FgcmBuildStarsTableTask']
48 """Config for FgcmBuildStarsTableTask"""
50 referenceCCD = pexConfig.Field(
51 doc=
"Reference CCD for checking PSF and background",
71 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
73 sourceSelector.flags.bad = [
'PixelFlags_edge',
74 'PixelFlags_interpolatedCenter',
75 'PixelFlags_saturatedCenter',
76 'PixelFlags_crCenter',
78 'PixelFlags_interpolated',
79 'PixelFlags_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
90 sourceSelector.isolated.parentName =
'parentSourceId'
91 sourceSelector.isolated.nChildName =
'Deblend_nChild'
93 sourceSelector.unresolved.name =
'extendedness'
98 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
100 ConfigClass = FgcmBuildStarsTableConfig
101 RunnerClass = FgcmBuildStarsRunner
102 _DefaultName =
"fgcmBuildStarsTable"
105 def _makeArgumentParser(cls):
106 """Create an argument parser"""
108 parser.add_id_argument(
"--id",
"sourceTable_visit", help=
"Data ID, e.g. --id visit=6789")
113 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
115 camera = butler.get(
'camera')
118 for detector
in camera:
119 ccdIds.append(detector.getId())
123 ccdIds.insert(0, self.config.referenceCCD)
130 groupedDataRefs = collections.defaultdict(list)
131 for dataRef
in dataRefs:
132 visit = dataRef.dataId[self.config.visitDataRefName]
138 calexpRef = butler.dataRef(
'calexp', dataId={self.config.visitDataRefName: visit,
139 self.config.ccdDataRefName: ccdId})
145 groupedDataRefs[visit].append(calexpRef)
149 groupedDataRefs[visit].append(dataRef)
151 return groupedDataRefs
154 calibFluxApertureRadius=None,
155 visitCatDataRef=None,
158 startTime = time.time()
164 if (visitCatDataRef
is not None and starObsDataRef
is None or
165 visitCatDataRef
is None and starObsDataRef
is not None):
166 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
167 "no checkpoint files will be persisted.")
169 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
170 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
174 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
175 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
177 outputSchema = sourceMapper.getOutputSchema()
180 camera = dataRef.get(
'camera')
182 for ccdIndex, detector
in enumerate(camera):
183 ccdMapping[detector.getId()] = ccdIndex
187 if inStarObsCat
is not None:
188 fullCatalog = inStarObsCat
189 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
190 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
191 if not comp1
or not comp2:
192 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
194 fullCatalog = afwTable.BaseCatalog(outputSchema)
196 visitKey = outputSchema[
'visit'].asKey()
197 ccdKey = outputSchema[
'ccd'].asKey()
198 instMagKey = outputSchema[
'instMag'].asKey()
199 instMagErrKey = outputSchema[
'instMagErr'].asKey()
202 if self.config.doSubtractLocalBackground:
203 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
205 localBackground = 0.0
212 for counter, visit
in enumerate(visitCat):
214 if visit[
'sources_read']:
217 expTime = visit[
'exptime']
219 dataRef = groupedDataRefs[visit[
'visit']][-1]
220 srcTable = dataRef.get()
222 df = srcTable.toDataFrame(columns)
224 if self.config.doSubtractLocalBackground:
225 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
226 df[self.config.instFluxField] -= localBackground
228 goodSrc = self.sourceSelector.selectSources(df)
229 use, = np.where(goodSrc.selected)
231 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
232 tempCat.resize(use.size)
234 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
235 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
236 tempCat[
'x'][:] = df[
'x'].values[use]
237 tempCat[
'y'][:] = df[
'y'].values[use]
238 tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
239 tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
240 tempCat[
'psf_candidate'] = df[
'Calib_psf_candidate'].values[use]
243 for detector
in camera:
244 ccdId = detector.getId()
246 use2 = (tempCat[ccdKey] == ccdId)
247 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
249 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
250 visit[
'scaling'][ccdMapping[ccdId]])
251 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
255 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use] /
256 df[self.config.instFluxField].values[use])
259 if self.config.doApplyWcsJacobian:
260 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
262 fullCatalog.extend(tempCat)
265 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
266 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use] /
267 df[self.config.apertureInnerInstFluxField].values[use])
268 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
269 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use] /
270 df[self.config.apertureOuterInstFluxField].values[use])
271 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
272 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
274 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
275 visit[
'sources_read'] =
True
277 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
278 use.size, visit[
'visit'], visit[
'deltaAper'])
280 if ((counter % self.config.nVisitsPerCheckpoint) == 0
and
281 starObsDataRef
is not None and visitCatDataRef
is not None):
284 starObsDataRef.put(fullCatalog)
285 visitCatDataRef.put(visitCat)
287 self.log.info(
"Found all good star observations in %.2f s" %
288 (time.time() - startTime))
292 def _get_sourceTable_visit_columns(self):
294 Get the sourceTable_visit columns from the config.
299 List of columns to read from sourceTable_visit
301 columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
302 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
303 self.config.instFluxField, self.config.instFluxField +
'Err',
304 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
305 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
306 if self.sourceSelector.config.doFlags:
307 columns.extend(self.sourceSelector.config.flags.bad)
308 if self.sourceSelector.config.doUnresolved:
309 columns.append(self.sourceSelector.config.unresolved.name)
310 if self.sourceSelector.config.doIsolated:
311 columns.append(self.sourceSelector.config.isolated.parentName)
312 columns.append(self.sourceSelector.config.isolated.nChildName)
313 if self.config.doSubtractLocalBackground:
314 columns.append(self.config.localBackgroundFluxField)