23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs
as much cleaning of the
29input catalog
as possible.
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
53 camera = connectionTypes.PrerequisiteInput(
54 doc=
"Camera instrument",
56 storageClass=
"Camera",
57 dimensions=(
"instrument",),
61 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
62 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
63 "chromatic corrections."),
64 name=
"fgcmLookUpTable",
65 storageClass=
"Catalog",
66 dimensions=(
"instrument",),
70 sourceSchema = connectionTypes.InitInput(
71 doc=
"Schema for source catalogs",
73 storageClass=
"SourceCatalog",
76 refCat = connectionTypes.PrerequisiteInput(
77 doc=
"Reference catalog to use for photometric calibration",
79 storageClass=
"SimpleCatalog",
80 dimensions=(
"skypix",),
85 sourceTable_visit = connectionTypes.Input(
86 doc=
"Source table in parquet format, per visit",
87 name=
"sourceTable_visit",
88 storageClass=
"DataFrame",
89 dimensions=(
"instrument",
"visit"),
94 visitSummary = connectionTypes.Input(
95 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
96 "detector id for the id and must be sorted for fast lookups of a "
99 storageClass=
"ExposureCatalog",
100 dimensions=(
"instrument",
"visit"),
105 fgcmVisitCatalog = connectionTypes.Output(
106 doc=
"Catalog of visit information for fgcm",
107 name=
"fgcmVisitCatalog",
108 storageClass=
"Catalog",
109 dimensions=(
"instrument",),
112 fgcmStarObservations = connectionTypes.Output(
113 doc=
"Catalog of star observations for fgcm",
114 name=
"fgcmStarObservations",
115 storageClass=
"Catalog",
116 dimensions=(
"instrument",),
119 fgcmStarIds = connectionTypes.Output(
120 doc=
"Catalog of fgcm calibration star IDs",
122 storageClass=
"Catalog",
123 dimensions=(
"instrument",),
126 fgcmStarIndices = connectionTypes.Output(
127 doc=
"Catalog of fgcm calibration star indices",
128 name=
"fgcmStarIndices",
129 storageClass=
"Catalog",
130 dimensions=(
"instrument",),
133 fgcmReferenceStars = connectionTypes.Output(
134 doc=
"Catalog of fgcm-matched reference stars",
135 name=
"fgcmReferenceStars",
136 storageClass=
"Catalog",
137 dimensions=(
"instrument",),
143 if not config.doReferenceMatches:
144 self.prerequisiteInputs.remove(
"refCat")
145 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
147 if not config.doReferenceMatches:
148 self.outputs.remove(
"fgcmReferenceStars")
151 return (
"visitSummary",)
155 pipelineConnections=FgcmBuildStarsTableConnections):
156 """Config for FgcmBuildStarsTableTask"""
158 referenceCCD = pexConfig.Field(
159 doc=
"Reference CCD for checking PSF and background",
181 sourceSelector.flags.bad = [
'pixelFlags_edge',
182 'pixelFlags_interpolatedCenter',
183 'pixelFlags_saturatedCenter',
184 'pixelFlags_crCenter',
186 'pixelFlags_interpolated',
187 'pixelFlags_saturated',
193 sourceSelector.flags.bad.append(localBackgroundFlagName)
198 sourceSelector.isolated.parentName =
'parentSourceId'
199 sourceSelector.isolated.nChildName =
'deblend_nChild'
201 sourceSelector.requireFiniteRaDec.raColName =
'ra'
202 sourceSelector.requireFiniteRaDec.decColName =
'dec'
204 sourceSelector.unresolved.name =
'extendedness'
206 sourceSelector.doRequirePrimary =
True
211 Build stars for the FGCM
global calibration, using sourceTable_visit catalogs.
213 ConfigClass = FgcmBuildStarsTableConfig
214 _DefaultName = "fgcmBuildStarsTable"
216 canMultiprocess =
False
219 super().
__init__(initInputs=initInputs, **kwargs)
220 if initInputs
is not None:
224 inputRefDict = butlerQC.get(inputRefs)
226 sourceTableHandles = inputRefDict[
'sourceTable_visit']
228 self.log.info(
"Running with %d sourceTable_visit handles",
229 len(sourceTableHandles))
231 sourceTableHandleDict = {sourceTableHandle.dataId[
'visit']: sourceTableHandle
for
232 sourceTableHandle
in sourceTableHandles}
234 if self.config.doReferenceMatches:
236 lutHandle = inputRefDict[
'fgcmLookUpTable']
239 refConfig = LoadReferenceObjectsConfig()
240 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
241 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
242 for ref
in inputRefs.refCat],
243 refCats=butlerQC.get(inputRefs.refCat),
244 name=self.config.connections.refCat,
247 self.makeSubtask(
'fgcmLoadReferenceCatalog',
248 refObjLoader=refObjLoader,
249 refCatName=self.config.connections.refCat)
255 calibFluxApertureRadius =
None
256 if self.config.doSubtractLocalBackground:
258 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
259 except RuntimeError
as e:
260 raise RuntimeError(
"Could not determine aperture radius from %s. "
261 "Cannot use doSubtractLocalBackground." %
262 (self.config.instFluxField))
from e
264 visitSummaryHandles = inputRefDict[
'visitSummary']
265 visitSummaryHandleDict = {visitSummaryHandle.dataId[
'visit']: visitSummaryHandle
for
266 visitSummaryHandle
in visitSummaryHandles}
268 camera = inputRefDict[
'camera']
270 visitSummaryHandleDict)
274 rad = calibFluxApertureRadius
279 calibFluxApertureRadius=rad)
281 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
282 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
284 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
285 fgcmStarObservationCat,
288 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
289 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
290 if fgcmRefCat
is not None:
291 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
294 """Group sourceTable and visitSummary handles.
298 sourceTableHandleDict : `dict` [`int`, `str`]
299 Dict of source tables, keyed by visit.
300 visitSummaryHandleDict : `dict` [int, `str`]
301 Dict of visit summary catalogs, keyed by visit.
305 groupedHandles : `dict` [`int`, `list`]
306 Dictionary with sorted visit keys,
and `list`s
with
307 `lsst.daf.butler.DeferredDataSetHandle`. The first
308 item
in the list will be the visitSummary ref,
and
309 the second will be the source table ref.
311 groupedHandles = collections.defaultdict(list)
312 visits = sorted(sourceTableHandleDict.keys())
315 groupedHandles[visit] = [visitSummaryHandleDict[visit],
316 sourceTableHandleDict[visit]]
318 return groupedHandles
323 calibFluxApertureRadius=None):
324 startTime = time.time()
326 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
327 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
332 outputSchema = sourceMapper.getOutputSchema()
336 for ccdIndex, detector
in enumerate(camera):
337 ccdMapping[detector.getId()] = ccdIndex
339 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
341 fullCatalog = afwTable.BaseCatalog(outputSchema)
343 visitKey = outputSchema[
'visit'].asKey()
344 ccdKey = outputSchema[
'ccd'].asKey()
345 instMagKey = outputSchema[
'instMag'].asKey()
346 instMagErrKey = outputSchema[
'instMagErr'].asKey()
347 deltaMagAperKey = outputSchema[
'deltaMagAper'].asKey()
350 if self.config.doSubtractLocalBackground:
351 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
357 for counter, visit
in enumerate(visitCat):
358 expTime = visit[
'exptime']
360 handle = groupedHandles[visit[
'visit']][-1]
363 inColumns = handle.get(component=
'columns')
365 df = handle.get(parameters={
'columns': columns})
367 goodSrc = self.sourceSelector.selectSources(df)
371 if self.config.doSubtractLocalBackground:
372 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
373 use, = np.where((goodSrc.selected)
374 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
376 use, = np.where(goodSrc.selected)
378 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
379 tempCat.resize(use.size)
381 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
382 tempCat[
'dec'][:] = np.deg2rad(df[
'dec'].values[use])
383 tempCat[
'x'][:] = df[
'x'].values[use]
384 tempCat[
'y'][:] = df[
'y'].values[use]
386 tempCat[visitKey][:] = df[
'visit'].values[use]
387 tempCat[ccdKey][:] = df[
'detector'].values[use]
388 tempCat[
'psf_candidate'] = df[self.config.psfCandidateName].values[use]
390 with warnings.catch_warnings():
392 warnings.simplefilter(
"ignore")
394 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
395 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
396 / df[self.config.apertureInnerInstFluxField].values[use])
397 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
398 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
399 / df[self.config.apertureOuterInstFluxField].values[use])
400 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
402 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
404 if self.config.doSubtractLocalBackground:
418 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
419 - localBackground[use]) -
420 -2.5*np.log10(df[self.config.instFluxField].values[use]))
422 tempCat[
'deltaMagBkg'][:] = 0.0
425 for detector
in camera:
426 ccdId = detector.getId()
428 use2 = (tempCat[ccdKey] == ccdId)
429 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
431 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
432 * visit[
'scaling'][ccdMapping[ccdId]])
433 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
437 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
438 / df[self.config.instFluxField].values[use])
441 if self.config.doApplyWcsJacobian:
442 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
444 fullCatalog.extend(tempCat)
446 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
447 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
449 visit[
'deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
450 visit[
'sources_read'] =
True
452 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
453 use.size, visit[
'visit'], visit[
'deltaAper'])
455 self.log.info(
"Found all good star observations in %.2f s" %
456 (time.time() - startTime))
462 Get the sourceTable_visit columns from the config.
467 List of columns available
in the sourceTable_visit
472 List of columns to read
from sourceTable_visit.
475 columns = [
'visit',
'detector',
476 'ra',
'dec',
'x',
'y', self.config.psfCandidateName,
477 self.config.instFluxField, self.config.instFluxField +
'Err',
478 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
479 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
480 if self.sourceSelector.config.doFlags:
481 columns.extend(self.sourceSelector.config.flags.bad)
482 if self.sourceSelector.config.doUnresolved:
483 columns.append(self.sourceSelector.config.unresolved.name)
484 if self.sourceSelector.config.doIsolated:
485 columns.append(self.sourceSelector.config.isolated.parentName)
486 columns.append(self.sourceSelector.config.isolated.nChildName)
487 if self.sourceSelector.config.doRequirePrimary:
488 columns.append(self.sourceSelector.config.requirePrimary.primaryColName)
489 if self.config.doSubtractLocalBackground:
490 columns.append(self.config.localBackgroundFluxField)
_makeSourceMapper(self, sourceSchema)
fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
fgcmMakeVisitCatalog(self, camera, groupedHandles)
fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
apertureOuterInstFluxField
apertureInnerInstFluxField
doSubtractLocalBackground
apertureInnerInstFluxField
apertureOuterInstFluxField
getSpatialBoundsConnections(self)
__init__(self, *config=None)
runQuantum(self, butlerQC, inputRefs, outputRefs)
__init__(self, initInputs=None, **kwargs)
fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
_get_sourceTable_visit_columns(self, inColumns)
_groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)