23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs
as much cleaning of the
29input catalog
as possible.
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
45from .utilities import lookupStaticCalibrations
47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument",),
53 camera = connectionTypes.PrerequisiteInput(
54 doc=
"Camera instrument",
56 storageClass=
"Camera",
57 dimensions=(
"instrument",),
58 lookupFunction=lookupStaticCalibrations,
62 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
63 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
64 "chromatic corrections."),
65 name=
"fgcmLookUpTable",
66 storageClass=
"Catalog",
67 dimensions=(
"instrument",),
71 sourceSchema = connectionTypes.InitInput(
72 doc=
"Schema for source catalogs",
74 storageClass=
"SourceCatalog",
77 refCat = connectionTypes.PrerequisiteInput(
78 doc=
"Reference catalog to use for photometric calibration",
80 storageClass=
"SimpleCatalog",
81 dimensions=(
"skypix",),
86 sourceTable_visit = connectionTypes.Input(
87 doc=
"Source table in parquet format, per visit",
88 name=
"sourceTable_visit",
89 storageClass=
"DataFrame",
90 dimensions=(
"instrument",
"visit"),
95 visitSummary = connectionTypes.Input(
96 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
97 "detector id for the id and must be sorted for fast lookups of a "
100 storageClass=
"ExposureCatalog",
101 dimensions=(
"instrument",
"visit"),
106 fgcmVisitCatalog = connectionTypes.Output(
107 doc=
"Catalog of visit information for fgcm",
108 name=
"fgcmVisitCatalog",
109 storageClass=
"Catalog",
110 dimensions=(
"instrument",),
113 fgcmStarObservations = connectionTypes.Output(
114 doc=
"Catalog of star observations for fgcm",
115 name=
"fgcmStarObservations",
116 storageClass=
"Catalog",
117 dimensions=(
"instrument",),
120 fgcmStarIds = connectionTypes.Output(
121 doc=
"Catalog of fgcm calibration star IDs",
123 storageClass=
"Catalog",
124 dimensions=(
"instrument",),
127 fgcmStarIndices = connectionTypes.Output(
128 doc=
"Catalog of fgcm calibration star indices",
129 name=
"fgcmStarIndices",
130 storageClass=
"Catalog",
131 dimensions=(
"instrument",),
134 fgcmReferenceStars = connectionTypes.Output(
135 doc=
"Catalog of fgcm-matched reference stars",
136 name=
"fgcmReferenceStars",
137 storageClass=
"Catalog",
138 dimensions=(
"instrument",),
144 if not config.doReferenceMatches:
145 self.prerequisiteInputs.remove(
"refCat")
146 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
148 if not config.doReferenceMatches:
149 self.outputs.remove(
"fgcmReferenceStars")
153 pipelineConnections=FgcmBuildStarsTableConnections):
154 """Config for FgcmBuildStarsTableTask"""
156 referenceCCD = pexConfig.Field(
157 doc=
"Reference CCD for checking PSF and background",
179 sourceSelector.flags.bad = [
'pixelFlags_edge',
180 'pixelFlags_interpolatedCenter',
181 'pixelFlags_saturatedCenter',
182 'pixelFlags_crCenter',
184 'pixelFlags_interpolated',
185 'pixelFlags_saturated',
191 sourceSelector.flags.bad.append(localBackgroundFlagName)
196 sourceSelector.isolated.parentName =
'parentSourceId'
197 sourceSelector.isolated.nChildName =
'deblend_nChild'
199 sourceSelector.requireFiniteRaDec.raColName =
'ra'
200 sourceSelector.requireFiniteRaDec.decColName =
'decl'
202 sourceSelector.unresolved.name =
'extendedness'
207 Build stars for the FGCM
global calibration, using sourceTable_visit catalogs.
209 ConfigClass = FgcmBuildStarsTableConfig
210 _DefaultName = "fgcmBuildStarsTable"
212 canMultiprocess =
False
215 super().
__init__(initInputs=initInputs, **kwargs)
216 if initInputs
is not None:
220 inputRefDict = butlerQC.get(inputRefs)
222 sourceTableHandles = inputRefDict[
'sourceTable_visit']
224 self.log.info(
"Running with %d sourceTable_visit handles",
225 len(sourceTableHandles))
227 sourceTableHandleDict = {sourceTableHandle.dataId[
'visit']: sourceTableHandle
for
228 sourceTableHandle
in sourceTableHandles}
230 if self.config.doReferenceMatches:
232 lutHandle = inputRefDict[
'fgcmLookUpTable']
235 refConfig = LoadReferenceObjectsConfig()
236 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
238 for ref
in inputRefs.refCat],
239 refCats=butlerQC.get(inputRefs.refCat),
240 name=self.config.connections.refCat,
243 self.makeSubtask(
'fgcmLoadReferenceCatalog',
244 refObjLoader=refObjLoader,
245 refCatName=self.config.connections.refCat)
251 calibFluxApertureRadius =
None
252 if self.config.doSubtractLocalBackground:
254 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
255 except RuntimeError
as e:
256 raise RuntimeError(
"Could not determine aperture radius from %s. "
257 "Cannot use doSubtractLocalBackground." %
258 (self.config.instFluxField))
from e
260 visitSummaryHandles = inputRefDict[
'visitSummary']
261 visitSummaryHandleDict = {visitSummaryHandle.dataId[
'visit']: visitSummaryHandle
for
262 visitSummaryHandle
in visitSummaryHandles}
264 camera = inputRefDict[
'camera']
266 visitSummaryHandleDict)
270 rad = calibFluxApertureRadius
275 calibFluxApertureRadius=rad)
277 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
278 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
280 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
281 fgcmStarObservationCat,
284 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
285 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
286 if fgcmRefCat
is not None:
287 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
289 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
290 """Group sourceTable and visitSummary handles.
294 sourceTableHandleDict : `dict` [`int`, `str`]
295 Dict of source tables, keyed by visit.
296 visitSummaryHandleDict : `dict` [int, `str`]
297 Dict of visit summary catalogs, keyed by visit.
301 groupedHandles : `dict` [`int`, `list`]
302 Dictionary with sorted visit keys,
and `list`s
with
303 `lsst.daf.butler.DeferredDataSetHandle`. The first
304 item
in the list will be the visitSummary ref,
and
305 the second will be the source table ref.
307 groupedHandles = collections.defaultdict(list)
308 visits = sorted(sourceTableHandleDict.keys())
311 groupedHandles[visit] = [visitSummaryHandleDict[visit],
312 sourceTableHandleDict[visit]]
314 return groupedHandles
319 calibFluxApertureRadius=None):
320 startTime = time.time()
322 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
323 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
328 outputSchema = sourceMapper.getOutputSchema()
332 for ccdIndex, detector
in enumerate(camera):
333 ccdMapping[detector.getId()] = ccdIndex
335 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
337 fullCatalog = afwTable.BaseCatalog(outputSchema)
339 visitKey = outputSchema[
'visit'].asKey()
340 ccdKey = outputSchema[
'ccd'].asKey()
341 instMagKey = outputSchema[
'instMag'].asKey()
342 instMagErrKey = outputSchema[
'instMagErr'].asKey()
343 deltaMagAperKey = outputSchema[
'deltaMagAper'].asKey()
346 if self.config.doSubtractLocalBackground:
347 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
353 for counter, visit
in enumerate(visitCat):
354 expTime = visit[
'exptime']
356 handle = groupedHandles[visit[
'visit']][-1]
359 inColumns = handle.get(component=
'columns')
361 df = handle.get(parameters={
'columns': columns})
363 goodSrc = self.sourceSelector.selectSources(df)
367 if self.config.doSubtractLocalBackground:
368 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
369 use, = np.where((goodSrc.selected)
370 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
372 use, = np.where(goodSrc.selected)
374 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
375 tempCat.resize(use.size)
377 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
378 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
379 tempCat[
'x'][:] = df[
'x'].values[use]
380 tempCat[
'y'][:] = df[
'y'].values[use]
382 tempCat[visitKey][:] = df[
'visit'].values[use]
383 tempCat[ccdKey][:] = df[
'detector'].values[use]
384 tempCat[
'psf_candidate'] = df[self.config.psfCandidateName].values[use]
386 with np.warnings.catch_warnings():
388 np.warnings.simplefilter(
"ignore")
390 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
391 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
392 / df[self.config.apertureInnerInstFluxField].values[use])
393 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
394 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
395 / df[self.config.apertureOuterInstFluxField].values[use])
396 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
398 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
400 if self.config.doSubtractLocalBackground:
414 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
415 - localBackground[use]) -
416 -2.5*np.log10(df[self.config.instFluxField].values[use]))
418 tempCat[
'deltaMagBkg'][:] = 0.0
421 for detector
in camera:
422 ccdId = detector.getId()
424 use2 = (tempCat[ccdKey] == ccdId)
425 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
427 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
428 * visit[
'scaling'][ccdMapping[ccdId]])
429 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
433 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
434 / df[self.config.instFluxField].values[use])
437 if self.config.doApplyWcsJacobian:
438 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
440 fullCatalog.extend(tempCat)
442 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
443 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
445 visit[
'deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
446 visit[
'sources_read'] =
True
448 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
449 use.size, visit[
'visit'], visit[
'deltaAper'])
451 self.log.info(
"Found all good star observations in %.2f s" %
452 (time.time() - startTime))
456 def _get_sourceTable_visit_columns(self, inColumns):
458 Get the sourceTable_visit columns from the config.
463 List of columns available
in the sourceTable_visit
468 List of columns to read
from sourceTable_visit.
471 columns = [
'visit',
'detector',
472 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
473 self.config.instFluxField, self.config.instFluxField +
'Err',
474 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
475 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
476 if self.sourceSelector.config.doFlags:
477 columns.extend(self.sourceSelector.config.flags.bad)
478 if self.sourceSelector.config.doUnresolved:
479 columns.append(self.sourceSelector.config.unresolved.name)
480 if self.sourceSelector.config.doIsolated:
481 columns.append(self.sourceSelector.config.isolated.parentName)
482 columns.append(self.sourceSelector.config.isolated.nChildName)
483 if self.config.doSubtractLocalBackground:
484 columns.append(self.config.localBackgroundFluxField)
def fgcmMakeVisitCatalog(self, camera, groupedHandles)
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def _makeSourceMapper(self, sourceSchema)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
apertureOuterInstFluxField
apertureInnerInstFluxField
doSubtractLocalBackground
apertureInnerInstFluxField
apertureOuterInstFluxField
def __init__(self, *config=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _get_sourceTable_visit_columns(self, inColumns)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)
def __init__(self, initInputs=None, **kwargs)