22"""Task to run a finalized image characterization, using additional data.
33import lsst.meas.extensions.psfex.psfexPsfDeterminer
35from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask
38from .reserveIsolatedStars
import ReserveIsolatedStarsTask
40__all__ = [
'FinalizeCharacterizationConnections',
41 'FinalizeCharacterizationConfig',
42 'FinalizeCharacterizationTask']
46 dimensions=(
'instrument',
'visit',),
48 src_schema = pipeBase.connectionTypes.InitInput(
49 doc=
'Input schema used for src catalogs.',
51 storageClass=
'SourceCatalog',
53 srcs = pipeBase.connectionTypes.Input(
54 doc=
'Source catalogs for the visit',
56 storageClass=
'SourceCatalog',
57 dimensions=(
'instrument',
'visit',
'detector'),
61 calexps = pipeBase.connectionTypes.Input(
62 doc=
'Calexps for the visit',
64 storageClass=
'ExposureF',
65 dimensions=(
'instrument',
'visit',
'detector'),
69 isolated_star_cats = pipeBase.connectionTypes.Input(
70 doc=(
'Catalog of isolated stars with average positions, number of associated '
71 'sources, and indexes to the isolated_star_sources catalogs.'),
72 name=
'isolated_star_cat',
73 storageClass=
'DataFrame',
74 dimensions=(
'instrument',
'tract',
'skymap'),
78 isolated_star_sources = pipeBase.connectionTypes.Input(
79 doc=(
'Catalog of isolated star sources with sourceIds, and indexes to the '
80 'isolated_star_cats catalogs.'),
81 name=
'isolated_star_sources',
82 storageClass=
'DataFrame',
83 dimensions=(
'instrument',
'tract',
'skymap'),
87 finalized_psf_ap_corr_cat = pipeBase.connectionTypes.Output(
88 doc=(
'Per-visit finalized psf models and aperture corrections. This '
89 'catalog uses detector id for the id and are sorted for fast '
90 'lookups of a detector.'),
91 name=
'finalized_psf_ap_corr_catalog',
92 storageClass=
'ExposureCatalog',
93 dimensions=(
'instrument',
'visit'),
95 finalized_src_table = pipeBase.connectionTypes.Output(
96 doc=(
'Per-visit catalog of measurements for psf/flag/etc.'),
97 name=
'finalized_src_table',
98 storageClass=
'DataFrame',
99 dimensions=(
'instrument',
'visit'),
104 pipelineConnections=FinalizeCharacterizationConnections):
105 """Configuration for FinalizeCharacterizationTask."""
106 source_selector = sourceSelectorRegistry.makeField(
107 doc=
"How to select sources",
110 id_column = pexConfig.Field(
111 doc=
'Name of column in isolated_star_sources with source id.',
115 reserve_selection = pexConfig.ConfigurableField(
116 target=ReserveIsolatedStarsTask,
117 doc=
'Task to select reserved stars',
119 make_psf_candidates = pexConfig.ConfigurableField(
120 target=measAlg.MakePsfCandidatesTask,
121 doc=
'Task to make psf candidates from selected stars.',
123 psf_determiner = measAlg.psfDeterminerRegistry.makeField(
124 'PSF Determination algorithm',
127 measurement = pexConfig.ConfigurableField(
128 target=SingleFrameMeasurementTask,
129 doc=
'Measure sources for aperture corrections'
131 measure_ap_corr = pexConfig.ConfigurableField(
132 target=MeasureApCorrTask,
133 doc=
"Subtask to measure aperture corrections"
135 apply_ap_corr = pexConfig.ConfigurableField(
136 target=ApplyApCorrTask,
137 doc=
"Subtask to apply aperture corrections"
144 source_selector.setDefaults()
150 source_selector.doFlags =
True
151 source_selector.doSignalToNoise =
True
152 source_selector.doFluxLimit =
False
153 source_selector.doUnresolved =
False
154 source_selector.doIsolated =
False
156 source_selector.signalToNoise.minimum = 20.0
157 source_selector.signalToNoise.maximum = 1000.0
159 source_selector.signalToNoise.fluxField =
'base_GaussianFlux_instFlux'
160 source_selector.signalToNoise.errField =
'base_GaussianFlux_instFluxErr'
162 source_selector.flags.bad = [
'base_PixelFlags_flag_edge',
163 'base_PixelFlags_flag_interpolatedCenter',
164 'base_PixelFlags_flag_saturatedCenter',
165 'base_PixelFlags_flag_crCenter',
166 'base_PixelFlags_flag_bad',
167 'base_PixelFlags_flag_interpolated',
168 'base_PixelFlags_flag_saturated',
169 'slot_Centroid_flag',
170 'base_GaussianFlux_flag']
172 self.
measure_ap_corrmeasure_ap_corr.sourceSelector[
'flagged'].field =
'final_psf_used'
174 import lsst.meas.modelfit
175 import lsst.meas.extensions.photometryKron
176 import lsst.meas.extensions.convolved
177 import lsst.meas.extensions.gaap
178 import lsst.meas.extensions.shapeHSM
184 'modelfit_DoubleShapeletPsfApprox',
186 'ext_photometryKron_KronFlux',
187 'ext_convolved_ConvolvedFlux',
189 'ext_shapeHSM_HsmShapeRegauss',
190 'ext_shapeHSM_HsmSourceMoments',
191 'ext_shapeHSM_HsmPsfMoments',
192 'ext_shapeHSM_HsmSourceMomentsRound',
194 self.
measurementmeasurement.slots.modelFlux =
'modelfit_CModel'
195 self.
measurementmeasurement.plugins[
'ext_convolved_ConvolvedFlux'].seeing.append(8.0)
196 self.
measurementmeasurement.plugins[
'ext_gaap_GaapFlux'].sigmas = [
204 self.
measurementmeasurement.plugins[
'ext_gaap_GaapFlux'].doPsfPhotometry =
True
205 self.
measurementmeasurement.slots.shape =
'ext_shapeHSM_HsmSourceMoments'
206 self.
measurementmeasurement.slots.psfShape =
'exp_shapeHSM_HsmPsfMoments'
207 self.
measurementmeasurement.plugins[
'ext_shapeHSM_HsmShapeRegauss'].deblendNChild =
""
214 names = self.
measurementmeasurement.plugins[
'ext_convolved_ConvolvedFlux'].getAllResultNames()
216 names = self.
measurementmeasurement.plugins[
"ext_gaap_GaapFlux"].getAllGaapResultNames()
221 """Run final characterization on exposures."""
222 ConfigClass = FinalizeCharacterizationConfig
223 _DefaultName =
'finalize_characterization'
226 super().
__init__(initInputs=initInputs, **kwargs)
229 initInputs[
'src_schema'].schema
232 self.makeSubtask(
'reserve_selection')
233 self.makeSubtask(
'source_selector')
234 self.makeSubtask(
'make_psf_candidates')
235 self.makeSubtask(
'psf_determiner')
236 self.makeSubtask(
'measurement', schema=self.
schemaschema)
237 self.makeSubtask(
'measure_ap_corr', schema=self.
schemaschema)
238 self.makeSubtask(
'apply_ap_corr', schema=self.
schemaschema)
241 self.source_selector.log.setLevel(self.source_selector.log.WARN)
244 input_handle_dict = butlerQC.get(inputRefs)
246 band = butlerQC.quantum.dataId[
'band']
247 visit = butlerQC.quantum.dataId[
'visit']
249 src_dict_temp = {handle.dataId[
'detector']: handle
250 for handle
in input_handle_dict[
'srcs']}
251 calexp_dict_temp = {handle.dataId[
'detector']: handle
252 for handle
in input_handle_dict[
'calexps']}
253 isolated_star_cat_dict_temp = {handle.dataId[
'tract']: handle
254 for handle
in input_handle_dict[
'isolated_star_cats']}
255 isolated_star_source_dict_temp = {handle.dataId[
'tract']: handle
256 for handle
in input_handle_dict[
'isolated_star_sources']}
259 src_dict = {detector: src_dict_temp[detector]
for
260 detector
in sorted(src_dict_temp.keys())}
261 calexp_dict = {detector: calexp_dict_temp[detector]
for
262 detector
in sorted(calexp_dict_temp.keys())}
263 isolated_star_cat_dict = {tract: isolated_star_cat_dict_temp[tract]
for
264 tract
in sorted(isolated_star_cat_dict_temp.keys())}
265 isolated_star_source_dict = {tract: isolated_star_source_dict_temp[tract]
for
266 tract
in sorted(isolated_star_source_dict_temp.keys())}
268 struct = self.
runrun(visit,
270 isolated_star_cat_dict,
271 isolated_star_source_dict,
275 butlerQC.put(struct.psf_ap_corr_cat,
276 outputRefs.finalized_psf_ap_corr_cat)
277 butlerQC.put(pd.DataFrame(struct.output_table),
278 outputRefs.finalized_src_table)
280 def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, src_dict, calexp_dict):
282 Run the FinalizeCharacterizationTask.
287 Visit number. Used in the output catalogs.
289 Band name. Used to select reserved stars.
290 isolated_star_cat_dict : `dict`
291 Per-tract dict of isolated star catalog handles.
292 isolated_star_source_dict : `dict`
293 Per-tract dict of isolated star source catalog handles.
295 Per-detector dict of src catalog handles.
297 Per-detector dict of calibrated exposure handles.
301 struct : `lsst.pipe.base.struct`
302 Struct
with outputs
for persistence.
308 isolated_star_cat_dict,
309 isolated_star_source_dict
312 exposure_cat_schema = afwTable.ExposureTable.makeMinimalSchema()
313 exposure_cat_schema.addField(
'visit', type=
'I', doc=
'Visit number')
315 metadata = dafBase.PropertyList()
316 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
317 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
319 psf_ap_corr_cat = afwTable.ExposureCatalog(exposure_cat_schema)
320 psf_ap_corr_cat.setMetadata(metadata)
322 measured_src_tables = []
324 for detector
in src_dict:
325 src = src_dict[detector].get()
326 exposure = calexp_dict[detector].get()
333 isolated_source_table
337 record = psf_ap_corr_cat.addNew()
338 record[
'id'] = int(detector)
339 record[
'visit'] = visit
342 if ap_corr_map
is not None:
343 record.setApCorrMap(ap_corr_map)
345 measured_src[
'visit'][:] = visit
346 measured_src[
'detector'][:] = detector
348 measured_src_tables.append(measured_src.asAstropy().as_array())
350 measured_src_table = np.concatenate(measured_src_tables)
352 return pipeBase.Struct(psf_ap_corr_cat=psf_ap_corr_cat,
353 output_table=measured_src_table)
355 def _make_output_schema_mapper(self, input_schema):
356 """Make the schema mapper from the input schema to the output schema.
368 Output schema (with alias map)
370 mapper = afwTable.SchemaMapper(input_schema)
371 mapper.addMinimalSchema(afwTable.SourceTable.makeMinimalSchema())
372 mapper.addMapping(input_schema['slot_Centroid_x'].asKey())
373 mapper.addMapping(input_schema[
'slot_Centroid_y'].asKey())
376 aper_fields = input_schema.extract(
'base_CircularApertureFlux_*')
377 for field, item
in aper_fields.items():
378 mapper.addMapping(item.key)
381 apflux_fields = input_schema.extract(
'slot_ApFlux_*')
382 for field, item
in apflux_fields.items():
383 mapper.addMapping(item.key)
385 calibflux_fields = input_schema.extract(
'slot_CalibFlux_*')
386 for field, item
in calibflux_fields.items():
387 mapper.addMapping(item.key)
390 input_schema[self.config.source_selector.active.signalToNoise.fluxField].asKey(),
391 'final_psf_selection_flux')
393 input_schema[self.config.source_selector.active.signalToNoise.errField].asKey(),
394 'final_psf_selection_flux_err')
396 output_schema = mapper.getOutputSchema()
398 output_schema.addField(
399 'final_psf_candidate',
401 doc=(
'set if the source was a candidate for PSF determination, '
402 'as determined from FinalizeCharacterizationTask.'),
404 output_schema.addField(
405 'final_psf_reserved',
407 doc=(
'set if source was reserved from PSF determination by '
408 'FinalizeCharacterizationTask.'),
410 output_schema.addField(
413 doc=(
'set if source was used in the PSF determination by '
414 'FinalizeCharacterizationTask.'),
416 output_schema.addField(
419 doc=
'Visit number for the sources.',
421 output_schema.addField(
424 doc=
'Detector number for the sources.',
427 alias_map = input_schema.getAliasMap()
428 alias_map_output = afwTable.AliasMap()
429 alias_map_output.set(
'slot_Centroid', alias_map.get(
'slot_Centroid'))
430 alias_map_output.set(
'slot_ApFlux', alias_map.get(
'slot_ApFlux'))
431 alias_map_output.set(
'slot_CalibFlux', alias_map.get(
'slot_CalibFlux'))
433 output_schema.setAliasMap(alias_map_output)
435 return mapper, output_schema
437 def _make_selection_schema_mapper(self, input_schema):
438 """Make the schema mapper from the input schema to the selection schema.
450 Selection schema (with alias map)
452 mapper = afwTable.SchemaMapper(input_schema)
453 mapper.addMinimalSchema(input_schema)
455 selection_schema = mapper.getOutputSchema()
457 selection_schema.addField(
458 'final_psf_candidate',
460 doc=(
'set if the source was a candidate for PSF determination, '
461 'as determined from FinalizeCharacterizationTask.'),
463 selection_schema.addField(
464 'final_psf_reserved',
466 doc=(
'set if source was reserved from PSF determination by '
467 'FinalizeCharacterizationTask.'),
469 selection_schema.addField(
472 doc=(
'set if source was used in the PSF determination by '
473 'FinalizeCharacterizationTask.'),
476 selection_schema.setAliasMap(input_schema.getAliasMap())
478 return mapper, selection_schema
482 Concatenate isolated star catalogs and make reserve selection.
487 Band name. Used to select reserved stars.
488 isolated_star_cat_dict : `dict`
489 Per-tract dict of isolated star catalog handles.
490 isolated_star_source_dict : `dict`
491 Per-tract dict of isolated star source catalog handles.
495 isolated_table : `np.ndarray` (N,)
496 Table of isolated stars,
with indexes to isolated sources.
497 isolated_source_table : `np.ndarray` (M,)
498 Table of isolated sources,
with indexes to isolated stars.
501 isolated_sources = []
502 merge_cat_counter = 0
503 merge_source_counter = 0
505 for tract
in isolated_star_cat_dict:
506 df_cat = isolated_star_cat_dict[tract].get()
507 table_cat = df_cat.to_records()
509 df_source = isolated_star_source_dict[tract].get(
510 parameters={
'columns': [self.config.id_column,
513 table_source = df_source.to_records()
516 (use_band,) = (table_cat[f
'nsource_{band}'] > 0).nonzero()
520 obj_index = table_source[
'obj_index'][:]
521 a, b = esutil.numpy_util.match(use_band, obj_index)
524 table_source[
'obj_index'][b] = a
525 _, index_new = np.unique(a, return_index=
True)
526 table_cat[f
'source_cat_index_{band}'][use_band] = index_new
537 table_source = table_source[b]
538 table_cat = table_cat[use_band]
541 table_cat = np.lib.recfunctions.append_fields(
544 np.zeros(table_cat.size, dtype=bool),
547 table_source = np.lib.recfunctions.append_fields(
550 np.zeros(table_source.size, dtype=bool),
555 table_cat[
'reserved'][:] = self.reserve_selection.
run(
557 extra=f
'{band}_{tract}',
559 table_source[
'reserved'][:] = table_cat[
'reserved'][table_source[
'obj_index']]
562 table_cat[f
'source_cat_index_{band}'] += merge_source_counter
563 table_source[
'obj_index'] += merge_cat_counter
565 isolated_tables.append(table_cat)
566 isolated_sources.append(table_source)
568 merge_cat_counter += len(table_cat)
569 merge_source_counter += len(table_source)
571 isolated_table = np.concatenate(isolated_tables)
572 isolated_source_table = np.concatenate(isolated_sources)
574 return isolated_table, isolated_source_table
577 """Compute psf model and aperture correction map for a single exposure.
582 Visit number (for logging).
584 Detector number (
for logging).
585 exposure : `lsst.afw.image.ExposureF`
587 isolated_source_table : `np.ndarray`
594 Aperture correction map.
596 Updated source catalog
with measurements, flags
and aperture corrections.
599 good_src = self.source_selector.selectSources(src)
608 selected_src = afwTable.SourceCatalog(selection_schema)
609 selected_src.reserve(good_src.selected.sum())
610 selected_src.extend(src[good_src.selected], mapper=selection_mapper)
613 matched_src, matched_iso = esutil.numpy_util.match(
615 isolated_source_table[self.config.id_column]
618 matched_arr = np.zeros(len(selected_src), dtype=bool)
619 matched_arr[matched_src] =
True
620 selected_src[
'final_psf_candidate'] = matched_arr
622 reserved_arr = np.zeros(len(selected_src), dtype=bool)
623 reserved_arr[matched_src] = isolated_source_table[
'reserved'][matched_iso]
624 selected_src[
'final_psf_reserved'] = reserved_arr
626 selected_src = selected_src[selected_src[
'final_psf_candidate']].copy(deep=
True)
629 measured_src = afwTable.SourceCatalog(self.
schemaschema)
630 measured_src.reserve(len(selected_src))
631 measured_src.extend(selected_src, mapper=self.schema_mapper)
634 measured_src[
'final_psf_candidate'] = selected_src[
'final_psf_candidate']
635 measured_src[
'final_psf_reserved'] = selected_src[
'final_psf_reserved']
639 psf_selection_result = self.make_psf_candidates.
run(selected_src, exposure=exposure)
640 except Exception
as e:
641 self.log.warning(
'Failed to make psf candidates for visit %d, detector %d: %s',
643 return None,
None, measured_src
645 psf_cand_cat = psf_selection_result.goodStarCat
649 psf_determiner_list = [cand
for cand, use
650 in zip(psf_selection_result.psfCandidates,
651 ~psf_cand_cat[
'final_psf_reserved'])
if use]
652 flag_key = psf_cand_cat.schema[
'final_psf_used'].asKey()
654 psf, cell_set = self.psf_determiner.determinePsf(exposure,
658 except Exception
as e:
659 self.log.warning(
'Failed to determine psf for visit %d, detector %d: %s',
661 return None,
None, measured_src
665 matched_selected, matched_measured = esutil.numpy_util.match(
669 measured_used = np.zeros(len(measured_src), dtype=bool)
670 measured_used[matched_measured] = selected_src[
'final_psf_used'][matched_selected]
671 measured_src[
'final_psf_used'] = measured_used
675 self.measurement.
run(measCat=measured_src, exposure=exposure)
676 except Exception
as e:
677 self.log.warning(
'Failed to make measurements for visit %d, detector %d: %s',
679 return psf,
None, measured_src
683 ap_corr_map = self.measure_ap_corr.
run(exposure=exposure,
684 catalog=measured_src).apCorrMap
685 except Exception
as e:
686 self.log.warning(
'Failed to compute aperture corrections for visit %d, detector %d: %s',
688 return psf,
None, measured_src
690 self.apply_ap_corr.
run(catalog=measured_src, apCorrMap=ap_corr_map)
692 return psf, ap_corr_map, measured_src
def _make_output_schema_mapper(self, input_schema)
def __init__(self, initInputs=None, **kwargs)
def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_source_table)
def _make_selection_schema_mapper(self, input_schema)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def concat_isolated_star_cats(self, band, isolated_star_cat_dict, isolated_star_source_dict)
def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, src_dict, calexp_dict)