198 input_ref_dict = butlerQC.get(inputRefs)
200 isolated_star_cat_handles = input_ref_dict[
"isolated_star_cats"]
201 isolated_star_source_handles = input_ref_dict[
"isolated_star_sources"]
203 isolated_star_cat_handle_dict = {
204 handle.dataId[
"tract"]: handle
for handle
in isolated_star_cat_handles
206 isolated_star_source_handle_dict = {
207 handle.dataId[
"tract"]: handle
for handle
in isolated_star_source_handles
210 if len(isolated_star_cat_handle_dict) != len(isolated_star_source_handle_dict):
211 raise RuntimeError(
"isolated_star_cats and isolate_star_sources must have same length.")
213 for tract
in isolated_star_cat_handle_dict:
214 if tract
not in isolated_star_source_handle_dict:
215 raise RuntimeError(f
"tract {tract} in isolated_star_cats but not isolated_star_sources")
217 if self.config.doReferenceMatches:
218 lookup_table_handle = input_ref_dict[
"fgcm_lookup_table"]
221 ref_config = LoadReferenceObjectsConfig()
222 ref_config.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
223 ref_obj_loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
224 for ref
in inputRefs.ref_cat],
225 refCats=butlerQC.get(inputRefs.ref_cat),
226 name=self.config.connections.ref_cat,
229 self.makeSubtask(
'fgcmLoadReferenceCatalog',
230 refObjLoader=ref_obj_loader,
231 refCatName=self.config.connections.ref_cat)
233 lookup_table_handle =
None
239 visit_summary_handle_dict = {handle.dataId[
'visit']: [handle,
None]
for
240 handle
in input_ref_dict[
'visit_summaries']}
242 camera = input_ref_dict[
"camera"]
246 visit_summary_handle_dict=visit_summary_handle_dict,
247 isolated_star_cat_handle_dict=isolated_star_cat_handle_dict,
248 isolated_star_source_handle_dict=isolated_star_source_handle_dict,
249 lookup_table_handle=lookup_table_handle,
252 butlerQC.put(struct.fgcm_visit_catalog, outputRefs.fgcm_visit_catalog)
253 butlerQC.put(struct.fgcm_star_observations, outputRefs.fgcm_star_observations)
254 butlerQC.put(struct.fgcm_star_ids, outputRefs.fgcm_star_ids)
255 if self.config.doReferenceMatches:
256 butlerQC.put(struct.fgcm_reference_stars, outputRefs.fgcm_reference_stars)
258 def run(self, *, camera, visit_summary_handle_dict, isolated_star_cat_handle_dict,
259 isolated_star_source_handle_dict, lookup_table_handle=None):
260 """Run the fgcmBuildFromIsolatedStarsTask.
264 camera : `lsst.afw.cameraGeom.Camera`
266 visit_summary_handle_dict : `dict` [`int`, [`lsst.daf.butler.DeferredDatasetHandle`]]
267 Visit summary dataset handles, with the visit as key.
268 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
269 Isolated star catalog dataset handles, with the tract as key.
270 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
271 Isolated star source dataset handles, with the tract as key.
272 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
273 Data reference to fgcm look-up table (used if matching reference stars).
277 struct : `lsst.pipe.base.struct`
278 Catalogs for persistence, with attributes:
280 ``fgcm_visit_catalog``
281 Catalog of per-visit data (`lsst.afw.table.ExposureCatalog`).
282 ``fgcm_star_observations``
283 Catalog of individual star observations (`astropy.table.Table`).
285 Catalog of per-star ids and positions (`astropy.table.Table`).
286 ``fgcm_reference_stars``
287 Catalog of reference stars matched to fgcm stars (`astropy.table.Table`).
291 calib_flux_aperture_radius =
None
292 if self.config.doSubtractLocalBackground:
294 calib_flux_aperture_radius = computeApertureRadiusFromName(self.config.instFluxField)
295 except RuntimeError
as e:
296 raise RuntimeError(
"Could not determine aperture radius from %s. "
297 "Cannot use doSubtractLocalBackground." %
298 (self.config.instFluxField))
from e
301 if self.config.doReferenceMatches:
302 if lookup_table_handle
is None:
303 raise RuntimeError(
"Must supply lookup_table_handle if config.doReferenceMatches is True.")
309 isolated_star_cat_handle_dict,
310 isolated_star_source_handle_dict,
313 calib_flux_aperture_radius=calib_flux_aperture_radius,
328 if self.config.doReferenceMatches:
333 return pipeBase.Struct(
334 fgcm_visit_catalog=visit_cat,
335 fgcm_star_observations=star_obs,
336 fgcm_star_ids=fgcm_obj,
337 fgcm_reference_stars=fgcm_ref,
342 isolated_star_cat_handle_dict,
343 isolated_star_source_handle_dict,
346 calib_flux_aperture_radius=None,
348 """Make all star observations from isolated star catalogs.
352 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
353 Isolated star catalog dataset handles, with the tract as key.
354 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
355 Isolated star source dataset handles, with the tract as key.
356 visit_cat : `lsst.afw.table.ExposureCatalog`
357 Catalog of per-visit data.
358 camera : `lsst.afw.cameraGeom.Camera`
360 calib_flux_aperture_radius : `float`, optional
361 Radius for the calibration flux aperture.
365 fgcm_obj : `astropy.table.Table`
366 Catalog of ids and positions for each star.
367 star_obs : `astropy.table.Table`
368 Catalog of individual star observations.
381 self.config.instFluxField,
382 self.config.instFluxField +
"Err",
383 self.config.apertureInnerInstFluxField,
384 self.config.apertureInnerInstFluxField +
"Err",
385 self.config.apertureOuterInstFluxField,
386 self.config.apertureOuterInstFluxField +
"Err",
389 if self.config.doSubtractLocalBackground:
390 source_columns.append(self.config.localBackgroundFluxField)
391 local_background_flag_name = self.config.localBackgroundFluxField[0: -len(
'instFlux')] +
'flag'
392 source_columns.append(local_background_flag_name)
394 if self.sourceSelector.config.doFlags:
395 source_columns.extend(self.sourceSelector.config.flags.bad)
397 local_background_area = np.pi*calib_flux_aperture_radius**2.
401 approx_pixel_area_fields = computeApproxPixelAreaFields(camera)
404 detector_mapping = {}
405 for detector_index, detector
in enumerate(camera):
406 detector_mapping[detector.getId()] = detector_index
416 (
"inst_mag_err",
"f4"),
418 (
"delta_mag_bkg",
"f4"),
419 (
"delta_mag_aper",
"f4"),
420 (
"delta_mag_err_aper",
"f4"),
425 (
"isolated_star_id",
"i8"),
428 (
"obs_arr_index",
"i4"),
435 merge_source_counter = 0
439 visit_cat_table = visit_cat.asAstropy()
441 for tract
in sorted(isolated_star_cat_handle_dict):
442 stars = isolated_star_cat_handle_dict[tract].get()
443 sources = isolated_star_source_handle_dict[tract].get(parameters={
"columns": source_columns})
446 good_sources = self.sourceSelector.selectSources(sources).selected
447 if self.config.doSubtractLocalBackground:
448 good_sources &= (~sources[local_background_flag_name])
449 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
450 good_sources &= ((sources[self.config.instFluxField] - local_background) > 0)
452 if good_sources.sum() == 0:
453 self.log.info(
"No good sources found in tract %d", tract)
461 if len(self.config.requiredBands) > 0:
462 loop_bands = self.config.requiredBands
464 loop_bands = np.unique(sources[
"band"])
466 n_req = np.zeros((len(loop_bands), len(stars)), dtype=np.int32)
467 for i, band
in enumerate(loop_bands):
468 (band_use,) = (sources[good_sources][
"band"] == band).nonzero()
471 (i, sources[good_sources][
"obj_index"][band_use]),
475 if len(self.config.requiredBands) > 0:
478 (good_stars,) = (n_req.min(axis=0) >= self.config.minPerBand).nonzero()
482 (good_stars,) = (n_req.max(axis=0) >= self.config.minPerBand).nonzero()
486 obj_index = sources[
"obj_index"][good_sources]
487 a, b = esutil.numpy_util.match(good_stars, obj_index)
490 _, index_new = np.unique(a, return_index=
True)
491 stars[
"source_cat_index"][good_stars] = index_new
492 sources = sources[good_sources][b]
493 sources[
"obj_index"][:] = a
494 stars = stars[good_stars]
496 nsource = np.zeros(len(stars), dtype=np.int32)
499 sources[
"obj_index"],
502 stars[
"nsource"][:] = nsource
515 star_obs = Table(data=np.zeros(len(sources), dtype=star_obs_dtype))
516 star_obs[
"ra"] = sources[
"ra"]
517 star_obs[
"dec"] = sources[
"dec"]
518 star_obs[
"x"] = sources[
"x"]
519 star_obs[
"y"] = sources[
"y"]
520 star_obs[
"visit"] = sources[
"visit"]
521 star_obs[
"detector"] = sources[
"detector"]
523 visit_match, obs_match = esutil.numpy_util.match(visit_cat_table[
"visit"], sources[
"visit"])
525 exp_time = np.zeros(len(star_obs))
526 exp_time[obs_match] = visit_cat_table[
"exptime"][visit_match]
528 with warnings.catch_warnings():
530 warnings.simplefilter(
"ignore")
532 inst_mag_inner = -2.5*np.log10(sources[self.config.apertureInnerInstFluxField])
533 inst_mag_err_inner = k*(sources[self.config.apertureInnerInstFluxField +
"Err"]
534 / sources[self.config.apertureInnerInstFluxField])
535 inst_mag_outer = -2.5*np.log10(sources[self.config.apertureOuterInstFluxField])
536 inst_mag_err_outer = k*(sources[self.config.apertureOuterInstFluxField +
"Err"]
537 / sources[self.config.apertureOuterInstFluxField])
538 star_obs[
"delta_mag_aper"] = inst_mag_inner - inst_mag_outer
539 star_obs[
"delta_mag_err_aper"] = np.sqrt(inst_mag_err_inner**2. + inst_mag_err_outer**2.)
541 bad = ~np.isfinite(star_obs[
"delta_mag_aper"])
542 star_obs[
"delta_mag_aper"][bad] = 99.0
543 star_obs[
"delta_mag_err_aper"][bad] = 99.0
545 if self.config.doSubtractLocalBackground:
559 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
560 star_obs[
"delta_mag_bkg"] = (-2.5*np.log10(sources[self.config.instFluxField]
561 - local_background) -
562 -2.5*np.log10(sources[self.config.instFluxField]))
565 for detector
in camera:
566 detector_id = detector.getId()
568 (use,) = (star_obs[
"detector"][obs_match] == detector_id).nonzero()
576 jac = approx_pixel_area_fields[detector_id].evaluate(
577 star_obs[
"x"][obs_match][use],
578 star_obs[
"y"][obs_match][use],
580 star_obs[
"jacobian"][obs_match[use]] = jac
581 scaled_inst_flux = (sources[self.config.instFluxField][obs_match[use]]
582 * visit_cat_table[
"scaling"][visit_match[use],
583 detector_mapping[detector_id]])
584 star_obs[
"inst_mag"][obs_match[use]] = (-2.5 * np.log10(scaled_inst_flux
588 star_obs[
"inst_mag_err"] = k*(sources[self.config.instFluxField +
"Err"]
589 / sources[self.config.instFluxField])
592 if self.config.doApplyWcsJacobian:
593 star_obs[
"inst_mag"] -= 2.5*np.log10(star_obs[
"jacobian"])
596 fgcm_obj = Table(data=np.zeros(len(stars), dtype=fgcm_obj_dtype))
597 fgcm_obj[
"isolated_star_id"] = stars[
"isolated_star_id"]
598 fgcm_obj[
"ra"] = stars[
"ra"]
599 fgcm_obj[
"dec"] = stars[
"dec"]
600 fgcm_obj[
"obs_arr_index"] = stars[
"source_cat_index"]
601 fgcm_obj[
"n_obs"] = stars[
"nsource"]
604 fgcm_obj[
"obs_arr_index"] += merge_source_counter
606 fgcm_objs.append(fgcm_obj)
607 star_obs_cats.append(star_obs)
609 merge_source_counter += len(star_obs)
611 fgcm_obj = vstack(fgcm_objs)
614 fgcm_obj[
"fgcm_id"][:] = np.arange(len(fgcm_obj)) + 1
616 return fgcm_obj, vstack(star_obs_cats)
619 """Downsample stars according to density.
621 Catalogs are modified in-place.
625 fgcm_obj : `astropy.table.Table`
626 Catalog of per-star ids and positions.
627 star_obs : `astropy.table.Table`
628 Catalog of individual star observations.
630 if self.config.randomSeed
is not None:
631 np.random.seed(seed=self.config.randomSeed)
633 ipnest = hpg.angle_to_pixel(self.config.densityCutNside, fgcm_obj[
"ra"], fgcm_obj[
"dec"])
637 hist, rev_indices = esutil.stat.histogram(ipnest, rev=
True)
639 obj_use = np.ones(len(fgcm_obj), dtype=bool)
641 (high,) = (hist > self.config.densityCutMaxPerPixel).nonzero()
642 (ok,) = (hist > 0).nonzero()
643 self.log.info(
"There are %d/%d pixels with high stellar density.", high.size, ok.size)
644 for i
in range(high.size):
646 pix_indices = rev_indices[rev_indices[high[i]]: rev_indices[high[i] + 1]]
648 cut = np.random.choice(
650 size=pix_indices.size - self.config.densityCutMaxPerPixel,
655 fgcm_obj = fgcm_obj[obj_use]
657 obs_index = np.zeros(np.sum(fgcm_obj[
"n_obs"]), dtype=np.int32)
659 for i
in range(len(fgcm_obj)):
660 n_obs = fgcm_obj[
"n_obs"][i]
661 obs_index[ctr: ctr + n_obs] = (
662 np.arange(fgcm_obj[
"obs_arr_index"][i], fgcm_obj[
"obs_arr_index"][i] + n_obs)
664 fgcm_obj[
"obs_arr_index"][i] = ctr
667 star_obs = star_obs[obs_index]
684 """Associate fgcm isolated stars with reference stars.
688 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
689 Data reference to fgcm look-up table (used if matching reference stars).
690 visit_cat : `lsst.afw.table.ExposureCatalog`
691 Catalog of per-visit data.
692 fgcm_obj : `astropy.table.Table`
693 Catalog of per-star ids and positions
697 ref_cat : `astropy.table.Table`
698 Catalog of reference stars matched to fgcm stars.
701 lut_cat = lookup_table_handle.get()
703 std_filter_dict = {filter_name: std_filter
for (filter_name, std_filter)
in
704 zip(lut_cat[0][
"physicalFilters"].split(
","),
705 lut_cat[0][
"stdPhysicalFilters"].split(
","))}
706 std_lambda_dict = {std_filter: std_lambda
for (std_filter, std_lambda)
in
707 zip(lut_cat[0][
"stdPhysicalFilters"].split(
","),
708 lut_cat[0][
"lambdaStdFilter"])}
716 self.log.info(
"Using the following reference filters: %s", (
", ".join(reference_filter_names)))
719 ipnest = hpg.angle_to_pixel(self.config.coarseNside, fgcm_obj[
"ra"], fgcm_obj[
"dec"])
720 hpix, revpix = esutil.stat.histogram(ipnest, rev=
True)
725 dtype = [(
"fgcm_id",
"i4"),
726 (
"refMag",
"f4", (len(reference_filter_names), )),
727 (
"refMagErr",
"f4", (len(reference_filter_names), ))]
729 (gdpix,) = (hpix > 0).nonzero()
730 for ii, gpix
in enumerate(gdpix):
731 p1a = revpix[revpix[gpix]: revpix[gpix + 1]]
734 ra_wrap = fgcm_obj[
"ra"][p1a]
735 if (ra_wrap.min() < 10.0)
and (ra_wrap.max() > 350.0):
736 ra_wrap[ra_wrap > 180.0] -= 360.0
737 mean_ra = np.mean(ra_wrap) % 360.0
739 mean_ra = np.mean(ra_wrap)
740 mean_dec = np.mean(fgcm_obj[
"dec"][p1a])
742 dist = esutil.coords.sphdist(mean_ra, mean_dec, fgcm_obj[
"ra"][p1a], fgcm_obj[
"dec"][p1a])
745 if rad < hpg.nside_to_resolution(self.config.coarseNside)/2.:
751 reference_filter_names,
756 self.config.coarseNside,
757 hpg.nest_to_ring(self.config.coarseNside, ipnest[p1a[0]]),
758 reference_filter_names,
760 if ref_cat.size == 0:
764 with Matcher(fgcm_obj[
"ra"][p1a], fgcm_obj[
"dec"][p1a])
as matcher:
765 idx, i1, i2, d = matcher.query_radius(
768 self.config.matchRadius/3600.,
776 pixel_cat = Table(data=np.zeros(i1.size, dtype=dtype))
777 pixel_cat[
"fgcm_id"] = fgcm_obj[
"fgcm_id"][p1a[i1]]
778 pixel_cat[
"refMag"][:, :] = ref_cat[
"refMag"][i2, :]
779 pixel_cat[
"refMagErr"][:, :] = ref_cat[
"refMagErr"][i2, :]
781 pixel_cats.append(pixel_cat)
784 "Found %d reference matches in pixel %d (%d of %d).",
791 ref_cat_full = vstack(pixel_cats)
792 ref_cat_full.meta.update({
'FILTERNAMES': reference_filter_names})