lsst.fgcmcal g7c13e6d1d1+fad4fd7449
Loading...
Searching...
No Matches
fgcmBuildFromIsolatedStars.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31import numpy as np
32import esutil
33import hpgeom as hpg
34from smatch.matcher import Matcher
35from astropy.table import Table, vstack
36
37from fgcm.fgcmUtilities import objFlagDict
38
39import lsst.pex.config as pexConfig
40import lsst.pipe.base as pipeBase
41from lsst.pipe.base import connectionTypes
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
43from lsst.pipe.tasks.reserveIsolatedStars import ReserveIsolatedStarsTask
44
45from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
46from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
47from .utilities import lookupStaticCalibrations
48
49__all__ = ["FgcmBuildFromIsolatedStarsConfig", "FgcmBuildFromIsolatedStarsTask"]
50
51
52class FgcmBuildFromIsolatedStarsConnections(pipeBase.PipelineTaskConnections,
53 dimensions=("instrument",),
54 defaultTemplates={}):
55 camera = connectionTypes.PrerequisiteInput(
56 doc="Camera instrument",
57 name="camera",
58 storageClass="Camera",
59 dimensions=("instrument",),
60 lookupFunction=lookupStaticCalibrations,
61 isCalibration=True,
62 )
63 fgcm_lookup_table = connectionTypes.PrerequisiteInput(
64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name="fgcmLookUpTable",
67 storageClass="Catalog",
68 dimensions=("instrument",),
69 deferLoad=True,
70 )
71 ref_cat = connectionTypes.PrerequisiteInput(
72 doc="Reference catalog to use for photometric calibration.",
73 name="cal_ref_cat",
74 storageClass="SimpleCatalog",
75 dimensions=("skypix",),
76 deferLoad=True,
77 multiple=True,
78 )
79 isolated_star_cats = pipeBase.connectionTypes.Input(
80 doc=("Catalog of isolated stars with average positions, number of associated "
81 "sources, and indexes to the isolated_star_sources catalogs."),
82 name="isolated_star_cat",
83 storageClass="ArrowAstropy",
84 dimensions=("instrument", "tract", "skymap"),
85 deferLoad=True,
86 multiple=True,
87 )
88 isolated_star_sources = pipeBase.connectionTypes.Input(
89 doc=("Catalog of isolated star sources with sourceIds, and indexes to the "
90 "isolated_star_cats catalogs."),
91 name="isolated_star_sources",
92 storageClass="ArrowAstropy",
93 dimensions=("instrument", "tract", "skymap"),
94 deferLoad=True,
95 multiple=True,
96 )
97 visit_summaries = connectionTypes.Input(
98 doc=("Per-visit consolidated exposure metadata. These catalogs use "
99 "detector id for the id and must be sorted for fast lookups of a "
100 "detector."),
101 name="visitSummary",
102 storageClass="ExposureCatalog",
103 dimensions=("instrument", "visit"),
104 deferLoad=True,
105 multiple=True,
106 )
107 fgcm_visit_catalog = connectionTypes.Output(
108 doc="Catalog of visit information for fgcm.",
109 name="fgcmVisitCatalog",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 )
113 fgcm_star_observations = connectionTypes.Output(
114 doc="Catalog of star observations for fgcm.",
115 name="fgcm_star_observations",
116 storageClass="ArrowAstropy",
117 dimensions=("instrument",),
118 )
119 fgcm_star_ids = connectionTypes.Output(
120 doc="Catalog of fgcm calibration star IDs.",
121 name="fgcm_star_ids",
122 storageClass="ArrowAstropy",
123 dimensions=("instrument",),
124 )
125 fgcm_reference_stars = connectionTypes.Output(
126 doc="Catalog of fgcm-matched reference stars.",
127 name="fgcm_reference_stars",
128 storageClass="ArrowAstropy",
129 dimensions=("instrument",),
130 )
131
132 def __init__(self, *, config=None):
133 super().__init__(config=config)
134
135 if not config.doReferenceMatches:
136 self.prerequisiteInputs.remove("ref_cat")
137 self.prerequisiteInputs.remove("fgcm_lookup_table")
138 self.outputs.remove("fgcm_reference_stars")
139
140
142 pipelineConnections=FgcmBuildFromIsolatedStarsConnections):
143 """Config for FgcmBuildFromIsolatedStarsTask."""
144 referenceCCD = pexConfig.Field(
145 doc="Reference detector for checking PSF and background.",
146 dtype=int,
147 default=40,
148 )
149 reserve_selection = pexConfig.ConfigurableField(
150 target=ReserveIsolatedStarsTask,
151 doc="Task to select reserved stars.",
152 )
153
154 def setDefaults(self):
155 super().setDefaults()
156
157 self.reserve_selection.reserve_name = "fgcmcal"
158 self.reserve_selection.reserve_fraction = 0.1
159
160 # The names here correspond to the isolated_star_sources.
161 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
162 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
165
166 source_selector = self.sourceSelector["science"]
167 source_selector.setDefaults()
168
169 source_selector.doFlags = False
170 source_selector.doSignalToNoise = True
171 source_selector.doUnresolved = False
172 source_selector.doIsolated = False
173 source_selector.doRequireFiniteRaDec = False
174
175 source_selector.flags.bad = []
176
177 source_selector.signalToNoise.minimum = 11.0
178 source_selector.signalToNoise.maximum = 1000.0
179 source_selector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
180 source_selector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
181
182
184 """Build star catalog for FGCM global calibration, using the isolated star catalogs.
185 """
186 ConfigClass = FgcmBuildFromIsolatedStarsConfig
187 _DefaultName = "fgcmBuildFromIsolatedStars"
188
189 canMultiprocess = False
190
191 def __init__(self, initInputs=None, **kwargs):
192 super().__init__(**kwargs)
193 self.makeSubtask('reserve_selection')
194
195 def runQuantum(self, butlerQC, inputRefs, outputRefs):
196 input_ref_dict = butlerQC.get(inputRefs)
197
198 isolated_star_cat_handles = input_ref_dict["isolated_star_cats"]
199 isolated_star_source_handles = input_ref_dict["isolated_star_sources"]
200
201 isolated_star_cat_handle_dict = {
202 handle.dataId["tract"]: handle for handle in isolated_star_cat_handles
203 }
204 isolated_star_source_handle_dict = {
205 handle.dataId["tract"]: handle for handle in isolated_star_source_handles
206 }
207
208 if len(isolated_star_cat_handle_dict) != len(isolated_star_source_handle_dict):
209 raise RuntimeError("isolated_star_cats and isolate_star_sources must have same length.")
210
211 for tract in isolated_star_cat_handle_dict:
212 if tract not in isolated_star_source_handle_dict:
213 raise RuntimeError(f"tract {tract} in isolated_star_cats but not isolated_star_sources")
214
215 if self.config.doReferenceMatches:
216 lookup_table_handle = input_ref_dict["fgcm_lookup_table"]
217
218 # Prepare the reference catalog loader
219 ref_config = LoadReferenceObjectsConfig()
220 ref_config.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
221 ref_obj_loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
222 for ref in inputRefs.ref_cat],
223 refCats=butlerQC.get(inputRefs.ref_cat),
224 name=self.config.connections.ref_cat,
225 log=self.log,
226 config=ref_config)
227 self.makeSubtask('fgcmLoadReferenceCatalog',
228 refObjLoader=ref_obj_loader,
229 refCatName=self.config.connections.ref_cat)
230 else:
231 lookup_table_handle = None
232
233 # The visit summary handles for use with fgcmMakeVisitCatalog must be keyed with
234 # visit, and values are a list with the first value being the visit_summary_handle,
235 # the second is the source catalog (which is not used, but a place-holder is
236 # left for compatibility reasons.)
237 visit_summary_handle_dict = {handle.dataId['visit']: [handle, None] for
238 handle in input_ref_dict['visit_summaries']}
239
240 camera = input_ref_dict["camera"]
241
242 struct = self.run(
243 camera=camera,
244 visit_summary_handle_dict=visit_summary_handle_dict,
245 isolated_star_cat_handle_dict=isolated_star_cat_handle_dict,
246 isolated_star_source_handle_dict=isolated_star_source_handle_dict,
247 lookup_table_handle=lookup_table_handle,
248 )
249
250 butlerQC.put(struct.fgcm_visit_catalog, outputRefs.fgcm_visit_catalog)
251 butlerQC.put(struct.fgcm_star_observations, outputRefs.fgcm_star_observations)
252 butlerQC.put(struct.fgcm_star_ids, outputRefs.fgcm_star_ids)
253 if self.config.doReferenceMatches:
254 butlerQC.put(struct.fgcm_reference_stars, outputRefs.fgcm_reference_stars)
255
256 def run(self, *, camera, visit_summary_handle_dict, isolated_star_cat_handle_dict,
257 isolated_star_source_handle_dict, lookup_table_handle=None):
258 """Run the fgcmBuildFromIsolatedStarsTask.
259
260 Parameters
261 ----------
262 camera : `lsst.afw.cameraGeom.Camera`
263 Camera object.
264 visit_summary_handle_dict : `dict` [`int`, [`lsst.daf.butler.DeferredDatasetHandle`]]
265 Visit summary dataset handles, with the visit as key.
266 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
267 Isolated star catalog dataset handles, with the tract as key.
268 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
269 Isolated star source dataset handles, with the tract as key.
270 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
271 Data reference to fgcm look-up table (used if matching reference stars).
272
273 Returns
274 -------
275 struct : `lsst.pipe.base.struct`
276 Catalogs for persistence, with attributes:
277
278 ``fgcm_visit_catalog``
279 Catalog of per-visit data (`lsst.afw.table.ExposureCatalog`).
280 ``fgcm_star_observations``
281 Catalog of individual star observations (`astropy.table.Table`).
282 ``fgcm_star_ids``
283 Catalog of per-star ids and positions (`astropy.table.Table`).
284 ``fgcm_reference_stars``
285 Catalog of reference stars matched to fgcm stars (`astropy.table.Table`).
286 """
287 # Compute aperture radius if necessary. This is useful to do now before
288 # any heave lifting has happened (fail early).
289 calib_flux_aperture_radius = None
290 if self.config.doSubtractLocalBackground:
291 try:
292 calib_flux_aperture_radius = computeApertureRadiusFromName(self.config.instFluxField)
293 except RuntimeError as e:
294 raise RuntimeError("Could not determine aperture radius from %s. "
295 "Cannot use doSubtractLocalBackground." %
296 (self.config.instFluxField)) from e
297
298 # Check that we have the lookup_table_handle if we are doing reference matches.
299 if self.config.doReferenceMatches:
300 if lookup_table_handle is None:
301 raise RuntimeError("Must supply lookup_table_handle if config.doReferenceMatches is True.")
302
303 visit_cat = self.fgcmMakeVisitCatalog(camera, visit_summary_handle_dict)
304
305 # Select and concatenate the isolated stars and sources.
306 fgcm_obj, star_obs = self._make_all_star_obs_from_isolated_stars(
307 isolated_star_cat_handle_dict,
308 isolated_star_source_handle_dict,
309 visit_cat,
310 camera,
311 calib_flux_aperture_radius=calib_flux_aperture_radius,
312 )
313
315 visit_cat,
316 star_obs,
317 )
318
319 # Do density down-sampling.
320 self._density_downsample(fgcm_obj, star_obs)
321
322 # Mark reserve stars
323 self._mark_reserve_stars(fgcm_obj)
324
325 # Do reference association.
326 if self.config.doReferenceMatches:
327 fgcm_ref = self._associate_reference_stars(lookup_table_handle, visit_cat, fgcm_obj)
328 else:
329 fgcm_ref = None
330
331 return pipeBase.Struct(
332 fgcm_visit_catalog=visit_cat,
333 fgcm_star_observations=star_obs,
334 fgcm_star_ids=fgcm_obj,
335 fgcm_reference_stars=fgcm_ref,
336 )
337
338 def _make_all_star_obs_from_isolated_stars(
339 self,
340 isolated_star_cat_handle_dict,
341 isolated_star_source_handle_dict,
342 visit_cat,
343 camera,
344 calib_flux_aperture_radius=None,
345 ):
346 """Make all star observations from isolated star catalogs.
347
348 Parameters
349 ----------
350 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
351 Isolated star catalog dataset handles, with the tract as key.
352 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
353 Isolated star source dataset handles, with the tract as key.
354 visit_cat : `lsst.afw.table.ExposureCatalog`
355 Catalog of per-visit data.
356 camera : `lsst.afw.cameraGeom.Camera`
357 The camera object.
358 calib_flux_aperture_radius : `float`, optional
359 Radius for the calibration flux aperture.
360
361 Returns
362 -------
363 fgcm_obj : `astropy.table.Table`
364 Catalog of ids and positions for each star.
365 star_obs : `astropy.table.Table`
366 Catalog of individual star observations.
367 """
368 source_columns = [
369 "sourceId",
370 "visit",
371 "detector",
372 "ra",
373 "decl",
374 "x",
375 "y",
376 "physical_filter",
377 "band",
378 "obj_index",
379 self.config.instFluxField,
380 self.config.instFluxField + "Err",
381 self.config.apertureInnerInstFluxField,
382 self.config.apertureInnerInstFluxField + "Err",
383 self.config.apertureOuterInstFluxField,
384 self.config.apertureOuterInstFluxField + "Err",
385 ]
386
387 if self.config.doSubtractLocalBackground:
388 source_columns.append(self.config.localBackgroundFluxField)
389 local_background_flag_name = self.config.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
390 source_columns.append(local_background_flag_name)
391
392 if self.sourceSelector.config.doFlags:
393 source_columns.extend(self.sourceSelector.config.flags.bad)
394
395 local_background_area = np.pi*calib_flux_aperture_radius**2.
396
397 # Compute the approximate pixel area over the full focal plane
398 # from the WCS jacobian using the camera model.
399 approx_pixel_area_fields = computeApproxPixelAreaFields(camera)
400
401 # Construct mapping from detector number to index.
402 detector_mapping = {}
403 for detector_index, detector in enumerate(camera):
404 detector_mapping[detector.getId()] = detector_index
405
406 star_obs_dtype = [
407 ("ra", "f8"),
408 ("dec", "f8"),
409 ("x", "f8"),
410 ("y", "f8"),
411 ("visit", "i8"),
412 ("detector", "i4"),
413 ("inst_mag", "f4"),
414 ("inst_mag_err", "f4"),
415 ("jacobian", "f4"),
416 ("delta_mag_bkg", "f4"),
417 ("delta_mag_aper", "f4"),
418 ("delta_mag_err_aper", "f4"),
419 ]
420
421 fgcm_obj_dtype = [
422 ("fgcm_id", "i8"),
423 ("isolated_star_id", "i8"),
424 ("ra", "f8"),
425 ("dec", "f8"),
426 ("obs_arr_index", "i4"),
427 ("n_obs", "i4"),
428 ("obj_flag", "i4"),
429 ]
430
431 fgcm_objs = []
432 star_obs_cats = []
433 merge_source_counter = 0
434
435 k = 2.5/np.log(10.)
436
437 visit_cat_table = visit_cat.asAstropy()
438
439 for tract in sorted(isolated_star_cat_handle_dict):
440 stars = isolated_star_cat_handle_dict[tract].get()
441 sources = isolated_star_source_handle_dict[tract].get(parameters={"columns": source_columns})
442
443 # Down-select sources.
444 good_sources = self.sourceSelector.selectSources(sources).selected
445 if self.config.doSubtractLocalBackground:
446 good_sources &= (~sources[local_background_flag_name])
447 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
448 good_sources &= ((sources[self.config.instFluxField] - local_background) > 0)
449
450 if good_sources.sum() == 0:
451 self.log.info("No good sources found in tract %d", tract)
452 continue
453
454 # Need to count the observations of each star after cuts, per band.
455 # If we have requiredBands specified, we must make sure that each star
456 # has the minumum number of observations in each of thos bands.
457 # Otherwise, we must make sure that each star has at least the minimum
458 # number of observations in _any_ band.
459 if len(self.config.requiredBands) > 0:
460 loop_bands = self.config.requiredBands
461 else:
462 loop_bands = np.unique(sources["band"])
463
464 n_req = np.zeros((len(loop_bands), len(stars)), dtype=np.int32)
465 for i, band in enumerate(loop_bands):
466 (band_use,) = (sources[good_sources]["band"] == band).nonzero()
467 np.add.at(
468 n_req,
469 (i, sources[good_sources]["obj_index"][band_use]),
470 1,
471 )
472
473 if len(self.config.requiredBands) > 0:
474 # The min gives us the band with the fewest observations, which must be
475 # above the limit.
476 (good_stars,) = (n_req.min(axis=0) >= self.config.minPerBand).nonzero()
477 else:
478 # The max gives us the band with the most observations, which must be
479 # above the limit.
480 (good_stars,) = (n_req.max(axis=0) >= self.config.minPerBand).nonzero()
481
482 # With the following matching:
483 # sources[good_sources][b] <-> stars[good_stars[a]]
484 obj_index = sources["obj_index"][good_sources]
485 a, b = esutil.numpy_util.match(good_stars, obj_index)
486
487 # Update indexes and cut to selected stars/sources
488 _, index_new = np.unique(a, return_index=True)
489 stars["source_cat_index"][good_stars] = index_new
490 sources = sources[good_sources][b]
491 sources["obj_index"][:] = a
492 stars = stars[good_stars]
493
494 nsource = np.zeros(len(stars), dtype=np.int32)
495 np.add.at(
496 nsource,
497 sources["obj_index"],
498 1,
499 )
500 stars["nsource"][:] = nsource
501
502 # After these cuts, the catalogs have the following properties:
503 # - ``stars`` only contains isolated stars that have the minimum number of good
504 # sources in the required bands.
505 # - ``sources`` has been cut to the good sources.
506 # - The slice [stars["source_cat_index"]: stars["source_cat_index"]
507 # + stars["nsource"]]
508 # applied to ``sources`` will give all the sources associated with the star.
509 # - For each source, sources["obj_index"] points to the index of the associated
510 # isolated star.
511
512 # We now reformat the sources and compute the ``observations`` that fgcm expects.
513 star_obs = Table(data=np.zeros(len(sources), dtype=star_obs_dtype))
514 star_obs["ra"] = sources["ra"]
515 star_obs["dec"] = sources["decl"]
516 star_obs["x"] = sources["x"]
517 star_obs["y"] = sources["y"]
518 star_obs["visit"] = sources["visit"]
519 star_obs["detector"] = sources["detector"]
520
521 visit_match, obs_match = esutil.numpy_util.match(visit_cat_table["visit"], sources["visit"])
522
523 exp_time = np.zeros(len(star_obs))
524 exp_time[obs_match] = visit_cat_table["exptime"][visit_match]
525
526 with np.warnings.catch_warnings():
527 # Ignore warnings, we will filter infinities and nans below.
528 np.warnings.simplefilter("ignore")
529
530 inst_mag_inner = -2.5*np.log10(sources[self.config.apertureInnerInstFluxField])
531 inst_mag_err_inner = k*(sources[self.config.apertureInnerInstFluxField + "Err"]
532 / sources[self.config.apertureInnerInstFluxField])
533 inst_mag_outer = -2.5*np.log10(sources[self.config.apertureOuterInstFluxField])
534 inst_mag_err_outer = k*(sources[self.config.apertureOuterInstFluxField + "Err"]
535 / sources[self.config.apertureOuterInstFluxField])
536 star_obs["delta_mag_aper"] = inst_mag_inner - inst_mag_outer
537 star_obs["delta_mag_err_aper"] = np.sqrt(inst_mag_err_inner**2. + inst_mag_err_outer**2.)
538 # Set bad values to sentinel values for fgcm.
539 bad = ~np.isfinite(star_obs["delta_mag_aper"])
540 star_obs["delta_mag_aper"][bad] = 99.0
541 star_obs["delta_mag_err_aper"][bad] = 99.0
542
543 if self.config.doSubtractLocalBackground:
544 # At the moment we only adjust the flux and not the flux
545 # error by the background because the error on
546 # base_LocalBackground_instFlux is the rms error in the
547 # background annulus, not the error on the mean in the
548 # background estimate (which is much smaller, by sqrt(n)
549 # pixels used to estimate the background, which we do not
550 # have access to in this task). In the default settings,
551 # the annulus is sufficiently large such that these
552 # additional errors are negligibly small (much less
553 # than a mmag in quadrature).
554
555 # This is the difference between the mag with local background correction
556 # and the mag without local background correction.
557 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
558 star_obs["delta_mag_bkg"] = (-2.5*np.log10(sources[self.config.instFluxField]
559 - local_background) -
560 -2.5*np.log10(sources[self.config.instFluxField]))
561
562 # Need to loop over detectors here.
563 for detector in camera:
564 detector_id = detector.getId()
565 # used index for all observations with a given detector
566 (use,) = (star_obs["detector"][obs_match] == detector_id).nonzero()
567 # Prior to running the calibration, we want to remove the effect
568 # of the jacobian of the WCS because that is a known quantity.
569 # Ideally, this would be done for each individual WCS, but this
570 # is extremely slow and makes small differences that are much
571 # smaller than the variation in the throughput due to other issues.
572 # Therefore, we use the approximate jacobian estimated from the
573 # camera model.
574 jac = approx_pixel_area_fields[detector_id].evaluate(
575 star_obs["x"][obs_match][use],
576 star_obs["y"][obs_match][use],
577 )
578 star_obs["jacobian"][obs_match[use]] = jac
579 scaled_inst_flux = (sources[self.config.instFluxField][obs_match[use]]
580 * visit_cat_table["scaling"][visit_match[use],
581 detector_mapping[detector_id]])
582 star_obs["inst_mag"][obs_match[use]] = (-2.5 * np.log10(scaled_inst_flux
583 / exp_time[use]))
584
585 # Compute instMagErr from inst_flux_err/inst_flux; scaling will cancel out.
586 star_obs["inst_mag_err"] = k*(sources[self.config.instFluxField + "Err"]
587 / sources[self.config.instFluxField])
588
589 # Apply the jacobian if configured to do so.
590 if self.config.doApplyWcsJacobian:
591 star_obs["inst_mag"] -= 2.5*np.log10(star_obs["jacobian"])
592
593 # We now reformat the stars and compute the ''objects'' that fgcm expects.
594 fgcm_obj = Table(data=np.zeros(len(stars), dtype=fgcm_obj_dtype))
595 fgcm_obj["isolated_star_id"] = stars["isolated_star_id"]
596 fgcm_obj["ra"] = stars["ra"]
597 fgcm_obj["dec"] = stars["decl"]
598 fgcm_obj["obs_arr_index"] = stars["source_cat_index"]
599 fgcm_obj["n_obs"] = stars["nsource"]
600
601 # Offset indexes to account for tract merging
602 fgcm_obj["obs_arr_index"] += merge_source_counter
603
604 fgcm_objs.append(fgcm_obj)
605 star_obs_cats.append(star_obs)
606
607 merge_source_counter += len(star_obs)
608
609 # Set the fgcm_id to a unique 64-bit integer for easier sorting.
610 fgcm_obj["fgcm_id"][:] = np.arange(len(fgcm_obj)) + 1
611
612 return vstack(fgcm_objs), vstack(star_obs_cats)
613
614 def _density_downsample(self, fgcm_obj, star_obs):
615 """Downsample stars according to density.
616
617 Catalogs are modified in-place.
618
619 Parameters
620 ----------
621 fgcm_obj : `astropy.table.Table`
622 Catalog of per-star ids and positions.
623 star_obs : `astropy.table.Table`
624 Catalog of individual star observations.
625 """
626 if self.config.randomSeed is not None:
627 np.random.seed(seed=self.config.randomSeed)
628
629 ipnest = hpg.angle_to_pixel(self.config.densityCutNside, fgcm_obj["ra"], fgcm_obj["dec"])
630 # Use the esutil.stat.histogram function to pull out the histogram of
631 # grouped pixels, and the rev_indices which describes which inputs
632 # are grouped together.
633 hist, rev_indices = esutil.stat.histogram(ipnest, rev=True)
634
635 obj_use = np.ones(len(fgcm_obj), dtype=bool)
636
637 (high,) = (hist > self.config.densityCutMaxPerPixel).nonzero()
638 (ok,) = (hist > 0).nonzero()
639 self.log.info("There are %d/%d pixels with high stellar density.", high.size, ok.size)
640 for i in range(high.size):
641 # The pix_indices are the indices of every star in the pixel.
642 pix_indices = rev_indices[rev_indices[high[i]]: rev_indices[high[i] + 1]]
643 # Cut down to the maximum number of stars in the pixel.
644 cut = np.random.choice(
645 pix_indices,
646 size=pix_indices.size - self.config.densityCutMaxPerPixel,
647 replace=False,
648 )
649 obj_use[cut] = False
650
651 fgcm_obj = fgcm_obj[obj_use]
652
653 obs_index = np.zeros(np.sum(fgcm_obj["n_obs"]), dtype=np.int32)
654 ctr = 0
655 for i in range(len(fgcm_obj)):
656 n_obs = fgcm_obj["n_obs"][i]
657 obs_index[ctr: ctr + n_obs] = (
658 np.arange(fgcm_obj["obs_arr_index"][i], fgcm_obj["obs_arr_index"][i] + n_obs)
659 )
660 fgcm_obj["obs_arr_index"][i] = ctr
661 ctr += n_obs
662
663 star_obs = star_obs[obs_index]
664
665 def _mark_reserve_stars(self, fgcm_obj):
666 """Run the star reservation task to flag reserved stars.
667
668 Parameters
669 ----------
670 fgcm_obj : `astropy.table.Table`
671 Catalog of per-star ids and positions.
672 """
673 reserved = self.reserve_selection.run(
674 len(fgcm_obj),
675 extra=','.join(self.config.requiredBands),
676 )
677 fgcm_obj["obj_flag"][reserved] |= objFlagDict["RESERVED"]
678
679 def _associate_reference_stars(self, lookup_table_handle, visit_cat, fgcm_obj):
680 """Associate fgcm isolated stars with reference stars.
681
682 Parameters
683 ----------
684 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
685 Data reference to fgcm look-up table (used if matching reference stars).
686 visit_cat : `lsst.afw.table.ExposureCatalog`
687 Catalog of per-visit data.
688 fgcm_obj : `astropy.table.Table`
689 Catalog of per-star ids and positions
690
691 Returns
692 -------
693 ref_cat : `astropy.table.Table`
694 Catalog of reference stars matched to fgcm stars.
695 """
696 # Figure out the correct bands/filters that we need based on the data.
697 lut_cat = lookup_table_handle.get()
698
699 std_filter_dict = {filter_name: std_filter for (filter_name, std_filter) in
700 zip(lut_cat[0]["physicalFilters"].split(","),
701 lut_cat[0]["stdPhysicalFilters"].split(","))}
702 std_lambda_dict = {std_filter: std_lambda for (std_filter, std_lambda) in
703 zip(lut_cat[0]["stdPhysicalFilters"].split(","),
704 lut_cat[0]["lambdaStdFilter"])}
705 del lut_cat
706
707 reference_filter_names = self._getReferenceFilterNames(
708 visit_cat,
709 std_filter_dict,
710 std_lambda_dict,
711 )
712 self.log.info("Using the following reference filters: %s", (", ".join(reference_filter_names)))
713
714 # Split into healpix pixels for more efficient matching.
715 ipnest = hpg.angle_to_pixel(self.config.coarseNside, fgcm_obj["ra"], fgcm_obj["dec"])
716 hpix, revpix = esutil.stat.histogram(ipnest, rev=True)
717
718 pixel_cats = []
719
720 # Compute the dtype from the filter names.
721 dtype = [("fgcm_id", "i4"),
722 ("refMag", "f4", (len(reference_filter_names), )),
723 ("refMagErr", "f4", (len(reference_filter_names), ))]
724
725 (gdpix,) = (hpix > 0).nonzero()
726 for ii, gpix in enumerate(gdpix):
727 p1a = revpix[revpix[gpix]: revpix[gpix + 1]]
728
729 # We do a simple wrapping of RA if we need to.
730 ra_wrap = fgcm_obj["ra"][p1a]
731 if (ra_wrap.min() < 10.0) and (ra_wrap.max() > 350.0):
732 ra_wrap[ra_wrap > 180.0] -= 360.0
733 mean_ra = np.mean(ra_wrap) % 360.0
734 else:
735 mean_ra = np.mean(ra_wrap)
736 mean_dec = np.mean(fgcm_obj["dec"][p1a])
737
738 dist = esutil.coords.sphdist(mean_ra, mean_dec, fgcm_obj["ra"][p1a], fgcm_obj["dec"][p1a])
739 rad = dist.max()
740
741 if rad < hpg.nside_to_resolution(self.config.coarseNside)/2.:
742 # Small radius, read just the circle.
743 ref_cat = self.fgcmLoadReferenceCatalog.getFgcmReferenceStarsSkyCircle(
744 mean_ra,
745 mean_dec,
746 rad,
747 reference_filter_names,
748 )
749 else:
750 # Otherwise the less efficient but full coverage.
751 ref_cat = self.fgcmLoadReferenceCatalog.getFgcmReferenceStarsHealpix(
752 self.config.coarseNside,
753 hpg.nest_to_ring(self.config.coarseNside, ipnest[p1a[0]]),
754 reference_filter_names,
755 )
756 if ref_cat.size == 0:
757 # No stars in this pixel; that's okay.
758 continue
759
760 with Matcher(fgcm_obj["ra"][p1a], fgcm_obj["dec"][p1a]) as matcher:
761 idx, i1, i2, d = matcher.query_radius(
762 ref_cat["ra"],
763 ref_cat["dec"],
764 self.config.matchRadius/3600.,
765 return_indices=True,
766 )
767
768 if len(i1) == 0:
769 # No matched stars in this pixel; that's okay.
770 continue
771
772 pixel_cat = Table(data=np.zeros(i1.size, dtype=dtype))
773 pixel_cat["fgcm_id"] = fgcm_obj["fgcm_id"][p1a[i1]]
774 pixel_cat["refMag"][:, :] = ref_cat["refMag"][i2, :]
775 pixel_cat["refMagErr"][:, :] = ref_cat["refMagErr"][i2, :]
776
777 pixel_cats.append(pixel_cat)
778
779 self.log.info(
780 "Found %d reference matches in pixel %d (%d of %d).",
781 len(pixel_cat),
782 ipnest[p1a[0]],
783 ii,
784 gdpix.size - 1,
785 )
786
787 ref_cat_full = vstack(pixel_cats)
788 ref_cat_full.meta.update({'FILTERNAMES': reference_filter_names})
789
790 return ref_cat_full
791
792 def _compute_delta_aper_summary_statistics(self, visit_cat, star_obs):
793 """Compute delta aperture summary statistics (per visit).
794
795 Parameters
796 ----------
797 visit_cat : `lsst.afw.table.ExposureCatalog`
798 Catalog of per-visit data.
799 star_obs : `astropy.table.Table`
800 Catalog of individual star observations.
801 """
802 (ok,) = ((star_obs["delta_mag_aper"] < 99.0)
803 & (star_obs["delta_mag_err_aper"] < 99.0)).nonzero()
804
805 visit_index = np.zeros(len(star_obs[ok]), dtype=np.int32)
806 a, b = esutil.numpy_util.match(visit_cat["visit"], star_obs["visit"][ok])
807 visit_index[b] = a
808
809 h, rev = esutil.stat.histogram(visit_index, rev=True)
810
811 visit_cat["deltaAper"] = -9999.0
812 h_use, = np.where(h >= 3)
813 for index in h_use:
814 i1a = rev[rev[index]: rev[index + 1]]
815 visit_cat["deltaAper"][index] = np.median(star_obs["delta_mag_aper"][ok[i1a]])
def _associate_reference_stars(self, lookup_table_handle, visit_cat, fgcm_obj)
def run(self, *camera, visit_summary_handle_dict, isolated_star_cat_handle_dict, isolated_star_source_handle_dict, lookup_table_handle=None)
def _make_all_star_obs_from_isolated_stars(self, isolated_star_cat_handle_dict, isolated_star_source_handle_dict, visit_cat, camera, calib_flux_aperture_radius=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)