lsst.fgcmcal g98cecbdb62+ee46cc4507
Loading...
Searching...
No Matches
fgcmBuildFromIsolatedStars.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Build star observations for input to FGCM using sourceTable_visit.
24
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs as much cleaning of the
29input catalog as possible.
30"""
31import warnings
32import numpy as np
33import esutil
34import hpgeom as hpg
35from smatch.matcher import Matcher
36from astropy.table import Table, vstack
37
38from fgcm.fgcmUtilities import objFlagDict
39
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from lsst.pipe.tasks.reserveIsolatedStars import ReserveIsolatedStarsTask
45
46from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
47from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
48from .utilities import lookupStaticCalibrations
49
50__all__ = ["FgcmBuildFromIsolatedStarsConfig", "FgcmBuildFromIsolatedStarsTask"]
51
52
53class FgcmBuildFromIsolatedStarsConnections(pipeBase.PipelineTaskConnections,
54 dimensions=("instrument",),
55 defaultTemplates={}):
56 camera = connectionTypes.PrerequisiteInput(
57 doc="Camera instrument",
58 name="camera",
59 storageClass="Camera",
60 dimensions=("instrument",),
61 lookupFunction=lookupStaticCalibrations,
62 isCalibration=True,
63 )
64 fgcm_lookup_table = connectionTypes.PrerequisiteInput(
65 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
66 "chromatic corrections."),
67 name="fgcmLookUpTable",
68 storageClass="Catalog",
69 dimensions=("instrument",),
70 deferLoad=True,
71 )
72 ref_cat = connectionTypes.PrerequisiteInput(
73 doc="Reference catalog to use for photometric calibration.",
74 name="cal_ref_cat",
75 storageClass="SimpleCatalog",
76 dimensions=("skypix",),
77 deferLoad=True,
78 multiple=True,
79 )
80 isolated_star_cats = pipeBase.connectionTypes.Input(
81 doc=("Catalog of isolated stars with average positions, number of associated "
82 "sources, and indexes to the isolated_star_sources catalogs."),
83 name="isolated_star_cat",
84 storageClass="ArrowAstropy",
85 dimensions=("instrument", "tract", "skymap"),
86 deferLoad=True,
87 multiple=True,
88 )
89 isolated_star_sources = pipeBase.connectionTypes.Input(
90 doc=("Catalog of isolated star sources with sourceIds, and indexes to the "
91 "isolated_star_cats catalogs."),
92 name="isolated_star_sources",
93 storageClass="ArrowAstropy",
94 dimensions=("instrument", "tract", "skymap"),
95 deferLoad=True,
96 multiple=True,
97 )
98 visit_summaries = connectionTypes.Input(
99 doc=("Per-visit consolidated exposure metadata. These catalogs use "
100 "detector id for the id and must be sorted for fast lookups of a "
101 "detector."),
102 name="visitSummary",
103 storageClass="ExposureCatalog",
104 dimensions=("instrument", "visit"),
105 deferLoad=True,
106 multiple=True,
107 )
108 fgcm_visit_catalog = connectionTypes.Output(
109 doc="Catalog of visit information for fgcm.",
110 name="fgcmVisitCatalog",
111 storageClass="Catalog",
112 dimensions=("instrument",),
113 )
114 fgcm_star_observations = connectionTypes.Output(
115 doc="Catalog of star observations for fgcm.",
116 name="fgcm_star_observations",
117 storageClass="ArrowAstropy",
118 dimensions=("instrument",),
119 )
120 fgcm_star_ids = connectionTypes.Output(
121 doc="Catalog of fgcm calibration star IDs.",
122 name="fgcm_star_ids",
123 storageClass="ArrowAstropy",
124 dimensions=("instrument",),
125 )
126 fgcm_reference_stars = connectionTypes.Output(
127 doc="Catalog of fgcm-matched reference stars.",
128 name="fgcm_reference_stars",
129 storageClass="ArrowAstropy",
130 dimensions=("instrument",),
131 )
132
133 def __init__(self, *, config=None):
134 super().__init__(config=config)
135
136 if not config.doReferenceMatches:
137 self.prerequisiteInputs.remove("ref_cat")
138 self.prerequisiteInputs.remove("fgcm_lookup_table")
139 self.outputs.remove("fgcm_reference_stars")
140
142 return ("isolated_star_cats", "visit_summaries")
143
144
146 pipelineConnections=FgcmBuildFromIsolatedStarsConnections):
147 """Config for FgcmBuildFromIsolatedStarsTask."""
148 referenceCCD = pexConfig.Field(
149 doc="Reference detector for checking PSF and background.",
150 dtype=int,
151 default=40,
152 )
153 reserve_selection = pexConfig.ConfigurableField(
154 target=ReserveIsolatedStarsTask,
155 doc="Task to select reserved stars.",
156 )
157
158 def setDefaults(self):
159 super().setDefaults()
160
161 self.reserve_selection.reserve_name = "fgcmcal"
162 self.reserve_selection.reserve_fraction = 0.1
163
164 # The names here correspond to the isolated_star_sources.
165 self.instFluxFieldinstFluxField = 'apFlux_12_0_instFlux'
166 self.localBackgroundFluxFieldlocalBackgroundFluxField = 'localBackground_instFlux'
169
170 source_selector = self.sourceSelector["science"]
171 source_selector.setDefaults()
172
173 source_selector.doFlags = False
174 source_selector.doSignalToNoise = True
175 source_selector.doUnresolved = False
176 source_selector.doIsolated = False
177 source_selector.doRequireFiniteRaDec = False
178
179 source_selector.flags.bad = []
180
181 source_selector.signalToNoise.minimum = 11.0
182 source_selector.signalToNoise.maximum = 1000.0
183 source_selector.signalToNoise.fluxField = self.instFluxFieldinstFluxField
184 source_selector.signalToNoise.errField = self.instFluxFieldinstFluxField + 'Err'
185
186
188 """Build star catalog for FGCM global calibration, using the isolated star catalogs.
189 """
190 ConfigClass = FgcmBuildFromIsolatedStarsConfig
191 _DefaultName = "fgcmBuildFromIsolatedStars"
192
193 canMultiprocess = False
194
195 def __init__(self, initInputs=None, **kwargs):
196 super().__init__(**kwargs)
197 self.makeSubtask('reserve_selection')
198
199 def runQuantum(self, butlerQC, inputRefs, outputRefs):
200 input_ref_dict = butlerQC.get(inputRefs)
201
202 isolated_star_cat_handles = input_ref_dict["isolated_star_cats"]
203 isolated_star_source_handles = input_ref_dict["isolated_star_sources"]
204
205 isolated_star_cat_handle_dict = {
206 handle.dataId["tract"]: handle for handle in isolated_star_cat_handles
207 }
208 isolated_star_source_handle_dict = {
209 handle.dataId["tract"]: handle for handle in isolated_star_source_handles
210 }
211
212 if len(isolated_star_cat_handle_dict) != len(isolated_star_source_handle_dict):
213 raise RuntimeError("isolated_star_cats and isolate_star_sources must have same length.")
214
215 for tract in isolated_star_cat_handle_dict:
216 if tract not in isolated_star_source_handle_dict:
217 raise RuntimeError(f"tract {tract} in isolated_star_cats but not isolated_star_sources")
218
219 if self.config.doReferenceMatches:
220 lookup_table_handle = input_ref_dict["fgcm_lookup_table"]
221
222 # Prepare the reference catalog loader
223 ref_config = LoadReferenceObjectsConfig()
224 ref_config.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
225 ref_obj_loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
226 for ref in inputRefs.ref_cat],
227 refCats=butlerQC.get(inputRefs.ref_cat),
228 name=self.config.connections.ref_cat,
229 log=self.log,
230 config=ref_config)
231 self.makeSubtask('fgcmLoadReferenceCatalog',
232 refObjLoader=ref_obj_loader,
233 refCatName=self.config.connections.ref_cat)
234 else:
235 lookup_table_handle = None
236
237 # The visit summary handles for use with fgcmMakeVisitCatalog must be keyed with
238 # visit, and values are a list with the first value being the visit_summary_handle,
239 # the second is the source catalog (which is not used, but a place-holder is
240 # left for compatibility reasons.)
241 visit_summary_handle_dict = {handle.dataId['visit']: [handle, None] for
242 handle in input_ref_dict['visit_summaries']}
243
244 camera = input_ref_dict["camera"]
245
246 struct = self.run(
247 camera=camera,
248 visit_summary_handle_dict=visit_summary_handle_dict,
249 isolated_star_cat_handle_dict=isolated_star_cat_handle_dict,
250 isolated_star_source_handle_dict=isolated_star_source_handle_dict,
251 lookup_table_handle=lookup_table_handle,
252 )
253
254 butlerQC.put(struct.fgcm_visit_catalog, outputRefs.fgcm_visit_catalog)
255 butlerQC.put(struct.fgcm_star_observations, outputRefs.fgcm_star_observations)
256 butlerQC.put(struct.fgcm_star_ids, outputRefs.fgcm_star_ids)
257 if self.config.doReferenceMatches:
258 butlerQC.put(struct.fgcm_reference_stars, outputRefs.fgcm_reference_stars)
259
260 def run(self, *, camera, visit_summary_handle_dict, isolated_star_cat_handle_dict,
261 isolated_star_source_handle_dict, lookup_table_handle=None):
262 """Run the fgcmBuildFromIsolatedStarsTask.
263
264 Parameters
265 ----------
266 camera : `lsst.afw.cameraGeom.Camera`
267 Camera object.
268 visit_summary_handle_dict : `dict` [`int`, [`lsst.daf.butler.DeferredDatasetHandle`]]
269 Visit summary dataset handles, with the visit as key.
270 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
271 Isolated star catalog dataset handles, with the tract as key.
272 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
273 Isolated star source dataset handles, with the tract as key.
274 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
275 Data reference to fgcm look-up table (used if matching reference stars).
276
277 Returns
278 -------
279 struct : `lsst.pipe.base.struct`
280 Catalogs for persistence, with attributes:
281
282 ``fgcm_visit_catalog``
283 Catalog of per-visit data (`lsst.afw.table.ExposureCatalog`).
284 ``fgcm_star_observations``
285 Catalog of individual star observations (`astropy.table.Table`).
286 ``fgcm_star_ids``
287 Catalog of per-star ids and positions (`astropy.table.Table`).
288 ``fgcm_reference_stars``
289 Catalog of reference stars matched to fgcm stars (`astropy.table.Table`).
290 """
291 # Compute aperture radius if necessary. This is useful to do now before
292 # any heave lifting has happened (fail early).
293 calib_flux_aperture_radius = None
294 if self.config.doSubtractLocalBackground:
295 try:
296 calib_flux_aperture_radius = computeApertureRadiusFromName(self.config.instFluxField)
297 except RuntimeError as e:
298 raise RuntimeError("Could not determine aperture radius from %s. "
299 "Cannot use doSubtractLocalBackground." %
300 (self.config.instFluxField)) from e
301
302 # Check that we have the lookup_table_handle if we are doing reference matches.
303 if self.config.doReferenceMatches:
304 if lookup_table_handle is None:
305 raise RuntimeError("Must supply lookup_table_handle if config.doReferenceMatches is True.")
306
307 visit_cat = self.fgcmMakeVisitCatalog(camera, visit_summary_handle_dict)
308
309 # Select and concatenate the isolated stars and sources.
310 fgcm_obj, star_obs = self._make_all_star_obs_from_isolated_stars(
311 isolated_star_cat_handle_dict,
312 isolated_star_source_handle_dict,
313 visit_cat,
314 camera,
315 calib_flux_aperture_radius=calib_flux_aperture_radius,
316 )
317
319 visit_cat,
320 star_obs,
321 )
322
323 # Do density down-sampling.
324 self._density_downsample(fgcm_obj, star_obs)
325
326 # Mark reserve stars
327 self._mark_reserve_stars(fgcm_obj)
328
329 # Do reference association.
330 if self.config.doReferenceMatches:
331 fgcm_ref = self._associate_reference_stars(lookup_table_handle, visit_cat, fgcm_obj)
332 else:
333 fgcm_ref = None
334
335 return pipeBase.Struct(
336 fgcm_visit_catalog=visit_cat,
337 fgcm_star_observations=star_obs,
338 fgcm_star_ids=fgcm_obj,
339 fgcm_reference_stars=fgcm_ref,
340 )
341
343 self,
344 isolated_star_cat_handle_dict,
345 isolated_star_source_handle_dict,
346 visit_cat,
347 camera,
348 calib_flux_aperture_radius=None,
349 ):
350 """Make all star observations from isolated star catalogs.
351
352 Parameters
353 ----------
354 isolated_star_cat_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
355 Isolated star catalog dataset handles, with the tract as key.
356 isolated_star_source_handle_dict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
357 Isolated star source dataset handles, with the tract as key.
358 visit_cat : `lsst.afw.table.ExposureCatalog`
359 Catalog of per-visit data.
360 camera : `lsst.afw.cameraGeom.Camera`
361 The camera object.
362 calib_flux_aperture_radius : `float`, optional
363 Radius for the calibration flux aperture.
364
365 Returns
366 -------
367 fgcm_obj : `astropy.table.Table`
368 Catalog of ids and positions for each star.
369 star_obs : `astropy.table.Table`
370 Catalog of individual star observations.
371 """
372 source_columns = [
373 "sourceId",
374 "visit",
375 "detector",
376 "ra",
377 "dec",
378 "x",
379 "y",
380 "physical_filter",
381 "band",
382 "obj_index",
383 self.config.instFluxField,
384 self.config.instFluxField + "Err",
385 self.config.apertureInnerInstFluxField,
386 self.config.apertureInnerInstFluxField + "Err",
387 self.config.apertureOuterInstFluxField,
388 self.config.apertureOuterInstFluxField + "Err",
389 ]
390
391 if self.config.doSubtractLocalBackground:
392 source_columns.append(self.config.localBackgroundFluxField)
393 local_background_flag_name = self.config.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
394 source_columns.append(local_background_flag_name)
395
396 if self.sourceSelector.config.doFlags:
397 source_columns.extend(self.sourceSelector.config.flags.bad)
398
399 local_background_area = np.pi*calib_flux_aperture_radius**2.
400
401 # Compute the approximate pixel area over the full focal plane
402 # from the WCS jacobian using the camera model.
403 approx_pixel_area_fields = computeApproxPixelAreaFields(camera)
404
405 # Construct mapping from detector number to index.
406 detector_mapping = {}
407 for detector_index, detector in enumerate(camera):
408 detector_mapping[detector.getId()] = detector_index
409
410 star_obs_dtype = [
411 ("ra", "f8"),
412 ("dec", "f8"),
413 ("x", "f8"),
414 ("y", "f8"),
415 ("visit", "i8"),
416 ("detector", "i4"),
417 ("inst_mag", "f4"),
418 ("inst_mag_err", "f4"),
419 ("jacobian", "f4"),
420 ("delta_mag_bkg", "f4"),
421 ("delta_mag_aper", "f4"),
422 ("delta_mag_err_aper", "f4"),
423 ]
424
425 fgcm_obj_dtype = [
426 ("fgcm_id", "i8"),
427 ("isolated_star_id", "i8"),
428 ("ra", "f8"),
429 ("dec", "f8"),
430 ("obs_arr_index", "i4"),
431 ("n_obs", "i4"),
432 ("obj_flag", "i4"),
433 ]
434
435 fgcm_objs = []
436 star_obs_cats = []
437 merge_source_counter = 0
438
439 k = 2.5/np.log(10.)
440
441 visit_cat_table = visit_cat.asAstropy()
442
443 for tract in sorted(isolated_star_cat_handle_dict):
444 stars = isolated_star_cat_handle_dict[tract].get()
445 sources = isolated_star_source_handle_dict[tract].get(parameters={"columns": source_columns})
446
447 # Down-select sources.
448 good_sources = self.sourceSelector.selectSources(sources).selected
449 if self.config.doSubtractLocalBackground:
450 good_sources &= (~sources[local_background_flag_name])
451 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
452 good_sources &= ((sources[self.config.instFluxField] - local_background) > 0)
453
454 if good_sources.sum() == 0:
455 self.log.info("No good sources found in tract %d", tract)
456 continue
457
458 # Need to count the observations of each star after cuts, per band.
459 # If we have requiredBands specified, we must make sure that each star
460 # has the minumum number of observations in each of thos bands.
461 # Otherwise, we must make sure that each star has at least the minimum
462 # number of observations in _any_ band.
463 if len(self.config.requiredBands) > 0:
464 loop_bands = self.config.requiredBands
465 else:
466 loop_bands = np.unique(sources["band"])
467
468 n_req = np.zeros((len(loop_bands), len(stars)), dtype=np.int32)
469 for i, band in enumerate(loop_bands):
470 (band_use,) = (sources[good_sources]["band"] == band).nonzero()
471 np.add.at(
472 n_req,
473 (i, sources[good_sources]["obj_index"][band_use]),
474 1,
475 )
476
477 if len(self.config.requiredBands) > 0:
478 # The min gives us the band with the fewest observations, which must be
479 # above the limit.
480 (good_stars,) = (n_req.min(axis=0) >= self.config.minPerBand).nonzero()
481 else:
482 # The max gives us the band with the most observations, which must be
483 # above the limit.
484 (good_stars,) = (n_req.max(axis=0) >= self.config.minPerBand).nonzero()
485
486 # With the following matching:
487 # sources[good_sources][b] <-> stars[good_stars[a]]
488 obj_index = sources["obj_index"][good_sources]
489 a, b = esutil.numpy_util.match(good_stars, obj_index)
490
491 # Update indexes and cut to selected stars/sources
492 _, index_new = np.unique(a, return_index=True)
493 stars["source_cat_index"][good_stars] = index_new
494 sources = sources[good_sources][b]
495 sources["obj_index"][:] = a
496 stars = stars[good_stars]
497
498 nsource = np.zeros(len(stars), dtype=np.int32)
499 np.add.at(
500 nsource,
501 sources["obj_index"],
502 1,
503 )
504 stars["nsource"][:] = nsource
505
506 # After these cuts, the catalogs have the following properties:
507 # - ``stars`` only contains isolated stars that have the minimum number of good
508 # sources in the required bands.
509 # - ``sources`` has been cut to the good sources.
510 # - The slice [stars["source_cat_index"]: stars["source_cat_index"]
511 # + stars["nsource"]]
512 # applied to ``sources`` will give all the sources associated with the star.
513 # - For each source, sources["obj_index"] points to the index of the associated
514 # isolated star.
515
516 # We now reformat the sources and compute the ``observations`` that fgcm expects.
517 star_obs = Table(data=np.zeros(len(sources), dtype=star_obs_dtype))
518 star_obs["ra"] = sources["ra"]
519 star_obs["dec"] = sources["dec"]
520 star_obs["x"] = sources["x"]
521 star_obs["y"] = sources["y"]
522 star_obs["visit"] = sources["visit"]
523 star_obs["detector"] = sources["detector"]
524
525 visit_match, obs_match = esutil.numpy_util.match(visit_cat_table["visit"], sources["visit"])
526
527 exp_time = np.zeros(len(star_obs))
528 exp_time[obs_match] = visit_cat_table["exptime"][visit_match]
529
530 with warnings.catch_warnings():
531 # Ignore warnings, we will filter infinities and nans below.
532 warnings.simplefilter("ignore")
533
534 inst_mag_inner = -2.5*np.log10(sources[self.config.apertureInnerInstFluxField])
535 inst_mag_err_inner = k*(sources[self.config.apertureInnerInstFluxField + "Err"]
536 / sources[self.config.apertureInnerInstFluxField])
537 inst_mag_outer = -2.5*np.log10(sources[self.config.apertureOuterInstFluxField])
538 inst_mag_err_outer = k*(sources[self.config.apertureOuterInstFluxField + "Err"]
539 / sources[self.config.apertureOuterInstFluxField])
540 star_obs["delta_mag_aper"] = inst_mag_inner - inst_mag_outer
541 star_obs["delta_mag_err_aper"] = np.sqrt(inst_mag_err_inner**2. + inst_mag_err_outer**2.)
542 # Set bad values to sentinel values for fgcm.
543 bad = ~np.isfinite(star_obs["delta_mag_aper"])
544 star_obs["delta_mag_aper"][bad] = 99.0
545 star_obs["delta_mag_err_aper"][bad] = 99.0
546
547 if self.config.doSubtractLocalBackground:
548 # At the moment we only adjust the flux and not the flux
549 # error by the background because the error on
550 # base_LocalBackground_instFlux is the rms error in the
551 # background annulus, not the error on the mean in the
552 # background estimate (which is much smaller, by sqrt(n)
553 # pixels used to estimate the background, which we do not
554 # have access to in this task). In the default settings,
555 # the annulus is sufficiently large such that these
556 # additional errors are negligibly small (much less
557 # than a mmag in quadrature).
558
559 # This is the difference between the mag with local background correction
560 # and the mag without local background correction.
561 local_background = local_background_area*sources[self.config.localBackgroundFluxField]
562 star_obs["delta_mag_bkg"] = (-2.5*np.log10(sources[self.config.instFluxField]
563 - local_background) -
564 -2.5*np.log10(sources[self.config.instFluxField]))
565
566 # Need to loop over detectors here.
567 for detector in camera:
568 detector_id = detector.getId()
569 # used index for all observations with a given detector
570 (use,) = (star_obs["detector"][obs_match] == detector_id).nonzero()
571 # Prior to running the calibration, we want to remove the effect
572 # of the jacobian of the WCS because that is a known quantity.
573 # Ideally, this would be done for each individual WCS, but this
574 # is extremely slow and makes small differences that are much
575 # smaller than the variation in the throughput due to other issues.
576 # Therefore, we use the approximate jacobian estimated from the
577 # camera model.
578 jac = approx_pixel_area_fields[detector_id].evaluate(
579 star_obs["x"][obs_match][use],
580 star_obs["y"][obs_match][use],
581 )
582 star_obs["jacobian"][obs_match[use]] = jac
583 scaled_inst_flux = (sources[self.config.instFluxField][obs_match[use]]
584 * visit_cat_table["scaling"][visit_match[use],
585 detector_mapping[detector_id]])
586 star_obs["inst_mag"][obs_match[use]] = (-2.5 * np.log10(scaled_inst_flux
587 / exp_time[use]))
588
589 # Compute instMagErr from inst_flux_err/inst_flux; scaling will cancel out.
590 star_obs["inst_mag_err"] = k*(sources[self.config.instFluxField + "Err"]
591 / sources[self.config.instFluxField])
592
593 # Apply the jacobian if configured to do so.
594 if self.config.doApplyWcsJacobian:
595 star_obs["inst_mag"] -= 2.5*np.log10(star_obs["jacobian"])
596
597 # We now reformat the stars and compute the ''objects'' that fgcm expects.
598 fgcm_obj = Table(data=np.zeros(len(stars), dtype=fgcm_obj_dtype))
599 fgcm_obj["isolated_star_id"] = stars["isolated_star_id"]
600 fgcm_obj["ra"] = stars["ra"]
601 fgcm_obj["dec"] = stars["dec"]
602 fgcm_obj["obs_arr_index"] = stars["source_cat_index"]
603 fgcm_obj["n_obs"] = stars["nsource"]
604
605 # Offset indexes to account for tract merging
606 fgcm_obj["obs_arr_index"] += merge_source_counter
607
608 fgcm_objs.append(fgcm_obj)
609 star_obs_cats.append(star_obs)
610
611 merge_source_counter += len(star_obs)
612
613 fgcm_obj = vstack(fgcm_objs)
614
615 # Set the fgcm_id to a unique 64-bit integer for easier sorting.
616 fgcm_obj["fgcm_id"][:] = np.arange(len(fgcm_obj)) + 1
617
618 return fgcm_obj, vstack(star_obs_cats)
619
620 def _density_downsample(self, fgcm_obj, star_obs):
621 """Downsample stars according to density.
622
623 Catalogs are modified in-place.
624
625 Parameters
626 ----------
627 fgcm_obj : `astropy.table.Table`
628 Catalog of per-star ids and positions.
629 star_obs : `astropy.table.Table`
630 Catalog of individual star observations.
631 """
632 if self.config.randomSeed is not None:
633 np.random.seed(seed=self.config.randomSeed)
634
635 ipnest = hpg.angle_to_pixel(self.config.densityCutNside, fgcm_obj["ra"], fgcm_obj["dec"])
636 # Use the esutil.stat.histogram function to pull out the histogram of
637 # grouped pixels, and the rev_indices which describes which inputs
638 # are grouped together.
639 hist, rev_indices = esutil.stat.histogram(ipnest, rev=True)
640
641 obj_use = np.ones(len(fgcm_obj), dtype=bool)
642
643 (high,) = (hist > self.config.densityCutMaxPerPixel).nonzero()
644 (ok,) = (hist > 0).nonzero()
645 self.log.info("There are %d/%d pixels with high stellar density.", high.size, ok.size)
646 for i in range(high.size):
647 # The pix_indices are the indices of every star in the pixel.
648 pix_indices = rev_indices[rev_indices[high[i]]: rev_indices[high[i] + 1]]
649 # Cut down to the maximum number of stars in the pixel.
650 cut = np.random.choice(
651 pix_indices,
652 size=pix_indices.size - self.config.densityCutMaxPerPixel,
653 replace=False,
654 )
655 obj_use[cut] = False
656
657 fgcm_obj = fgcm_obj[obj_use]
658
659 obs_index = np.zeros(np.sum(fgcm_obj["n_obs"]), dtype=np.int32)
660 ctr = 0
661 for i in range(len(fgcm_obj)):
662 n_obs = fgcm_obj["n_obs"][i]
663 obs_index[ctr: ctr + n_obs] = (
664 np.arange(fgcm_obj["obs_arr_index"][i], fgcm_obj["obs_arr_index"][i] + n_obs)
665 )
666 fgcm_obj["obs_arr_index"][i] = ctr
667 ctr += n_obs
668
669 star_obs = star_obs[obs_index]
670
671 def _mark_reserve_stars(self, fgcm_obj):
672 """Run the star reservation task to flag reserved stars.
673
674 Parameters
675 ----------
676 fgcm_obj : `astropy.table.Table`
677 Catalog of per-star ids and positions.
678 """
679 reserved = self.reserve_selection.run(
680 len(fgcm_obj),
681 extra=','.join(self.config.requiredBands),
682 )
683 fgcm_obj["obj_flag"][reserved] |= objFlagDict["RESERVED"]
684
685 def _associate_reference_stars(self, lookup_table_handle, visit_cat, fgcm_obj):
686 """Associate fgcm isolated stars with reference stars.
687
688 Parameters
689 ----------
690 lookup_table_handle : `lsst.daf.butler.DeferredDatasetHandle`, optional
691 Data reference to fgcm look-up table (used if matching reference stars).
692 visit_cat : `lsst.afw.table.ExposureCatalog`
693 Catalog of per-visit data.
694 fgcm_obj : `astropy.table.Table`
695 Catalog of per-star ids and positions
696
697 Returns
698 -------
699 ref_cat : `astropy.table.Table`
700 Catalog of reference stars matched to fgcm stars.
701 """
702 # Figure out the correct bands/filters that we need based on the data.
703 lut_cat = lookup_table_handle.get()
704
705 std_filter_dict = {filter_name: std_filter for (filter_name, std_filter) in
706 zip(lut_cat[0]["physicalFilters"].split(","),
707 lut_cat[0]["stdPhysicalFilters"].split(","))}
708 std_lambda_dict = {std_filter: std_lambda for (std_filter, std_lambda) in
709 zip(lut_cat[0]["stdPhysicalFilters"].split(","),
710 lut_cat[0]["lambdaStdFilter"])}
711 del lut_cat
712
713 reference_filter_names = self._getReferenceFilterNames(
714 visit_cat,
715 std_filter_dict,
716 std_lambda_dict,
717 )
718 self.log.info("Using the following reference filters: %s", (", ".join(reference_filter_names)))
719
720 # Split into healpix pixels for more efficient matching.
721 ipnest = hpg.angle_to_pixel(self.config.coarseNside, fgcm_obj["ra"], fgcm_obj["dec"])
722 hpix, revpix = esutil.stat.histogram(ipnest, rev=True)
723
724 pixel_cats = []
725
726 # Compute the dtype from the filter names.
727 dtype = [("fgcm_id", "i4"),
728 ("refMag", "f4", (len(reference_filter_names), )),
729 ("refMagErr", "f4", (len(reference_filter_names), ))]
730
731 (gdpix,) = (hpix > 0).nonzero()
732 for ii, gpix in enumerate(gdpix):
733 p1a = revpix[revpix[gpix]: revpix[gpix + 1]]
734
735 # We do a simple wrapping of RA if we need to.
736 ra_wrap = fgcm_obj["ra"][p1a]
737 if (ra_wrap.min() < 10.0) and (ra_wrap.max() > 350.0):
738 ra_wrap[ra_wrap > 180.0] -= 360.0
739 mean_ra = np.mean(ra_wrap) % 360.0
740 else:
741 mean_ra = np.mean(ra_wrap)
742 mean_dec = np.mean(fgcm_obj["dec"][p1a])
743
744 dist = esutil.coords.sphdist(mean_ra, mean_dec, fgcm_obj["ra"][p1a], fgcm_obj["dec"][p1a])
745 rad = dist.max()
746
747 if rad < hpg.nside_to_resolution(self.config.coarseNside)/2.:
748 # Small radius, read just the circle.
749 ref_cat = self.fgcmLoadReferenceCatalog.getFgcmReferenceStarsSkyCircle(
750 mean_ra,
751 mean_dec,
752 rad,
753 reference_filter_names,
754 )
755 else:
756 # Otherwise the less efficient but full coverage.
757 ref_cat = self.fgcmLoadReferenceCatalog.getFgcmReferenceStarsHealpix(
758 self.config.coarseNside,
759 hpg.nest_to_ring(self.config.coarseNside, ipnest[p1a[0]]),
760 reference_filter_names,
761 )
762 if ref_cat.size == 0:
763 # No stars in this pixel; that's okay.
764 continue
765
766 with Matcher(fgcm_obj["ra"][p1a], fgcm_obj["dec"][p1a]) as matcher:
767 idx, i1, i2, d = matcher.query_radius(
768 ref_cat["ra"],
769 ref_cat["dec"],
770 self.config.matchRadius/3600.,
771 return_indices=True,
772 )
773
774 if len(i1) == 0:
775 # No matched stars in this pixel; that's okay.
776 continue
777
778 pixel_cat = Table(data=np.zeros(i1.size, dtype=dtype))
779 pixel_cat["fgcm_id"] = fgcm_obj["fgcm_id"][p1a[i1]]
780 pixel_cat["refMag"][:, :] = ref_cat["refMag"][i2, :]
781 pixel_cat["refMagErr"][:, :] = ref_cat["refMagErr"][i2, :]
782
783 pixel_cats.append(pixel_cat)
784
785 self.log.info(
786 "Found %d reference matches in pixel %d (%d of %d).",
787 len(pixel_cat),
788 ipnest[p1a[0]],
789 ii,
790 gdpix.size - 1,
791 )
792
793 ref_cat_full = vstack(pixel_cats)
794 ref_cat_full.meta.update({'FILTERNAMES': reference_filter_names})
795
796 return ref_cat_full
797
798 def _compute_delta_aper_summary_statistics(self, visit_cat, star_obs):
799 """Compute delta aperture summary statistics (per visit).
800
801 Parameters
802 ----------
803 visit_cat : `lsst.afw.table.ExposureCatalog`
804 Catalog of per-visit data.
805 star_obs : `astropy.table.Table`
806 Catalog of individual star observations.
807 """
808 (ok,) = ((star_obs["delta_mag_aper"] < 99.0)
809 & (star_obs["delta_mag_err_aper"] < 99.0)).nonzero()
810
811 visit_index = np.zeros(len(star_obs[ok]), dtype=np.int32)
812 a, b = esutil.numpy_util.match(visit_cat["visit"], star_obs["visit"][ok])
813 visit_index[b] = a
814
815 h, rev = esutil.stat.histogram(visit_index, rev=True)
816
817 visit_cat["deltaAper"] = -9999.0
818 h_use, = np.where(h >= 3)
819 for index in h_use:
820 i1a = rev[rev[index]: rev[index + 1]]
821 visit_cat["deltaAper"][index] = np.median(star_obs["delta_mag_aper"][ok[i1a]])
run(self, *camera, visit_summary_handle_dict, isolated_star_cat_handle_dict, isolated_star_source_handle_dict, lookup_table_handle=None)
_make_all_star_obs_from_isolated_stars(self, isolated_star_cat_handle_dict, isolated_star_source_handle_dict, visit_cat, camera, calib_flux_aperture_radius=None)
_getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)