lsst.fgcmcal  22.0.1-9-ga0ac561+381b27b2e4
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 import lsst.afw.table as afwTable
35 import lsst.geom as geom
36 from lsst.daf.base import PropertyList
37 from lsst.daf.base.dateTime import DateTime
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .utilities import computeApertureRadiusFromDataRef
41 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42 
43 import fgcm
44 
45 REFSTARS_FORMAT_VERSION = 1
46 
47 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48 
49 
50 class FgcmBuildStarsConfigBase(pexConfig.Config):
51  """Base config for FgcmBuildStars tasks"""
52 
53  instFluxField = pexConfig.Field(
54  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55  "The associated flag will be implicitly included in badFlags"),
56  dtype=str,
57  default='slot_CalibFlux_instFlux',
58  )
59  minPerBand = pexConfig.Field(
60  doc="Minimum observations per band",
61  dtype=int,
62  default=2,
63  )
64  matchRadius = pexConfig.Field(
65  doc="Match radius (arcseconds)",
66  dtype=float,
67  default=1.0,
68  )
69  isolationRadius = pexConfig.Field(
70  doc="Isolation radius (arcseconds)",
71  dtype=float,
72  default=2.0,
73  )
74  densityCutNside = pexConfig.Field(
75  doc="Density cut healpix nside",
76  dtype=int,
77  default=128,
78  )
79  densityCutMaxPerPixel = pexConfig.Field(
80  doc="Density cut number of stars per pixel",
81  dtype=int,
82  default=1000,
83  )
84  randomSeed = pexConfig.Field(
85  doc="Random seed for high density down-sampling.",
86  dtype=int,
87  default=None,
88  optional=True,
89  )
90  matchNside = pexConfig.Field(
91  doc="Healpix Nside for matching",
92  dtype=int,
93  default=4096,
94  )
95  coarseNside = pexConfig.Field(
96  doc="Healpix coarse Nside for partitioning matches",
97  dtype=int,
98  default=8,
99  )
100  # The following config will not be necessary after Gen2 retirement.
101  # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
102  # which is easiest to access in the config file.
103  physicalFilterMap = pexConfig.DictField(
104  doc="Mapping from 'physicalFilter' to band.",
105  keytype=str,
106  itemtype=str,
107  default={},
108  )
109  requiredBands = pexConfig.ListField(
110  doc="Bands required for each star",
111  dtype=str,
112  default=(),
113  )
114  primaryBands = pexConfig.ListField(
115  doc=("Bands for 'primary' star matches. "
116  "A star must be observed in one of these bands to be considered "
117  "as a calibration star."),
118  dtype=str,
119  default=None
120  )
121  visitDataRefName = pexConfig.Field(
122  doc="dataRef name for the 'visit' field, usually 'visit'.",
123  dtype=str,
124  default="visit"
125  )
126  ccdDataRefName = pexConfig.Field(
127  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
128  dtype=str,
129  default="ccd"
130  )
131  doApplyWcsJacobian = pexConfig.Field(
132  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
133  dtype=bool,
134  default=True
135  )
136  doModelErrorsWithBackground = pexConfig.Field(
137  doc="Model flux errors with background term?",
138  dtype=bool,
139  default=True
140  )
141  psfCandidateName = pexConfig.Field(
142  doc="Name of field with psf candidate flag for propagation",
143  dtype=str,
144  default="calib_psf_candidate"
145  )
146  doSubtractLocalBackground = pexConfig.Field(
147  doc=("Subtract the local background before performing calibration? "
148  "This is only supported for circular aperture calibration fluxes."),
149  dtype=bool,
150  default=False
151  )
152  localBackgroundFluxField = pexConfig.Field(
153  doc="Full name of the local background instFlux field to use.",
154  dtype=str,
155  default='base_LocalBackground_instFlux'
156  )
157  sourceSelector = sourceSelectorRegistry.makeField(
158  doc="How to select sources",
159  default="science"
160  )
161  apertureInnerInstFluxField = pexConfig.Field(
162  doc=("Full name of instFlux field that contains inner aperture "
163  "flux for aperture correction proxy"),
164  dtype=str,
165  default='base_CircularApertureFlux_12_0_instFlux'
166  )
167  apertureOuterInstFluxField = pexConfig.Field(
168  doc=("Full name of instFlux field that contains outer aperture "
169  "flux for aperture correction proxy"),
170  dtype=str,
171  default='base_CircularApertureFlux_17_0_instFlux'
172  )
173  doReferenceMatches = pexConfig.Field(
174  doc="Match reference catalog as additional constraint on calibration",
175  dtype=bool,
176  default=True,
177  )
178  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
179  target=FgcmLoadReferenceCatalogTask,
180  doc="FGCM reference object loader",
181  )
182  nVisitsPerCheckpoint = pexConfig.Field(
183  doc="Number of visits read between checkpoints",
184  dtype=int,
185  default=500,
186  )
187 
188  def setDefaults(self):
189  sourceSelector = self.sourceSelectorsourceSelector["science"]
190  sourceSelector.setDefaults()
191 
192  sourceSelector.doFlags = True
193  sourceSelector.doUnresolved = True
194  sourceSelector.doSignalToNoise = True
195  sourceSelector.doIsolated = True
196 
197  sourceSelector.signalToNoise.minimum = 10.0
198  sourceSelector.signalToNoise.maximum = 1000.0
199 
200  # FGCM operates on unresolved sources, and this setting is
201  # appropriate for the current base_ClassificationExtendedness
202  sourceSelector.unresolved.maximum = 0.5
203 
204 
205 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
206  """Subclass of TaskRunner for FgcmBuildStars tasks
207 
208  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
209  arguments, one of which is the butler (for persistence and mapper data),
210  and a list of dataRefs extracted from the command line. Note that FGCM
211  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
212  This class transforms the process arguments generated by the ArgumentParser
213  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
214  not use any parallelization.
215  """
216  @staticmethod
217  def getTargetList(parsedCmd):
218  """
219  Return a list with one element: a tuple with the butler and
220  list of dataRefs
221  """
222  # we want to combine the butler with any (or no!) dataRefs
223  return [(parsedCmd.butler, parsedCmd.id.refList)]
224 
225  def __call__(self, args):
226  """
227  Parameters
228  ----------
229  args: `tuple` with (butler, dataRefList)
230 
231  Returns
232  -------
233  exitStatus: `list` with `lsst.pipe.base.Struct`
234  exitStatus (0: success; 1: failure)
235  """
236  butler, dataRefList = args
237 
238  task = self.TaskClass(config=self.config, log=self.log)
239 
240  exitStatus = 0
241  if self.doRaise:
242  task.runDataRef(butler, dataRefList)
243  else:
244  try:
245  task.runDataRef(butler, dataRefList)
246  except Exception as e:
247  exitStatus = 1
248  task.log.fatal("Failed: %s" % e)
249  if not isinstance(e, pipeBase.TaskError):
250  traceback.print_exc(file=sys.stderr)
251 
252  task.writeMetadata(butler)
253 
254  # The task does not return any results:
255  return [pipeBase.Struct(exitStatus=exitStatus)]
256 
257  def run(self, parsedCmd):
258  """
259  Run the task, with no multiprocessing
260 
261  Parameters
262  ----------
263  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
264  """
265 
266  resultList = []
267 
268  if self.precall(parsedCmd):
269  targetList = self.getTargetListgetTargetList(parsedCmd)
270  resultList = self(targetList[0])
271 
272  return resultList
273 
274 
275 class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
276  """
277  Base task to build stars for FGCM global calibration
278 
279  Parameters
280  ----------
281  butler : `lsst.daf.persistence.Butler`
282  """
283  def __init__(self, initInputs=None, butler=None, **kwargs):
284  super().__init__(**kwargs)
285 
286  self.makeSubtask("sourceSelector")
287  # Only log warning and fatal errors from the sourceSelector
288  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
289 
290  # no saving of metadata for now
291  def _getMetadataName(self):
292  return None
293 
294  @pipeBase.timeMethod
295  def runDataRef(self, butler, dataRefs):
296  """
297  Cross-match and make star list for FGCM Input
298 
299  Parameters
300  ----------
301  butler: `lsst.daf.persistence.Butler`
302  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
303  Source data references for the input visits.
304 
305  Raises
306  ------
307  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
308  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
309  fails if the calibFlux is not a CircularAperture flux.
310  """
311  datasetType = dataRefs[0].butlerSubset.datasetType
312  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
313 
314  if self.config.doReferenceMatches:
315  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
316  # Ensure that we have a LUT
317  if not butler.datasetExists('fgcmLookUpTable'):
318  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
319  # Compute aperture radius if necessary. This is useful to do now before
320  # any heavy lifting has happened (fail early).
321  calibFluxApertureRadius = None
322  if self.config.doSubtractLocalBackground:
323  try:
324  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
325  self.config.instFluxField)
326  except RuntimeError as e:
327  raise RuntimeError("Could not determine aperture radius from %s. "
328  "Cannot use doSubtractLocalBackground." %
329  (self.config.instFluxField)) from e
330 
331  camera = butler.get('camera')
332  groupedDataRefs = self._findAndGroupDataRefsGen2_findAndGroupDataRefsGen2(butler, camera, dataRefs)
333 
334  # Make the visit catalog if necessary
335  # First check if the visit catalog is in the _current_ path
336  # We cannot use Gen2 datasetExists() because that checks all parent
337  # directories as well, which would make recovering from faults
338  # and fgcmcal reruns impossible.
339  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
340  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
341  if os.path.exists(filename):
342  # This file exists and we should continue processing
343  inVisitCat = visitCatDataRef.get()
344  if len(inVisitCat) != len(groupedDataRefs):
345  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
346  "number of visits. Cannot continue.")
347  else:
348  inVisitCat = None
349 
350  visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
351  visitCatDataRef=visitCatDataRef,
352  inVisitCat=inVisitCat)
353 
354  # Persist the visitCat as a checkpoint file.
355  visitCatDataRef.put(visitCat)
356 
357  starObsDataRef = butler.dataRef('fgcmStarObservations')
358  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
359  if os.path.exists(filename):
360  inStarObsCat = starObsDataRef.get()
361  else:
362  inStarObsCat = None
363 
364  rad = calibFluxApertureRadius
365  sourceSchemaDataRef = butler.dataRef('src_schema')
366  sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
367  fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
368  visitCat,
369  sourceSchema,
370  camera,
371  calibFluxApertureRadius=rad,
372  starObsDataRef=starObsDataRef,
373  visitCatDataRef=visitCatDataRef,
374  inStarObsCat=inStarObsCat)
375  visitCatDataRef.put(visitCat)
376  starObsDataRef.put(fgcmStarObservationCat)
377 
378  # Always do the matching.
379  if self.config.doReferenceMatches:
380  lutDataRef = butler.dataRef('fgcmLookUpTable')
381  else:
382  lutDataRef = None
383  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
384  fgcmStarObservationCat,
385  lutDataRef=lutDataRef)
386 
387  # Persist catalogs via the butler
388  butler.put(fgcmStarIdCat, 'fgcmStarIds')
389  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
390  if fgcmRefCat is not None:
391  butler.put(fgcmRefCat, 'fgcmReferenceStars')
392 
393  @abc.abstractmethod
394  def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
395  """
396  Find and group dataRefs (by visit); Gen2 only.
397 
398  Parameters
399  ----------
400  butler : `lsst.daf.persistence.Butler`
401  Gen2 butler.
402  camera : `lsst.afw.cameraGeom.Camera`
403  Camera from the butler.
404  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
405  Data references for the input visits.
406 
407  Returns
408  -------
409  groupedDataRefs : `dict` [`int`, `list`]
410  Dictionary with sorted visit keys, and `list`s of
411  `lsst.daf.persistence.ButlerDataRef`
412  """
413  raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
414 
415  @abc.abstractmethod
416  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
417  sourceSchema,
418  camera,
419  calibFluxApertureRadius=None,
420  visitCatDataRef=None,
421  starObsDataRef=None,
422  inStarObsCat=None):
423  """
424  Compile all good star observations from visits in visitCat. Checkpoint files
425  will be stored if both visitCatDataRef and starObsDataRef are not None.
426 
427  Parameters
428  ----------
429  groupedDataRefs : `dict` of `list`s
430  Lists of `~lsst.daf.persistence.ButlerDataRef` or
431  `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
432  visitCat : `~afw.table.BaseCatalog`
433  Catalog with visit data for FGCM
434  sourceSchema : `~lsst.afw.table.Schema`
435  Schema for the input src catalogs.
436  camera : `~lsst.afw.cameraGeom.Camera`
437  calibFluxApertureRadius : `float`, optional
438  Aperture radius for calibration flux.
439  visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
440  Dataref to write visitCat for checkpoints
441  starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
442  Dataref to write the star observation catalog for checkpoints.
443  inStarObsCat : `~afw.table.BaseCatalog`
444  Input observation catalog. If this is incomplete, observations
445  will be appended from when it was cut off.
446 
447  Returns
448  -------
449  fgcmStarObservations : `afw.table.BaseCatalog`
450  Full catalog of good observations.
451 
452  Raises
453  ------
454  RuntimeError: Raised if doSubtractLocalBackground is True and
455  calibFluxApertureRadius is not set.
456  """
457  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
458 
459  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
460  visitCatDataRef=None, inVisitCat=None):
461  """
462  Make a visit catalog with all the keys from each visit
463 
464  Parameters
465  ----------
466  camera: `lsst.afw.cameraGeom.Camera`
467  Camera from the butler
468  groupedDataRefs: `dict`
469  Dictionary with visit keys, and `list`s of
470  `lsst.daf.persistence.ButlerDataRef`
471  bkgDataRefDict: `dict`, optional
472  Dictionary of gen3 dataRefHandles for background info.
473  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
474  Dataref to write visitCat for checkpoints
475  inVisitCat: `afw.table.BaseCatalog`, optional
476  Input (possibly incomplete) visit catalog
477 
478  Returns
479  -------
480  visitCat: `afw.table.BaseCatalog`
481  """
482 
483  self.log.info("Assembling visitCatalog from %d %ss" %
484  (len(groupedDataRefs), self.config.visitDataRefName))
485 
486  nCcd = len(camera)
487 
488  if inVisitCat is None:
489  schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
490 
491  visitCat = afwTable.BaseCatalog(schema)
492  visitCat.reserve(len(groupedDataRefs))
493  visitCat.resize(len(groupedDataRefs))
494 
495  visitCat['visit'] = list(groupedDataRefs.keys())
496  visitCat['used'] = 0
497  visitCat['sources_read'] = False
498  else:
499  visitCat = inVisitCat
500 
501  # No matter what, fill the catalog. This will check if it was
502  # already read.
503  self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
504  bkgDataRefDict=bkgDataRefDict,
505  visitCatDataRef=visitCatDataRef)
506 
507  return visitCat
508 
509  def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
510  visitCatDataRef=None):
511  """
512  Fill the visit catalog with visit metadata
513 
514  Parameters
515  ----------
516  visitCat : `afw.table.BaseCatalog`
517  Visit catalog. See _makeFgcmVisitSchema() for schema definition.
518  groupedDataRefs : `dict`
519  Dictionary with visit keys, and `list`s of
520  `lsst.daf.persistence.ButlerDataRef` or
521  `lsst.daf.butler.DeferredDatasetHandle`
522  visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional
523  Dataref to write ``visitCat`` for checkpoints. Gen2 only.
524  bkgDataRefDict : `dict`, optional
525  Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle`
526  for background info.
527  """
528  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
529 
530  for i, visit in enumerate(groupedDataRefs):
531  # We don't use the bypasses since we need the psf info which does
532  # not have a bypass
533  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
534  # can be fixed
535 
536  # Do not read those that have already been read
537  if visitCat['used'][i]:
538  continue
539 
540  if (i % self.config.nVisitsPerCheckpoint) == 0:
541  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
542  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
543  # Save checkpoint if desired
544  if visitCatDataRef is not None:
545  visitCatDataRef.put(visitCat)
546 
547  dataRef = groupedDataRefs[visit][0]
548  if isinstance(dataRef, dafPersist.ButlerDataRef):
549  # Gen2: calexp dataRef
550  # The first dataRef in the group will be the reference ccd (if available)
551  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
552  visitInfo = exp.getInfo().getVisitInfo()
553  label = dataRef.get(datasetType='calexp_filterLabel')
554  physicalFilter = label.physicalLabel
555  psf = exp.getPsf()
556  psfSigma = psf.computeShape().getDeterminantRadius()
557  else:
558  # Gen3: use the visitSummary dataRef
559  summary = dataRef.get()
560 
561  summaryRow = summary.find(self.config.referenceCCD)
562  if summaryRow is None:
563  # Take the first available ccd if reference isn't available
564  summaryRow = summary[0]
565 
566  summaryDetector = summaryRow['id']
567  visitInfo = summaryRow.getVisitInfo()
568  physicalFilter = summaryRow['physical_filter']
569  # Compute the median psf sigma if possible
570  goodSigma, = np.where(summary['psfSigma'] > 0)
571  if goodSigma.size > 2:
572  psfSigma = np.median(summary['psfSigma'][goodSigma])
573  elif goodSigma > 0:
574  psfSigma = np.mean(summary['psfSigma'][goodSigma])
575  else:
576  psfSigma = 0.0
577 
578  rec = visitCat[i]
579  rec['visit'] = visit
580  rec['physicalFilter'] = physicalFilter
581  # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
582  # much easier to get the wcs's necessary to recompute the pointing
583  # ra/dec at the center of the camera.
584  radec = visitInfo.getBoresightRaDec()
585  rec['telra'] = radec.getRa().asDegrees()
586  rec['teldec'] = radec.getDec().asDegrees()
587  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
588  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
589  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
590  rec['exptime'] = visitInfo.getExposureTime()
591  # convert from Pa to millibar
592  # Note that I don't know if this unit will need to be per-camera config
593  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
594  # Flag to signify if this is a "deep" field. Not currently used
595  rec['deepFlag'] = 0
596  # Relative flat scaling (1.0 means no relative scaling)
597  rec['scaling'][:] = 1.0
598  # Median delta aperture, to be measured from stars
599  rec['deltaAper'] = 0.0
600  rec['psfSigma'] = psfSigma
601 
602  if self.config.doModelErrorsWithBackground:
603  foundBkg = False
604  if isinstance(dataRef, dafPersist.ButlerDataRef):
605  # Gen2-style dataRef
606  det = dataRef.dataId[self.config.ccdDataRefName]
607  if dataRef.datasetExists(datasetType='calexpBackground'):
608  bgList = dataRef.get(datasetType='calexpBackground')
609  foundBkg = True
610  else:
611  # Gen3-style dataRef
612  try:
613  # Use the same detector used from the summary.
614  bkgRef = bkgDataRefDict[(visit, summaryDetector)]
615  bgList = bkgRef.get()
616  foundBkg = True
617  except KeyError:
618  pass
619 
620  if foundBkg:
621  bgStats = (bg[0].getStatsImage().getImage().array
622  for bg in bgList)
623  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
624  else:
625  self.log.warning('Sky background not found for visit %d / ccd %d' %
626  (visit, det))
627  rec['skyBackground'] = -1.0
628  else:
629  rec['skyBackground'] = -1.0
630 
631  rec['used'] = 1
632 
633  def _makeSourceMapper(self, sourceSchema):
634  """
635  Make a schema mapper for fgcm sources
636 
637  Parameters
638  ----------
639  sourceSchema: `afwTable.Schema`
640  Default source schema from the butler
641 
642  Returns
643  -------
644  sourceMapper: `afwTable.schemaMapper`
645  Mapper to the FGCM source schema
646  """
647 
648  # create a mapper to the preferred output
649  sourceMapper = afwTable.SchemaMapper(sourceSchema)
650 
651  # map to ra/dec
652  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
653  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
654  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
655  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
656  # Add the mapping if the field exists in the input catalog.
657  # If the field does not exist, simply add it (set to False).
658  # This field is not required for calibration, but is useful
659  # to collate if available.
660  try:
661  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
662  'psf_candidate')
663  except LookupError:
664  sourceMapper.editOutputSchema().addField(
665  "psf_candidate", type='Flag',
666  doc=("Flag set if the source was a candidate for PSF determination, "
667  "as determined by the star selector."))
668 
669  # and add the fields we want
670  sourceMapper.editOutputSchema().addField(
671  "visit", type=np.int32, doc="Visit number")
672  sourceMapper.editOutputSchema().addField(
673  "ccd", type=np.int32, doc="CCD number")
674  sourceMapper.editOutputSchema().addField(
675  "instMag", type=np.float32, doc="Instrumental magnitude")
676  sourceMapper.editOutputSchema().addField(
677  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
678  sourceMapper.editOutputSchema().addField(
679  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
680  sourceMapper.editOutputSchema().addField(
681  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
682 
683  return sourceMapper
684 
685  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
686  """
687  Use FGCM code to match observations into unique stars.
688 
689  Parameters
690  ----------
691  visitCat: `afw.table.BaseCatalog`
692  Catalog with visit data for fgcm
693  obsCat: `afw.table.BaseCatalog`
694  Full catalog of star observations for fgcm
695  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
696  `lsst.daf.butler.DeferredDatasetHandle`, optional
697  Data reference to fgcm look-up table (used if matching reference stars).
698 
699  Returns
700  -------
701  fgcmStarIdCat: `afw.table.BaseCatalog`
702  Catalog of unique star identifiers and index keys
703  fgcmStarIndicesCat: `afwTable.BaseCatalog`
704  Catalog of unique star indices
705  fgcmRefCat: `afw.table.BaseCatalog`
706  Catalog of matched reference stars.
707  Will be None if `config.doReferenceMatches` is False.
708  """
709  # get filter names into a numpy array...
710  # This is the type that is expected by the fgcm code
711  visitFilterNames = np.zeros(len(visitCat), dtype='a30')
712  for i in range(len(visitCat)):
713  visitFilterNames[i] = visitCat[i]['physicalFilter']
714 
715  # match to put filterNames with observations
716  visitIndex = np.searchsorted(visitCat['visit'],
717  obsCat['visit'])
718 
719  obsFilterNames = visitFilterNames[visitIndex]
720 
721  if self.config.doReferenceMatches:
722  # Get the reference filter names, using the LUT
723  lutCat = lutDataRef.get()
724 
725  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
726  zip(lutCat[0]['physicalFilters'].split(','),
727  lutCat[0]['stdPhysicalFilters'].split(','))}
728  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
729  zip(lutCat[0]['stdPhysicalFilters'].split(','),
730  lutCat[0]['lambdaStdFilter'])}
731 
732  del lutCat
733 
734  referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
735  stdFilterDict,
736  stdLambdaDict)
737  self.log.info("Using the following reference filters: %s" %
738  (', '.join(referenceFilterNames)))
739 
740  else:
741  # This should be an empty list
742  referenceFilterNames = []
743 
744  # make the fgcm starConfig dict
745  starConfig = {'logger': self.log,
746  'filterToBand': self.config.physicalFilterMap,
747  'requiredBands': self.config.requiredBands,
748  'minPerBand': self.config.minPerBand,
749  'matchRadius': self.config.matchRadius,
750  'isolationRadius': self.config.isolationRadius,
751  'matchNSide': self.config.matchNside,
752  'coarseNSide': self.config.coarseNside,
753  'densNSide': self.config.densityCutNside,
754  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
755  'randomSeed': self.config.randomSeed,
756  'primaryBands': self.config.primaryBands,
757  'referenceFilterNames': referenceFilterNames}
758 
759  # initialize the FgcmMakeStars object
760  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
761 
762  # make the primary stars
763  # note that the ra/dec native Angle format is radians
764  # We determine the conversion from the native units (typically
765  # radians) to degrees for the first observation. This allows us
766  # to treate ra/dec as numpy arrays rather than Angles, which would
767  # be approximately 600x slower.
768  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
769  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
770  obsCat['dec'] * conv,
771  filterNameArray=obsFilterNames,
772  bandSelected=False)
773 
774  # and match all the stars
775  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
776  obsCat['dec'] * conv,
777  obsFilterNames)
778 
779  if self.config.doReferenceMatches:
780  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
781 
782  # now persist
783 
784  objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
785 
786  # make catalog and records
787  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
788  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
789  for i in range(fgcmMakeStars.objIndexCat.size):
790  fgcmStarIdCat.addNew()
791 
792  # fill the catalog
793  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
794  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
795  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
796  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
797  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
798 
799  obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
800 
801  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
802  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
803  for i in range(fgcmMakeStars.obsIndexCat.size):
804  fgcmStarIndicesCat.addNew()
805 
806  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
807 
808  if self.config.doReferenceMatches:
809  refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
810 
811  fgcmRefCat = afwTable.BaseCatalog(refSchema)
812  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
813 
814  for i in range(fgcmMakeStars.referenceCat.size):
815  fgcmRefCat.addNew()
816 
817  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
818  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
819  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
820 
821  md = PropertyList()
822  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
823  md.set("FILTERNAMES", referenceFilterNames)
824  fgcmRefCat.setMetadata(md)
825 
826  else:
827  fgcmRefCat = None
828 
829  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
830 
831  def _makeFgcmVisitSchema(self, nCcd):
832  """
833  Make a schema for an fgcmVisitCatalog
834 
835  Parameters
836  ----------
837  nCcd: `int`
838  Number of CCDs in the camera
839 
840  Returns
841  -------
842  schema: `afwTable.Schema`
843  """
844 
845  schema = afwTable.Schema()
846  schema.addField('visit', type=np.int32, doc="Visit number")
847  schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
848  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
849  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
850  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
851  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
852  schema.addField('mjd', type=np.float64, doc="MJD of visit")
853  schema.addField('exptime', type=np.float32, doc="Exposure time")
854  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
855  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
856  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
857  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
858  # the following field is not used yet
859  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
860  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
861  size=nCcd)
862  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
863  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
864 
865  return schema
866 
867  def _makeFgcmObjSchema(self):
868  """
869  Make a schema for the objIndexCat from fgcmMakeStars
870 
871  Returns
872  -------
873  schema: `afwTable.Schema`
874  """
875 
876  objSchema = afwTable.Schema()
877  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
878  # Will investigate making these angles...
879  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
880  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
881  objSchema.addField('obsArrIndex', type=np.int32,
882  doc='Index in obsIndexTable for first observation')
883  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
884 
885  return objSchema
886 
887  def _makeFgcmObsSchema(self):
888  """
889  Make a schema for the obsIndexCat from fgcmMakeStars
890 
891  Returns
892  -------
893  schema: `afwTable.Schema`
894  """
895 
896  obsSchema = afwTable.Schema()
897  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
898 
899  return obsSchema
900 
901  def _makeFgcmRefSchema(self, nReferenceBands):
902  """
903  Make a schema for the referenceCat from fgcmMakeStars
904 
905  Parameters
906  ----------
907  nReferenceBands: `int`
908  Number of reference bands
909 
910  Returns
911  -------
912  schema: `afwTable.Schema`
913  """
914 
915  refSchema = afwTable.Schema()
916  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
917  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
918  size=nReferenceBands)
919  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
920  size=nReferenceBands)
921 
922  return refSchema
923 
924  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
925  """
926  Get the reference filter names, in wavelength order, from the visitCat and
927  information from the look-up-table.
928 
929  Parameters
930  ----------
931  visitCat: `afw.table.BaseCatalog`
932  Catalog with visit data for FGCM
933  stdFilterDict: `dict`
934  Mapping of filterName to stdFilterName from LUT
935  stdLambdaDict: `dict`
936  Mapping of stdFilterName to stdLambda from LUT
937 
938  Returns
939  -------
940  referenceFilterNames: `list`
941  Wavelength-ordered list of reference filter names
942  """
943 
944  # Find the unique list of filter names in visitCat
945  filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
946 
947  # Find the unique list of "standard" filters
948  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
949 
950  # And sort these by wavelength
951  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
952 
953  return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchema, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def __init__(self, initInputs=None, butler=None, **kwargs)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:807