lsst.fgcmcal  21.0.0-7-g6531d7b+dd63a67816
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 import lsst.afw.table as afwTable
35 import lsst.geom as geom
36 from lsst.daf.base import PropertyList
37 from lsst.daf.base.dateTime import DateTime
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .utilities import computeApertureRadiusFromDataRef
41 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42 
43 import fgcm
44 
45 REFSTARS_FORMAT_VERSION = 1
46 
47 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48 
49 
50 class FgcmBuildStarsConfigBase(pexConfig.Config):
51  """Base config for FgcmBuildStars tasks"""
52 
53  instFluxField = pexConfig.Field(
54  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55  "The associated flag will be implicitly included in badFlags"),
56  dtype=str,
57  default='slot_CalibFlux_instFlux',
58  )
59  minPerBand = pexConfig.Field(
60  doc="Minimum observations per band",
61  dtype=int,
62  default=2,
63  )
64  matchRadius = pexConfig.Field(
65  doc="Match radius (arcseconds)",
66  dtype=float,
67  default=1.0,
68  )
69  isolationRadius = pexConfig.Field(
70  doc="Isolation radius (arcseconds)",
71  dtype=float,
72  default=2.0,
73  )
74  densityCutNside = pexConfig.Field(
75  doc="Density cut healpix nside",
76  dtype=int,
77  default=128,
78  )
79  densityCutMaxPerPixel = pexConfig.Field(
80  doc="Density cut number of stars per pixel",
81  dtype=int,
82  default=1000,
83  )
84  matchNside = pexConfig.Field(
85  doc="Healpix Nside for matching",
86  dtype=int,
87  default=4096,
88  )
89  coarseNside = pexConfig.Field(
90  doc="Healpix coarse Nside for partitioning matches",
91  dtype=int,
92  default=8,
93  )
94  filterMap = pexConfig.DictField(
95  doc="Mapping from 'filterName' to band.",
96  keytype=str,
97  itemtype=str,
98  default={},
99  deprecated=("This field is no longer used, and has been deprecated by "
100  "DM-28088. It will be removed after v22. Use "
101  "physicalFilterMap instead.")
102  )
103  # The following config will not be necessary after Gen2 retirement.
104  # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
105  # which is easiest to access in the config file.
106  physicalFilterMap = pexConfig.DictField(
107  doc="Mapping from 'physicalFilter' to band.",
108  keytype=str,
109  itemtype=str,
110  default={},
111  )
112  requiredBands = pexConfig.ListField(
113  doc="Bands required for each star",
114  dtype=str,
115  default=(),
116  )
117  primaryBands = pexConfig.ListField(
118  doc=("Bands for 'primary' star matches. "
119  "A star must be observed in one of these bands to be considered "
120  "as a calibration star."),
121  dtype=str,
122  default=None
123  )
124  visitDataRefName = pexConfig.Field(
125  doc="dataRef name for the 'visit' field, usually 'visit'.",
126  dtype=str,
127  default="visit"
128  )
129  ccdDataRefName = pexConfig.Field(
130  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
131  dtype=str,
132  default="ccd"
133  )
134  doApplyWcsJacobian = pexConfig.Field(
135  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
136  dtype=bool,
137  default=True
138  )
139  doModelErrorsWithBackground = pexConfig.Field(
140  doc="Model flux errors with background term?",
141  dtype=bool,
142  default=True
143  )
144  psfCandidateName = pexConfig.Field(
145  doc="Name of field with psf candidate flag for propagation",
146  dtype=str,
147  default="calib_psf_candidate"
148  )
149  doSubtractLocalBackground = pexConfig.Field(
150  doc=("Subtract the local background before performing calibration? "
151  "This is only supported for circular aperture calibration fluxes."),
152  dtype=bool,
153  default=False
154  )
155  localBackgroundFluxField = pexConfig.Field(
156  doc="Full name of the local background instFlux field to use.",
157  dtype=str,
158  default='base_LocalBackground_instFlux'
159  )
160  sourceSelector = sourceSelectorRegistry.makeField(
161  doc="How to select sources",
162  default="science"
163  )
164  apertureInnerInstFluxField = pexConfig.Field(
165  doc=("Full name of instFlux field that contains inner aperture "
166  "flux for aperture correction proxy"),
167  dtype=str,
168  default='base_CircularApertureFlux_12_0_instFlux'
169  )
170  apertureOuterInstFluxField = pexConfig.Field(
171  doc=("Full name of instFlux field that contains outer aperture "
172  "flux for aperture correction proxy"),
173  dtype=str,
174  default='base_CircularApertureFlux_17_0_instFlux'
175  )
176  doReferenceMatches = pexConfig.Field(
177  doc="Match reference catalog as additional constraint on calibration",
178  dtype=bool,
179  default=True,
180  )
181  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
182  target=FgcmLoadReferenceCatalogTask,
183  doc="FGCM reference object loader",
184  )
185  nVisitsPerCheckpoint = pexConfig.Field(
186  doc="Number of visits read between checkpoints",
187  dtype=int,
188  default=500,
189  )
190 
191  def setDefaults(self):
192  sourceSelector = self.sourceSelectorsourceSelector["science"]
193  sourceSelector.setDefaults()
194 
195  sourceSelector.doFlags = True
196  sourceSelector.doUnresolved = True
197  sourceSelector.doSignalToNoise = True
198  sourceSelector.doIsolated = True
199 
200  sourceSelector.signalToNoise.minimum = 10.0
201  sourceSelector.signalToNoise.maximum = 1000.0
202 
203  # FGCM operates on unresolved sources, and this setting is
204  # appropriate for the current base_ClassificationExtendedness
205  sourceSelector.unresolved.maximum = 0.5
206 
207 
208 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
209  """Subclass of TaskRunner for FgcmBuildStars tasks
210 
211  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
212  arguments, one of which is the butler (for persistence and mapper data),
213  and a list of dataRefs extracted from the command line. Note that FGCM
214  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
215  This class transforms the process arguments generated by the ArgumentParser
216  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
217  not use any parallelization.
218  """
219  @staticmethod
220  def getTargetList(parsedCmd):
221  """
222  Return a list with one element: a tuple with the butler and
223  list of dataRefs
224  """
225  # we want to combine the butler with any (or no!) dataRefs
226  return [(parsedCmd.butler, parsedCmd.id.refList)]
227 
228  def __call__(self, args):
229  """
230  Parameters
231  ----------
232  args: `tuple` with (butler, dataRefList)
233 
234  Returns
235  -------
236  exitStatus: `list` with `lsst.pipe.base.Struct`
237  exitStatus (0: success; 1: failure)
238  """
239  butler, dataRefList = args
240 
241  task = self.TaskClass(config=self.config, log=self.log)
242 
243  exitStatus = 0
244  if self.doRaise:
245  task.runDataRef(butler, dataRefList)
246  else:
247  try:
248  task.runDataRef(butler, dataRefList)
249  except Exception as e:
250  exitStatus = 1
251  task.log.fatal("Failed: %s" % e)
252  if not isinstance(e, pipeBase.TaskError):
253  traceback.print_exc(file=sys.stderr)
254 
255  task.writeMetadata(butler)
256 
257  # The task does not return any results:
258  return [pipeBase.Struct(exitStatus=exitStatus)]
259 
260  def run(self, parsedCmd):
261  """
262  Run the task, with no multiprocessing
263 
264  Parameters
265  ----------
266  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
267  """
268 
269  resultList = []
270 
271  if self.precall(parsedCmd):
272  targetList = self.getTargetListgetTargetList(parsedCmd)
273  resultList = self(targetList[0])
274 
275  return resultList
276 
277 
278 class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
279  """
280  Base task to build stars for FGCM global calibration
281 
282  Parameters
283  ----------
284  butler : `lsst.daf.persistence.Butler`
285  """
286  def __init__(self, butler=None, initInputs=None, **kwargs):
287  super().__init__(**kwargs)
288 
289  self.makeSubtask("sourceSelector")
290  # Only log warning and fatal errors from the sourceSelector
291  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
292 
293  # no saving of metadata for now
294  def _getMetadataName(self):
295  return None
296 
297  @pipeBase.timeMethod
298  def runDataRef(self, butler, dataRefs):
299  """
300  Cross-match and make star list for FGCM Input
301 
302  Parameters
303  ----------
304  butler: `lsst.daf.persistence.Butler`
305  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
306  Source data references for the input visits.
307 
308  Raises
309  ------
310  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
311  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
312  fails if the calibFlux is not a CircularAperture flux.
313  """
314  datasetType = dataRefs[0].butlerSubset.datasetType
315  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
316 
317  if self.config.doReferenceMatches:
318  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
319  # Ensure that we have a LUT
320  if not butler.datasetExists('fgcmLookUpTable'):
321  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
322  # Compute aperture radius if necessary. This is useful to do now before
323  # any heavy lifting has happened (fail early).
324  calibFluxApertureRadius = None
325  if self.config.doSubtractLocalBackground:
326  try:
327  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
328  self.config.instFluxField)
329  except RuntimeError as e:
330  raise RuntimeError("Could not determine aperture radius from %s. "
331  "Cannot use doSubtractLocalBackground." %
332  (self.config.instFluxField)) from e
333 
334  camera = butler.get('camera')
335  groupedDataRefs = self._findAndGroupDataRefs_findAndGroupDataRefs(camera, dataRefs, butler=butler)
336 
337  # Make the visit catalog if necessary
338  # First check if the visit catalog is in the _current_ path
339  # We cannot use Gen2 datasetExists() because that checks all parent
340  # directories as well, which would make recovering from faults
341  # and fgcmcal reruns impossible.
342  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
343  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
344  if os.path.exists(filename):
345  # This file exists and we should continue processing
346  inVisitCat = visitCatDataRef.get()
347  if len(inVisitCat) != len(groupedDataRefs):
348  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
349  "number of visits. Cannot continue.")
350  else:
351  inVisitCat = None
352 
353  visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
354  visitCatDataRef=visitCatDataRef,
355  inVisitCat=inVisitCat)
356 
357  # Persist the visitCat as a checkpoint file.
358  visitCatDataRef.put(visitCat)
359 
360  starObsDataRef = butler.dataRef('fgcmStarObservations')
361  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
362  if os.path.exists(filename):
363  inStarObsCat = starObsDataRef.get()
364  else:
365  inStarObsCat = None
366 
367  rad = calibFluxApertureRadius
368  sourceSchemaDataRef = butler.dataRef('src_schema')
369  fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
370  visitCat,
371  sourceSchemaDataRef,
372  camera,
373  calibFluxApertureRadius=rad,
374  starObsDataRef=starObsDataRef,
375  visitCatDataRef=visitCatDataRef,
376  inStarObsCat=inStarObsCat)
377  visitCatDataRef.put(visitCat)
378  starObsDataRef.put(fgcmStarObservationCat)
379 
380  # Always do the matching.
381  if self.config.doReferenceMatches:
382  lutDataRef = butler.dataRef('fgcmLookUpTable')
383  else:
384  lutDataRef = None
385  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
386  fgcmStarObservationCat,
387  lutDataRef=lutDataRef)
388 
389  # Persist catalogs via the butler
390  butler.put(fgcmStarIdCat, 'fgcmStarIds')
391  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
392  if fgcmRefCat is not None:
393  butler.put(fgcmRefCat, 'fgcmReferenceStars')
394 
395  @abc.abstractmethod
396  def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
397  """
398  Find and group dataRefs (by visit). For Gen2 usage, set butler, and for
399  Gen3, use calexpDataRefDict
400 
401  Parameters
402  ----------
403  camera : `lsst.afw.cameraGeom.Camera`
404  Camera from the butler.
405  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or
406  `lsst.daf.butler.DeferredDatasetHandle`
407  Data references for the input visits.
408  butler : `lsst.daf.persistence.Butler`, optional
409  Gen2 butler when used as CommandLineTask
410  calexpDataRefDict : `dict`, optional
411  Dictionary of Gen3 deferred data refs for calexps
412 
413  Returns
414  -------
415  groupedDataRefs : `OrderedDict` [`int`, `list`]
416  Dictionary with sorted visit keys, and `list`s of
417  `lsst.daf.persistence.ButlerDataRef` or
418  `lsst.daf.butler.DeferredDatasetHandle`
419 
420  Raises
421  ------
422  RuntimeError : Raised if neither or both of butler and dataRefDict are set.
423  """
424  raise NotImplementedError("_findAndGroupDataRefs not implemented.")
425 
426  @abc.abstractmethod
427  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
428  sourceSchemaDataRef,
429  camera,
430  calibFluxApertureRadius=None,
431  visitCatDataRef=None,
432  starObsDataRef=None,
433  inStarObsCat=None):
434  """
435  Compile all good star observations from visits in visitCat. Checkpoint files
436  will be stored if both visitCatDataRef and starObsDataRef are not None.
437 
438  Parameters
439  ----------
440  groupedDataRefs: `dict` of `list`s
441  Lists of `~lsst.daf.persistence.ButlerDataRef` or
442  `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
443  visitCat: `~afw.table.BaseCatalog`
444  Catalog with visit data for FGCM
445  sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or
446  `~lsst.daf.butler.DeferredDatasetHandle`
447  DataRef for the schema of the src catalogs.
448  camera: `~lsst.afw.cameraGeom.Camera`
449  calibFluxApertureRadius: `float`, optional
450  Aperture radius for calibration flux.
451  visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
452  Dataref to write visitCat for checkpoints
453  starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional
454  Dataref to write the star observation catalog for checkpoints.
455  inStarObsCat: `~afw.table.BaseCatalog`
456  Input observation catalog. If this is incomplete, observations
457  will be appended from when it was cut off.
458 
459  Returns
460  -------
461  fgcmStarObservations: `afw.table.BaseCatalog`
462  Full catalog of good observations.
463 
464  Raises
465  ------
466  RuntimeError: Raised if doSubtractLocalBackground is True and
467  calibFluxApertureRadius is not set.
468  """
469  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
470 
471  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
472  visitCatDataRef=None, inVisitCat=None):
473  """
474  Make a visit catalog with all the keys from each visit
475 
476  Parameters
477  ----------
478  camera: `lsst.afw.cameraGeom.Camera`
479  Camera from the butler
480  groupedDataRefs: `dict`
481  Dictionary with visit keys, and `list`s of
482  `lsst.daf.persistence.ButlerDataRef`
483  bkgDataRefDict: `dict`, optional
484  Dictionary of gen3 dataRefHandles for background info.
485  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
486  Dataref to write visitCat for checkpoints
487  inVisitCat: `afw.table.BaseCatalog`, optional
488  Input (possibly incomplete) visit catalog
489 
490  Returns
491  -------
492  visitCat: `afw.table.BaseCatalog`
493  """
494 
495  self.log.info("Assembling visitCatalog from %d %ss" %
496  (len(groupedDataRefs), self.config.visitDataRefName))
497 
498  nCcd = len(camera)
499 
500  if inVisitCat is None:
501  schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
502 
503  visitCat = afwTable.BaseCatalog(schema)
504  visitCat.reserve(len(groupedDataRefs))
505  visitCat.resize(len(groupedDataRefs))
506 
507  visitCat['visit'] = list(groupedDataRefs.keys())
508  visitCat['used'] = 0
509  visitCat['sources_read'] = False
510  else:
511  visitCat = inVisitCat
512 
513  # No matter what, fill the catalog. This will check if it was
514  # already read.
515  self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
516  bkgDataRefDict=bkgDataRefDict,
517  visitCatDataRef=visitCatDataRef)
518 
519  return visitCat
520 
521  def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
522  visitCatDataRef=None):
523  """
524  Fill the visit catalog with visit metadata
525 
526  Parameters
527  ----------
528  visitCat: `afw.table.BaseCatalog`
529  Catalog with schema from _makeFgcmVisitSchema()
530  groupedDataRefs: `dict`
531  Dictionary with visit keys, and `list`s of
532  `lsst.daf.persistence.ButlerDataRef`
533  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
534  Dataref to write visitCat for checkpoints
535  bkgDataRefDict: `dict`, optional
536  Dictionary of gen3 dataRefHandles for background info. FIXME
537  """
538  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
539 
540  for i, visit in enumerate(groupedDataRefs):
541  # We don't use the bypasses since we need the psf info which does
542  # not have a bypass
543  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
544  # can be fixed
545 
546  # Do not read those that have already been read
547  if visitCat['used'][i]:
548  continue
549 
550  if (i % self.config.nVisitsPerCheckpoint) == 0:
551  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
552  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
553  # Save checkpoint if desired
554  if visitCatDataRef is not None:
555  visitCatDataRef.put(visitCat)
556 
557  # Note that the reference ccd is first in the list (if available).
558 
559  # The first dataRef in the group will be the reference ccd (if available)
560  dataRef = groupedDataRefs[visit][0]
561  if isinstance(dataRef, dafPersist.ButlerDataRef):
562  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
563  visitInfo = exp.getInfo().getVisitInfo()
564  label = dataRef.get(datasetType='calexp_filterLabel')
565  physicalFilter = label.physicalLabel
566  psf = exp.getPsf()
567  else:
568  visitInfo = dataRef.get(component='visitInfo')
569  # TODO: When DM-28583 is fixed we can get the filterLabel
570  # via dataRef.get(component='filterLabel')
571  physicalFilter = dataRef.dataId['physical_filter']
572  psf = dataRef.get(component='psf')
573 
574  rec = visitCat[i]
575  rec['visit'] = visit
576  rec['physicalFilter'] = physicalFilter
577  # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
578  # much easier to get the wcs's necessary to recompute the pointing
579  # ra/dec at the center of the camera.
580  radec = visitInfo.getBoresightRaDec()
581  rec['telra'] = radec.getRa().asDegrees()
582  rec['teldec'] = radec.getDec().asDegrees()
583  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
584  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
585  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
586  rec['exptime'] = visitInfo.getExposureTime()
587  # convert from Pa to millibar
588  # Note that I don't know if this unit will need to be per-camera config
589  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
590  # Flag to signify if this is a "deep" field. Not currently used
591  rec['deepFlag'] = 0
592  # Relative flat scaling (1.0 means no relative scaling)
593  rec['scaling'][:] = 1.0
594  # Median delta aperture, to be measured from stars
595  rec['deltaAper'] = 0.0
596 
597  rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
598 
599  if self.config.doModelErrorsWithBackground:
600  foundBkg = False
601  if isinstance(dataRef, dafPersist.ButlerDataRef):
602  det = dataRef.dataId[self.config.ccdDataRefName]
603  if dataRef.datasetExists(datasetType='calexpBackground'):
604  bgList = dataRef.get(datasetType='calexpBackground')
605  foundBkg = True
606  else:
607  det = dataRef.dataId.byName()['detector']
608  try:
609  bkgRef = bkgDataRefDict[(visit, det)]
610  bgList = bkgRef.get()
611  foundBkg = True
612  except KeyError:
613  pass
614 
615  if foundBkg:
616  bgStats = (bg[0].getStatsImage().getImage().array
617  for bg in bgList)
618  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
619  else:
620  self.log.warn('Sky background not found for visit %d / ccd %d' %
621  (visit, det))
622  rec['skyBackground'] = -1.0
623  else:
624  rec['skyBackground'] = -1.0
625 
626  rec['used'] = 1
627 
628  def _makeSourceMapper(self, sourceSchema):
629  """
630  Make a schema mapper for fgcm sources
631 
632  Parameters
633  ----------
634  sourceSchema: `afwTable.Schema`
635  Default source schema from the butler
636 
637  Returns
638  -------
639  sourceMapper: `afwTable.schemaMapper`
640  Mapper to the FGCM source schema
641  """
642 
643  # create a mapper to the preferred output
644  sourceMapper = afwTable.SchemaMapper(sourceSchema)
645 
646  # map to ra/dec
647  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
648  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
649  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
650  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
651  # Add the mapping if the field exists in the input catalog.
652  # If the field does not exist, simply add it (set to False).
653  # This field is not required for calibration, but is useful
654  # to collate if available.
655  try:
656  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
657  'psf_candidate')
658  except LookupError:
659  sourceMapper.editOutputSchema().addField(
660  "psf_candidate", type='Flag',
661  doc=("Flag set if the source was a candidate for PSF determination, "
662  "as determined by the star selector."))
663 
664  # and add the fields we want
665  sourceMapper.editOutputSchema().addField(
666  "visit", type=np.int32, doc="Visit number")
667  sourceMapper.editOutputSchema().addField(
668  "ccd", type=np.int32, doc="CCD number")
669  sourceMapper.editOutputSchema().addField(
670  "instMag", type=np.float32, doc="Instrumental magnitude")
671  sourceMapper.editOutputSchema().addField(
672  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
673  sourceMapper.editOutputSchema().addField(
674  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
675  sourceMapper.editOutputSchema().addField(
676  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
677 
678  return sourceMapper
679 
680  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
681  """
682  Use FGCM code to match observations into unique stars.
683 
684  Parameters
685  ----------
686  visitCat: `afw.table.BaseCatalog`
687  Catalog with visit data for fgcm
688  obsCat: `afw.table.BaseCatalog`
689  Full catalog of star observations for fgcm
690  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
691  `lsst.daf.butler.DeferredDatasetHandle`, optional
692  Data reference to fgcm look-up table (used if matching reference stars).
693 
694  Returns
695  -------
696  fgcmStarIdCat: `afw.table.BaseCatalog`
697  Catalog of unique star identifiers and index keys
698  fgcmStarIndicesCat: `afwTable.BaseCatalog`
699  Catalog of unique star indices
700  fgcmRefCat: `afw.table.BaseCatalog`
701  Catalog of matched reference stars.
702  Will be None if `config.doReferenceMatches` is False.
703  """
704  # get filter names into a numpy array...
705  # This is the type that is expected by the fgcm code
706  visitFilterNames = np.zeros(len(visitCat), dtype='a30')
707  for i in range(len(visitCat)):
708  visitFilterNames[i] = visitCat[i]['physicalFilter']
709 
710  # match to put filterNames with observations
711  visitIndex = np.searchsorted(visitCat['visit'],
712  obsCat['visit'])
713 
714  obsFilterNames = visitFilterNames[visitIndex]
715 
716  if self.config.doReferenceMatches:
717  # Get the reference filter names, using the LUT
718  lutCat = lutDataRef.get()
719 
720  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
721  zip(lutCat[0]['physicalFilters'].split(','),
722  lutCat[0]['stdPhysicalFilters'].split(','))}
723  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
724  zip(lutCat[0]['stdPhysicalFilters'].split(','),
725  lutCat[0]['lambdaStdFilter'])}
726 
727  del lutCat
728 
729  referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
730  stdFilterDict,
731  stdLambdaDict)
732  self.log.info("Using the following reference filters: %s" %
733  (', '.join(referenceFilterNames)))
734 
735  else:
736  # This should be an empty list
737  referenceFilterNames = []
738 
739  # make the fgcm starConfig dict
740  starConfig = {'logger': self.log,
741  'filterToBand': self.config.physicalFilterMap,
742  'requiredBands': self.config.requiredBands,
743  'minPerBand': self.config.minPerBand,
744  'matchRadius': self.config.matchRadius,
745  'isolationRadius': self.config.isolationRadius,
746  'matchNSide': self.config.matchNside,
747  'coarseNSide': self.config.coarseNside,
748  'densNSide': self.config.densityCutNside,
749  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
750  'primaryBands': self.config.primaryBands,
751  'referenceFilterNames': referenceFilterNames}
752 
753  # initialize the FgcmMakeStars object
754  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
755 
756  # make the primary stars
757  # note that the ra/dec native Angle format is radians
758  # We determine the conversion from the native units (typically
759  # radians) to degrees for the first observation. This allows us
760  # to treate ra/dec as numpy arrays rather than Angles, which would
761  # be approximately 600x slower.
762  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
763  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
764  obsCat['dec'] * conv,
765  filterNameArray=obsFilterNames,
766  bandSelected=False)
767 
768  # and match all the stars
769  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
770  obsCat['dec'] * conv,
771  obsFilterNames)
772 
773  if self.config.doReferenceMatches:
774  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
775 
776  # now persist
777 
778  objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
779 
780  # make catalog and records
781  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
782  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
783  for i in range(fgcmMakeStars.objIndexCat.size):
784  fgcmStarIdCat.addNew()
785 
786  # fill the catalog
787  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
788  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
789  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
790  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
791  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
792 
793  obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
794 
795  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
796  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
797  for i in range(fgcmMakeStars.obsIndexCat.size):
798  fgcmStarIndicesCat.addNew()
799 
800  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
801 
802  if self.config.doReferenceMatches:
803  refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
804 
805  fgcmRefCat = afwTable.BaseCatalog(refSchema)
806  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
807 
808  for i in range(fgcmMakeStars.referenceCat.size):
809  fgcmRefCat.addNew()
810 
811  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
812  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
813  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
814 
815  md = PropertyList()
816  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
817  md.set("FILTERNAMES", referenceFilterNames)
818  fgcmRefCat.setMetadata(md)
819 
820  else:
821  fgcmRefCat = None
822 
823  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
824 
825  def _makeFgcmVisitSchema(self, nCcd):
826  """
827  Make a schema for an fgcmVisitCatalog
828 
829  Parameters
830  ----------
831  nCcd: `int`
832  Number of CCDs in the camera
833 
834  Returns
835  -------
836  schema: `afwTable.Schema`
837  """
838 
839  schema = afwTable.Schema()
840  schema.addField('visit', type=np.int32, doc="Visit number")
841  schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
842  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
843  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
844  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
845  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
846  schema.addField('mjd', type=np.float64, doc="MJD of visit")
847  schema.addField('exptime', type=np.float32, doc="Exposure time")
848  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
849  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
850  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
851  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
852  # the following field is not used yet
853  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
854  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
855  size=nCcd)
856  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
857  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
858 
859  return schema
860 
861  def _makeFgcmObjSchema(self):
862  """
863  Make a schema for the objIndexCat from fgcmMakeStars
864 
865  Returns
866  -------
867  schema: `afwTable.Schema`
868  """
869 
870  objSchema = afwTable.Schema()
871  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
872  # Will investigate making these angles...
873  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
874  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
875  objSchema.addField('obsArrIndex', type=np.int32,
876  doc='Index in obsIndexTable for first observation')
877  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
878 
879  return objSchema
880 
881  def _makeFgcmObsSchema(self):
882  """
883  Make a schema for the obsIndexCat from fgcmMakeStars
884 
885  Returns
886  -------
887  schema: `afwTable.Schema`
888  """
889 
890  obsSchema = afwTable.Schema()
891  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
892 
893  return obsSchema
894 
895  def _makeFgcmRefSchema(self, nReferenceBands):
896  """
897  Make a schema for the referenceCat from fgcmMakeStars
898 
899  Parameters
900  ----------
901  nReferenceBands: `int`
902  Number of reference bands
903 
904  Returns
905  -------
906  schema: `afwTable.Schema`
907  """
908 
909  refSchema = afwTable.Schema()
910  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
911  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
912  size=nReferenceBands)
913  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
914  size=nReferenceBands)
915 
916  return refSchema
917 
918  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
919  """
920  Get the reference filter names, in wavelength order, from the visitCat and
921  information from the look-up-table.
922 
923  Parameters
924  ----------
925  visitCat: `afw.table.BaseCatalog`
926  Catalog with visit data for FGCM
927  stdFilterDict: `dict`
928  Mapping of filterName to stdFilterName from LUT
929  stdLambdaDict: `dict`
930  Mapping of stdFilterName to stdLambda from LUT
931 
932  Returns
933  -------
934  referenceFilterNames: `list`
935  Wavelength-ordered list of reference filter names
936  """
937 
938  # Find the unique list of filter names in visitCat
939  filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
940 
941  # Find the unique list of "standard" filters
942  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
943 
944  # And sort these by wavelength
945  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
946 
947  return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchemaDataRef, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def __init__(self, butler=None, initInputs=None, **kwargs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:799