lsst.jointcal  17.0.1-15-g1718489+8
testUtils.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Functions to help create jointcal tests by generating fake data."""
23 
24 __all__ = ['createFakeCatalog', 'createTwoFakeCcdImages', 'getMeasuredStarsFromCatalog']
25 
26 import os
27 import unittest
28 
29 import numpy as np
30 
31 import lsst.afw.geom
32 import lsst.afw.table
34 import lsst.pipe.base
35 
36 import lsst.jointcal.star
37 
38 
40  """Returns True if the necessary packages and files are available.
41 
42  We need ``obs_cfht`` to load the test/data/cfht_minimal dataset, which
43  includes the metadata that is used to build the fake catalogs.
44  """
45  try:
46  import lsst.obs.cfht # noqa: F401
47  return True
48  except ImportError:
49  return False
50 
51 
52 def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeCcdId=12,
53  photoCalibMean1=1e-2, photoCalibMean2=1.2e-2):
54  """Return two fake ccdImages built on CFHT Megacam metadata.
55 
56  If ``num1 == num2``, the catalogs will align on-sky so each source will
57  have a match in the other catalog.
58 
59  This uses the butler dataset stored in `tests/data/cfht_minimal` to
60  bootstrap the metadata.
61 
62  Parameters
63  ----------
64  num1, num2 : `int`, optional
65  Number of sources to put in the first and second catalogs. Should be
66  a square, to have sqrt(num) centroids on a grid.
67  seed : `int`, optional
68  Seed value for np.random.
69  fakeCcdId : `int`, optional
70  Sensor identifier to use for both CcdImages. The wcs, bbox, photoCalib, etc.
71  will still be drawn from the CFHT ccd=12 files, as that is the only
72  testdata that is included in this simple test dataset.
73  photoCalibMean1, photoCalibMean2: `float`, optional
74  The mean photometric calibration to pass to each ccdImage construction.
75  Note: this value is 1/instFluxMag0, so it should be less than 1.
76 
77  Returns
78  -------
79  struct : `lsst.pipe.base.Struct`
80  Result struct with components:
81 
82  - `camera` : Camera representing these catalogs
83  (`lsst.afw.cameraGeom.Camera`).
84  - `catalogs` : Catalogs containing fake sources
85  (`list` of `lsst.afw.table.SourceCatalog`).
86  - `ccdImageList` : CcdImages containing the metadata and fake sources
87  (`list` of `lsst.jointcal.CcdImage`).
88  - `bbox` : Bounding Box of the image (`lsst.afw.geom.Box2I`).
89  - 'fluxFieldName' : name of the instFlux field in the catalogs ('str').
90  """
91  if not canRunTests():
92  msg = "Necessary packages not available to run tests that use the cfht_minimal dataset."
93  raise unittest.SkipTest(msg)
94 
95  np.random.seed(seed)
96 
97  visit1 = 849375
98  visit2 = 850587
99  fluxFieldName = "SomeFlux"
100 
101  # Load or fake the necessary metadata for each CcdImage
102  dataDir = lsst.utils.getPackageDir('jointcal')
103  inputDir = os.path.join(dataDir, 'tests/data/cfht_minimal')
104  butler = lsst.daf.persistence.Butler(inputDir)
105 
106  # so we can access parts of the camera later (e.g. focal plane)
107  camera = butler.get('camera', visit=visit1)
108 
109  struct1 = createFakeCcdImage(butler, visit1, num1, fluxFieldName,
110  photoCalibMean=photoCalibMean1, photoCalibErr=1.0, fakeCcdId=fakeCcdId)
111  struct2 = createFakeCcdImage(butler, visit2, num2, fluxFieldName,
112  photoCalibMean=photoCalibMean2, photoCalibErr=5.0, fakeCcdId=fakeCcdId)
113 
114  return lsst.pipe.base.Struct(camera=camera,
115  catalogs=[struct1.catalog, struct2.catalog],
116  ccdImageList=[struct1.ccdImage, struct2.ccdImage],
117  bbox=struct1.bbox,
118  fluxFieldName=fluxFieldName)
119 
120 
121 def createFakeCcdImage(butler, visit, num, fluxFieldName,
122  photoCalibMean=1e-2, photoCalibErr=1.0, fakeCcdId=12):
123  """Create a fake CcdImage by making a fake catalog.
124 
125  Parameters
126  ----------
127  butler : `lsst.daf.persistence.Butler`
128  Butler to load metadata from.
129  visit : `int`
130  Visit identifier to build a butler dataId.
131  num : `int`
132  Number of sources to put in the catalogs. Should be
133  a square, to have sqrt(num) centroids on a grid.
134  fluxFieldName : `str`
135  Name of the flux field to populate in the catalog, without `_instFlux`
136  (e.g. "slot_CalibFlux").
137  photoCalibMean : `float`, optional
138  Value to set for calibrationMean in the created PhotoCalib.
139  Note: this value is 1/instFluxMag0, so it should be less than 1.
140  photoCalibErr : `float`, optional
141  Value to set for calibrationErr in the created PhotoCalib.
142  fakeCcdId : `int`, optional
143  Use this as the ccdId in the returned CcdImage.
144 
145  Returns
146  -------
147  struct : `lsst.pipe.base.Struct`
148  Result struct with components:
149 
150  - `catalog` : Catalogs containing fake sources
151  (`lsst.afw.table.SourceCatalog`).
152  - `ccdImage` : CcdImage containing the metadata and fake sources
153  (`lsst.jointcal.CcdImage`).
154  - `bbox` : Bounding Box of the image (`lsst.afw.geom.Box2I`).
155  """
156  ccdId = 12 # we only have data for ccd=12
157 
158  dataId = dict(visit=visit, ccd=ccdId)
159  skyWcs = butler.get('calexp_wcs', dataId=dataId)
160  visitInfo = butler.get('calexp_visitInfo', dataId=dataId)
161  bbox = butler.get('calexp_bbox', dataId=dataId)
162  detector = butler.get('calexp_detector', dataId=dataId)
163  filt = butler.get("calexp_filter", dataId=dataId).getName()
164  photoCalib = lsst.afw.image.PhotoCalib(photoCalibMean, photoCalibErr)
165 
166  catalog = createFakeCatalog(num, bbox, fluxFieldName, skyWcs=skyWcs)
167  ccdImage = lsst.jointcal.ccdImage.CcdImage(catalog, skyWcs, visitInfo, bbox, filt, photoCalib,
168  detector, visit, fakeCcdId, fluxFieldName)
169 
170  return lsst.pipe.base.Struct(catalog=catalog, ccdImage=ccdImage, bbox=bbox)
171 
172 
173 def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False):
174  """Return a fake minimally-useful catalog for jointcal.
175 
176  Parameters
177  ----------
178  num : `int`
179  Number of sources to put in the catalogs. Should be
180  a square, to have sqrt(num) centroids on a grid.
181  bbox : `lsst.afw.geom.Box2I`
182  Bounding Box of the detector to populate.
183  fluxFieldName : `str`
184  Name of the flux field to populate in the catalog, without `_instFlux`
185  (e.g. "slot_CalibFlux").
186  skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
187  If supplied, use this to fill in coordinates from centroids.
188  refCat : `bool`, optional
189  Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
190 
191  Returns
192  -------
193  catalog : `lsst.afw.table.SourceCatalog`
194  A populated source catalog.
195  """
197  # centroid
198  centroidKey = lsst.afw.table.Point2DKey.addFields(schema, "centroid", "centroid", "pixels")
199  xErrKey = schema.addField("centroid_xErr", type="F")
200  yErrKey = schema.addField("centroid_yErr", type="F")
201  # shape
202  shapeKey = lsst.afw.table.QuadrupoleKey.addFields(schema, "shape", "",
203  lsst.afw.table.CoordinateType.PIXEL)
204  # Put the fake sources in the minimal catalog.
205  schema.addField(fluxFieldName+"_instFlux", type="D", doc="post-ISR instFlux")
206  schema.addField(fluxFieldName+"_instFluxErr", type="D", doc="post-ISR instFlux stddev")
207  schema.addField(fluxFieldName+"_flux", type="D", doc="source flux (nJy)")
208  schema.addField(fluxFieldName+"_fluxErr", type="D", doc="flux stddev (nJy)")
209  schema.addField(fluxFieldName+"_mag", type="D", doc="magnitude")
210  schema.addField(fluxFieldName+"_magErr", type="D", doc="magnitude stddev")
211  return fillCatalog(schema, num, bbox,
212  centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
213  skyWcs=skyWcs, refCat=refCat)
214 
215 
216 def fillCatalog(schema, num, bbox,
217  centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
218  skyWcs=None, fluxErrFraction=0.05, refCat=False):
219  """Return a catalog populated with fake, but reasonable, sources.
220 
221  Centroids are placed on a uniform grid, errors are normally distributed.
222 
223  Parameters
224  ----------
225  schema : `lsst.afw.table.Schema`
226  Pre-built schema to make the catalog from.
227  num : `int`
228  Number of sources to put in the catalog.
229  bbox : `lsst.afw.geom.Box2I`
230  Bounding box of the ccd to put sources in.
231  centroidKey : `lsst.afw.table.Key`
232  Key for the centroid field to populate.
233  xErrKey : `lsst.afw.table.Key`
234  Key for the xErr field to populate.
235  yErrKey : `lsst.afw.table.Key`
236  Key for the yErr field to populate.
237  shapeKey : `lsst.afw.table.Key`
238  Key for the shape field to populate.
239  fluxFieldName : `str`
240  Name of the flux field to populate in the catalog, without `_instFlux`
241  (e.g. "slot_CalibFlux").
242  skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
243  If supplied, use this to fill in coordinates from centroids.
244  fluxErrFraction : `float`, optional
245  Fraction of instFlux to use for the instFluxErr.
246  refCat : `bool`, optional
247  Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
248 
249  Returns
250  -------
251  catalog : `lsst.afw.table.SourceCatalog`
252  The filled catalog.
253  """
254  table = lsst.afw.table.SourceTable.make(schema)
255  table.defineCentroid('centroid')
256  table.defineShape('shape')
257  table.defineCalibFlux(fluxFieldName)
258  if refCat:
259  catalog = lsst.afw.table.SimpleCatalog(table)
260  else:
261  catalog = lsst.afw.table.SourceCatalog(table)
262 
263  instFlux = np.random.random(num)*10000
264  instFluxErr = np.abs(instFlux * np.random.normal(fluxErrFraction, scale=0.1, size=num))
265  xx = np.linspace(bbox.getMinX(), bbox.getMaxX(), int(np.sqrt(num)))
266  yy = np.linspace(bbox.getMinY(), bbox.getMaxY(), int(np.sqrt(num)))
267  xv, yv = np.meshgrid(xx, yy)
268  vx = np.random.normal(scale=0.1, size=num)
269  vy = np.random.normal(scale=0.1, size=num)
270 
271  # make all the sources perfectly spherical, for simplicity.
272  mxx = 1
273  myy = 1
274  mxy = 0
275 
276  for i, (x, y) in enumerate(zip(xv.ravel(), yv.ravel())):
277  record = catalog.addNew()
278  record.set('id', i)
279  record.set(centroidKey, lsst.afw.geom.Point2D(x, y))
280  record.set(shapeKey, lsst.afw.geom.ellipses.Quadrupole(mxx, myy, mxy))
281 
282  if skyWcs is not None:
283  lsst.afw.table.updateSourceCoords(skyWcs, catalog)
284 
285  catalog[xErrKey] = vx
286  catalog[yErrKey] = vy
287  catalog[fluxFieldName + '_instFlux'] = instFlux
288  catalog[fluxFieldName + '_instFluxErr'] = instFluxErr
289 
290  return catalog
291 
292 
293 def getMeasuredStarsFromCatalog(catalog, pixToFocal):
294  """Return a list of measuredStars built from a catalog.
295 
296  Parameters
297  ----------
298  catalog : `lsst.afw.table.SourceCatalog`
299  The table to get sources from.
300  pixToFocal : `lsst.afw.geom.TransformPoint2ToPoint2`
301  Transform that goes from pixel to focal plane coordinates, to set the
302  MeasuredStar x/y focal points.
303 
304  Returns
305  -------
306  stars : `list` of `lsst.jointcal.MeasuredStar`
307  MeasuredStars built from the catalog sources.
308  """
309  stars = []
310  for record in catalog:
311  star = lsst.jointcal.star.MeasuredStar()
312  star.x = record.getX()
313  star.y = record.getY()
314  star.setInstFluxAndErr(record.getCalibInstFlux(), record.getCalibInstFluxErr())
315  # TODO: cleanup after DM-4044
316  point = lsst.afw.geom.Point2D(star.x, star.y)
317  pointFocal = pixToFocal.applyForward(point)
318  star.setXFocal(pointFocal.getX())
319  star.setYFocal(pointFocal.getY())
320  stars.append(star)
321 
322  return stars
def createFakeCcdImage(butler, visit, num, fluxFieldName, photoCalibMean=1e-2, photoCalibErr=1.0, fakeCcdId=12)
Definition: testUtils.py:122
def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeCcdId=12, photoCalibMean1=1e-2, photoCalibMean2=1.2e-2)
Definition: testUtils.py:53
def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False)
Definition: testUtils.py:173
void updateSourceCoords(geom::SkyWcs const &wcs, SourceCollection &sourceList)
std::string getPackageDir(std::string const &packageName)
def fillCatalog(schema, num, bbox, centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName, skyWcs=None, fluxErrFraction=0.05, refCat=False)
Definition: testUtils.py:218
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
static Schema makeMinimalSchema()
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
def getMeasuredStarsFromCatalog(catalog, pixToFocal)
Definition: testUtils.py:293