lsst.jointcal 22.0.1-26-g6000b2f+22c1dfde1c
testUtils.py
Go to the documentation of this file.
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""Functions to help create jointcal tests by generating fake data."""
23
24__all__ = ['createFakeCatalog', 'createTwoFakeCcdImages', 'getMeasuredStarsFromCatalog']
25
26import os
27import unittest
28
29import numpy as np
30
31import lsst.afw.geom
32import lsst.afw.table
34import lsst.pipe.base
35
36import lsst.jointcal.star
37
38
40 """Returns True if the necessary packages and files are available.
41
42 We need ``obs_cfht`` to load the test/data/cfht_minimal dataset, which
43 includes the metadata that is used to build the fake catalogs.
44 """
45 try:
46 import lsst.obs.cfht # noqa: F401
47 return True
48 except ImportError:
49 return False
50
51
52def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeCcdId=12,
53 photoCalibMean1=1e-2, photoCalibMean2=1.2e-2,
54 fakeWcses=(None, None),
55 fakeVisitInfos=(None, None)):
56 """Return two fake ccdImages built on CFHT Megacam metadata.
57
58 If ``num1 == num2``, the catalogs will align on-sky so each source will
59 have a match in the other catalog.
60
61 This uses the butler dataset stored in `tests/data/cfht_minimal` to
62 bootstrap the metadata.
63
64 Parameters
65 ----------
66 num1, num2 : `int`, optional
67 Number of sources to put in the first and second catalogs. Should be
68 a square, to have sqrt(num) centroids on a grid.
69 seed : `int`, optional
70 Seed value for np.random.
71 fakeCcdId : `int`, optional
72 Sensor identifier to use for both CcdImages. The wcs, bbox, photoCalib, etc.
73 will still be drawn from the CFHT ccd=12 files, as that is the only
74 testdata that is included in this simple test dataset.
75 photoCalibMean1, photoCalibMean2: `float`, optional
76 The mean photometric calibration to pass to each ccdImage construction.
77 Note: this value is 1/instFluxMag0, so it should be less than 1.
78 fakeWcses : `list` [`lsst.afw.geom.SkyWcs`], optional
79 The SkyWcses to use instead of the ones read from disk.
80 fakeWcses : `list` [`lsst.afw.image.VisitInfo`], optional
81 The VisitInfos to use instead of the ones read from disk.
82
83 Returns
84 -------
85 struct : `lsst.pipe.base.Struct`
86 Result struct with components:
87
88 - `camera` : Camera representing these catalogs
90 - `catalogs` : Catalogs containing fake sources
91 (`list` of `lsst.afw.table.SourceCatalog`).
92 - `ccdImageList` : CcdImages containing the metadata and fake sources
93 (`list` of `lsst.jointcal.CcdImage`).
94 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`).
95 - 'fluxFieldName' : name of the instFlux field in the catalogs ('str').
96 """
97 if not canRunTests():
98 msg = "Necessary packages not available to run tests that use the cfht_minimal dataset."
99 raise unittest.SkipTest(msg)
100
101 np.random.seed(seed)
102
103 visit1 = 849375
104 visit2 = 850587
105 fluxFieldName = "SomeFlux"
106
107 # Load or fake the necessary metadata for each CcdImage
108 dataDir = lsst.utils.getPackageDir('jointcal')
109 inputDir = os.path.join(dataDir, 'tests/data/cfht_minimal')
110 butler = lsst.daf.persistence.Butler(inputDir)
111
112 # so we can access parts of the camera later (e.g. focal plane)
113 camera = butler.get('camera', visit=visit1)
114
115 struct1 = createFakeCcdImage(butler, visit1, num1, fluxFieldName,
116 photoCalibMean=photoCalibMean1, photoCalibErr=1.0, fakeCcdId=fakeCcdId,
117 fakeWcs=fakeWcses[0], fakeVisitInfo=fakeVisitInfos[0])
118 struct2 = createFakeCcdImage(butler, visit2, num2, fluxFieldName,
119 photoCalibMean=photoCalibMean2, photoCalibErr=5.0, fakeCcdId=fakeCcdId,
120 fakeWcs=fakeWcses[1], fakeVisitInfo=fakeVisitInfos[1])
121
122 return lsst.pipe.base.Struct(camera=camera,
123 catalogs=[struct1.catalog, struct2.catalog],
124 ccdImageList=[struct1.ccdImage, struct2.ccdImage],
125 bbox=struct1.bbox,
126 skyWcs=[struct1.skyWcs, struct2.skyWcs],
127 fluxFieldName=fluxFieldName)
128
129
130def createFakeCcdImage(butler, visit, num, fluxFieldName,
131 photoCalibMean=1e-2, photoCalibErr=1.0, fakeCcdId=12,
132 fakeWcs=None, fakeVisitInfo=None):
133 """Create a fake CcdImage by making a fake catalog.
134
135 Parameters
136 ----------
137 butler : `lsst.daf.persistence.Butler`
138 Butler to load metadata from.
139 visit : `int`
140 Visit identifier to build a butler dataId.
141 num : `int`
142 Number of sources to put in the catalogs. Should be
143 a square, to have sqrt(num) centroids on a grid.
144 fluxFieldName : `str`
145 Name of the flux field to populate in the catalog, without `_instFlux`
146 (e.g. "slot_CalibFlux").
147 photoCalibMean : `float`, optional
148 Value to set for calibrationMean in the created PhotoCalib.
149 Note: this value is 1/instFluxMag0, so it should be less than 1.
150 photoCalibErr : `float`, optional
151 Value to set for calibrationErr in the created PhotoCalib.
152 fakeCcdId : `int`, optional
153 Use this as the ccdId in the returned CcdImage.
154 fakeWcs : `lsst.afw.geom.SkyWcs`, optional
155 A SkyWcs to use instead of one read from disk.
156 fakeVisitInfo : `lsst.afw.image.VisitInfo`, optional
157 A VisitInfo to use instead of one read from disk.
158
159 Returns
160 -------
161 struct : `lsst.pipe.base.Struct`
162 Result struct with components:
163
164 - `catalog` : Catalogs containing fake sources
166 - `ccdImage` : CcdImage containing the metadata and fake sources
168 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`).
169 - `skyWcs` : SkyWcs of the image (`lsst.afw.geom.SkyWcs`).
170 """
171 ccdId = 12 # we only have data for ccd=12
172
173 dataId = dict(visit=visit, ccd=ccdId)
174 skyWcs = fakeWcs if fakeWcs is not None else butler.get('calexp_wcs', dataId=dataId)
175 visitInfo = fakeVisitInfo if fakeVisitInfo is not None else butler.get('calexp_visitInfo', dataId=dataId)
176 bbox = butler.get('calexp_bbox', dataId=dataId)
177 detector = butler.get('calexp_detector', dataId=dataId)
178 filt = butler.get("calexp_filter", dataId=dataId).getName()
179 photoCalib = lsst.afw.image.PhotoCalib(photoCalibMean, photoCalibErr)
180
181 catalog = createFakeCatalog(num, bbox, fluxFieldName, skyWcs=skyWcs)
182 ccdImage = lsst.jointcal.ccdImage.CcdImage(catalog, skyWcs, visitInfo, bbox, filt, photoCalib,
183 detector, visit, fakeCcdId, fluxFieldName)
184
185 return lsst.pipe.base.Struct(catalog=catalog, ccdImage=ccdImage, bbox=bbox, skyWcs=skyWcs)
186
187
188def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False):
189 """Return a fake minimally-useful catalog for jointcal.
190
191 Parameters
192 ----------
193 num : `int`
194 Number of sources to put in the catalogs. Should be
195 a square, to have sqrt(num) centroids on a grid.
196 bbox : `lsst.geom.Box2I`
197 Bounding Box of the detector to populate.
198 fluxFieldName : `str`
199 Name of the flux field to populate in the catalog, without `_instFlux`
200 (e.g. "slot_CalibFlux").
201 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
202 If supplied, use this to fill in coordinates from centroids.
203 refCat : `bool`, optional
204 Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
205
206 Returns
207 -------
209 A populated source catalog.
210 """
212 # centroid
213 centroidKey = lsst.afw.table.Point2DKey.addFields(schema, "centroid", "centroid", "pixels")
214 xErrKey = schema.addField("centroid_xErr", type="F")
215 yErrKey = schema.addField("centroid_yErr", type="F")
216 # shape
217 shapeKey = lsst.afw.table.QuadrupoleKey.addFields(schema, "shape", "",
218 lsst.afw.table.CoordinateType.PIXEL)
219 # Put the fake sources in the minimal catalog.
220 schema.addField(fluxFieldName+"_instFlux", type="D", doc="post-ISR instFlux")
221 schema.addField(fluxFieldName+"_instFluxErr", type="D", doc="post-ISR instFlux stddev")
222 schema.addField(fluxFieldName+"_flux", type="D", doc="source flux (nJy)")
223 schema.addField(fluxFieldName+"_fluxErr", type="D", doc="flux stddev (nJy)")
224 schema.addField(fluxFieldName+"_mag", type="D", doc="magnitude")
225 schema.addField(fluxFieldName+"_magErr", type="D", doc="magnitude stddev")
226 return fillCatalog(schema, num, bbox,
227 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
228 skyWcs=skyWcs, refCat=refCat)
229
230
231def fillCatalog(schema, num, bbox,
232 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
233 skyWcs=None, fluxErrFraction=0.05, refCat=False):
234 """Return a catalog populated with fake, but reasonable, sources.
235
236 Centroids are placed on a uniform grid, errors are normally distributed.
237
238 Parameters
239 ----------
240 schema : `lsst.afw.table.Schema`
241 Pre-built schema to make the catalog from.
242 num : `int`
243 Number of sources to put in the catalog.
244 bbox : `lsst.geom.Box2I`
245 Bounding box of the ccd to put sources in.
246 centroidKey : `lsst.afw.table.Key`
247 Key for the centroid field to populate.
248 xErrKey : `lsst.afw.table.Key`
249 Key for the xErr field to populate.
250 yErrKey : `lsst.afw.table.Key`
251 Key for the yErr field to populate.
252 shapeKey : `lsst.afw.table.Key`
253 Key for the shape field to populate.
254 fluxFieldName : `str`
255 Name of the flux field to populate in the catalog, without `_instFlux`
256 (e.g. "slot_CalibFlux").
257 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
258 If supplied, use this to fill in coordinates from centroids.
259 fluxErrFraction : `float`, optional
260 Fraction of instFlux to use for the instFluxErr.
261 refCat : `bool`, optional
262 Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
263
264 Returns
265 -------
267 The filled catalog.
268 """
269 table = lsst.afw.table.SourceTable.make(schema)
270 table.defineCentroid('centroid')
271 table.defineShape('shape')
272 table.defineCalibFlux(fluxFieldName)
273 if refCat:
274 catalog = lsst.afw.table.SimpleCatalog(table)
275 else:
276 catalog = lsst.afw.table.SourceCatalog(table)
277
278 instFlux = np.random.random(num)*10000
279 instFluxErr = np.abs(instFlux * np.random.normal(fluxErrFraction, scale=0.1, size=num))
280 xx = np.linspace(bbox.getMinX(), bbox.getMaxX(), int(np.sqrt(num)))
281 yy = np.linspace(bbox.getMinY(), bbox.getMaxY(), int(np.sqrt(num)))
282 xv, yv = np.meshgrid(xx, yy)
283 vx = np.random.normal(scale=0.1, size=num)
284 vy = np.random.normal(scale=0.1, size=num)
285
286 # make all the sources perfectly spherical, for simplicity.
287 mxx = 1
288 myy = 1
289 mxy = 0
290
291 for i, (x, y) in enumerate(zip(xv.ravel(), yv.ravel())):
292 record = catalog.addNew()
293 record.set('id', i)
294 record.set(centroidKey, lsst.geom.Point2D(x, y))
295 record.set(shapeKey, lsst.afw.geom.ellipses.Quadrupole(mxx, myy, mxy))
296
297 if skyWcs is not None:
298 lsst.afw.table.updateSourceCoords(skyWcs, catalog)
299
300 catalog[xErrKey] = vx
301 catalog[yErrKey] = vy
302 catalog[fluxFieldName + '_instFlux'] = instFlux
303 catalog[fluxFieldName + '_instFluxErr'] = instFluxErr
304
305 return catalog
306
307
308def getMeasuredStarsFromCatalog(catalog, pixToFocal):
309 """Return a list of measuredStars built from a catalog.
310
311 Parameters
312 ----------
314 The table to get sources from.
316 Transform that goes from pixel to focal plane coordinates, to set the
317 MeasuredStar x/y focal points.
318
319 Returns
320 -------
321 stars : `list` of `lsst.jointcal.MeasuredStar`
322 MeasuredStars built from the catalog sources.
323 """
324 stars = []
325 for record in catalog:
326 star = lsst.jointcal.star.MeasuredStar()
327 star.x = record.getX()
328 star.y = record.getY()
329 star.setInstFluxAndErr(record.getCalibInstFlux(), record.getCalibInstFluxErr())
330 # TODO: cleanup after DM-4044
331 point = lsst.geom.Point2D(star.x, star.y)
332 pointFocal = pixToFocal.applyForward(point)
333 star.setXFocal(pointFocal.getX())
334 star.setYFocal(pointFocal.getY())
335 stars.append(star)
336
337 return stars
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
Handler of an actual image from a single CCD.
Definition: CcdImage.h:64
Sources measured on images.
Definition: MeasuredStar.h:51
void updateSourceCoords(geom::SkyWcs const &wcs, SourceCollection &sourceList)
def fillCatalog(schema, num, bbox, centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName, skyWcs=None, fluxErrFraction=0.05, refCat=False)
Definition: testUtils.py:233
def createFakeCcdImage(butler, visit, num, fluxFieldName, photoCalibMean=1e-2, photoCalibErr=1.0, fakeCcdId=12, fakeWcs=None, fakeVisitInfo=None)
Definition: testUtils.py:132
def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeCcdId=12, photoCalibMean1=1e-2, photoCalibMean2=1.2e-2, fakeWcses=(None, None), fakeVisitInfos=(None, None))
Definition: testUtils.py:55
def getMeasuredStarsFromCatalog(catalog, pixToFocal)
Definition: testUtils.py:308
def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False)
Definition: testUtils.py:188