Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

#!/usr/bin/env python 

# 

# LSST Data Management System 

# Copyright 2008-2015 AURA/LSST. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <https://www.lsstcorp.org/LegalNotices/>. 

# 

import numpy 

 

from .multiBandUtils import (MergeSourcesRunner, _makeGetSchemaCatalogs, makeMergeArgumentParser, 

getInputSchema, getShortFilterName, readCatalog) 

 

import lsst.afw.table as afwTable 

 

from lsst.pex.config import Config, Field, ListField 

from lsst.pipe.base import CmdLineTask, Struct 

 

 

class MergeMeasurementsConfig(Config): 

"""! 

@anchor MergeMeasurementsConfig_ 

 

@brief Configuration parameters for the MergeMeasurementsTask 

""" 

pseudoFilterList = ListField(dtype=str, default=["sky"], 

doc="Names of filters which may have no associated detection\n" 

"(N.b. should include MergeDetectionsConfig.skyFilterName)") 

snName = Field(dtype=str, default="base_PsfFlux", 

doc="Name of flux measurement for calculating the S/N when choosing the reference band.") 

minSN = Field(dtype=float, default=10., 

doc="If the S/N from the priority band is below this value (and the S/N " 

"is larger than minSNDiff compared to the priority band), use the band with " 

"the largest S/N as the reference band.") 

minSNDiff = Field(dtype=float, default=3., 

doc="If the difference in S/N between another band and the priority band is larger " 

"than this value (and the S/N in the priority band is less than minSN) " 

"use the band with the largest S/N as the reference band") 

flags = ListField(dtype=str, doc="Require that these flags, if available, are not set", 

default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag", 

"ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ]) 

priorityList = ListField(dtype=str, default=[], 

doc="Priority-ordered list of bands for the merge.") 

coaddName = Field(dtype=str, default="deep", doc="Name of coadd") 

 

def validate(self): 

Config.validate(self) 

if len(self.priorityList) == 0: 

raise RuntimeError("No priority list provided") 

 

 

## @addtogroup LSST_task_documentation 

## @{ 

## @page MergeMeasurementsTask 

## @ref MergeMeasurementsTask_ "MergeMeasurementsTask" 

## @copybrief MergeMeasurementsTask 

## @} 

 

 

class MergeMeasurementsTask(CmdLineTask): 

r"""! 

@anchor MergeMeasurementsTask_ 

 

@brief Merge measurements from multiple bands 

 

@section pipe_tasks_multiBand_Contents Contents 

 

- @ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose 

- @ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize 

- @ref pipe_tasks_multiBand_MergeMeasurementsTask_Run 

- @ref pipe_tasks_multiBand_MergeMeasurementsTask_Config 

- @ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug 

- @ref pipe_tasks_multiband_MergeMeasurementsTask_Example 

 

@section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description 

 

Command-line task that merges measurements from multiple bands. 

 

Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter 

bands to construct a unified catalog that is suitable for driving forced photometry. Every source is 

required to have centroid, shape and flux measurements in each band. 

 

@par Inputs: 

deepCoadd_meas{tract,patch,filter}: SourceCatalog 

@par Outputs: 

deepCoadd_ref{tract,patch}: SourceCatalog 

@par Data Unit: 

tract, patch 

 

MergeMeasurementsTask subclasses @ref CmdLineTask_ "CmdLineTask". 

 

@section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization 

 

@copydoc \_\_init\_\_ 

 

@section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task 

 

@copydoc run 

 

@section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters 

 

See @ref MergeMeasurementsConfig_ 

 

@section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables 

 

The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 

flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 

files. 

 

MergeMeasurementsTask has no debug variables. 

 

@section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example 

of using MergeMeasurementsTask 

 

MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band. 

The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in 

coadds and individual exposures. 

Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list 

of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help` 

command line argument: 

@code 

mergeCoaddMeasurements.py --help 

@endcode 

 

To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 

will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 

step 7 at @ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring 

as follows: 

@code 

mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 

@endcode 

This will merge the HSC-I & HSC-R band catalogs. The results are written in 

`$CI_HSC_DIR/DATA/deepCoadd-results/`. 

""" 

_DefaultName = "mergeCoaddMeasurements" 

ConfigClass = MergeMeasurementsConfig 

RunnerClass = MergeSourcesRunner 

inputDataset = "meas" 

outputDataset = "ref" 

getSchemaCatalogs = _makeGetSchemaCatalogs("ref") 

 

@classmethod 

def _makeArgumentParser(cls): 

return makeMergeArgumentParser(cls._DefaultName, cls.inputDataset) 

 

def getInputSchema(self, butler=None, schema=None): 

return getInputSchema(self, butler, schema) 

 

def __init__(self, butler=None, schema=None, **kwargs): 

"""! 

Initialize the task. 

 

@param[in] schema: the schema of the detection catalogs used as input to this one 

@param[in] butler: a butler used to read the input schema from disk, if schema is None 

 

The task will set its own self.schema attribute to the schema of the output merged catalog. 

""" 

CmdLineTask.__init__(self, **kwargs) 

inputSchema = self.getInputSchema(butler=butler, schema=schema) 

self.schemaMapper = afwTable.SchemaMapper(inputSchema, True) 

self.schemaMapper.addMinimalSchema(inputSchema, True) 

self.instFluxKey = inputSchema.find(self.config.snName + "_instFlux").getKey() 

self.instFluxErrKey = inputSchema.find(self.config.snName + "_instFluxErr").getKey() 

self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey() 

 

self.flagKeys = {} 

for band in self.config.priorityList: 

short = getShortFilterName(band) 

outputKey = self.schemaMapper.editOutputSchema().addField( 

"merge_measurement_%s" % short, 

type="Flag", 

doc="Flag field set if the measurements here are from the %s filter" % band 

) 

peakKey = inputSchema.find("merge_peak_%s" % short).key 

footprintKey = inputSchema.find("merge_footprint_%s" % short).key 

self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey) 

self.schema = self.schemaMapper.getOutputSchema() 

 

self.pseudoFilterKeys = [] 

for filt in self.config.pseudoFilterList: 

try: 

self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey()) 

except Exception as e: 

self.log.warn("merge_peak is not set for pseudo-filter %s: %s" % (filt, e)) 

 

self.badFlags = {} 

for flag in self.config.flags: 

try: 

self.badFlags[flag] = self.schema.find(flag).getKey() 

except KeyError as exc: 

self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,)) 

 

def runDataRef(self, patchRefList): 

"""! 

@brief Merge coadd sources from multiple bands. Calls @ref `run`. 

@param[in] patchRefList list of data references for each filter 

""" 

catalogs = dict(readCatalog(self, patchRef) for patchRef in patchRefList) 

mergedCatalog = self.run(catalogs) 

self.write(patchRefList[0], mergedCatalog) 

 

def run(self, catalogs): 

"""! 

Merge measurement catalogs to create a single reference catalog for forced photometry 

 

@param[in] catalogs: the catalogs to be merged 

 

For parent sources, we choose the first band in config.priorityList for which the 

merge_footprint flag for that band is is True. 

 

For child sources, the logic is the same, except that we use the merge_peak flags. 

""" 

# Put catalogs, filters in priority order 

orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()] 

orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()] 

 

mergedCatalog = afwTable.SourceCatalog(self.schema) 

mergedCatalog.reserve(len(orderedCatalogs[0])) 

 

idKey = orderedCatalogs[0].table.getIdKey() 

for catalog in orderedCatalogs[1:]: 

if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)): 

raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match") 

 

# This first zip iterates over all the catalogs simultaneously, yielding a sequence of one 

# record for each band, in priority order. 

for orderedRecords in zip(*orderedCatalogs): 

 

maxSNRecord = None 

maxSNFlagKeys = None 

maxSN = 0. 

priorityRecord = None 

priorityFlagKeys = None 

prioritySN = 0. 

hasPseudoFilter = False 

 

# Now we iterate over those record-band pairs, keeping track of the priority and the 

# largest S/N band. 

for inputRecord, flagKeys in zip(orderedRecords, orderedKeys): 

parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint)) 

child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak)) 

 

if not (parent or child): 

for pseudoFilterKey in self.pseudoFilterKeys: 

if inputRecord.get(pseudoFilterKey): 

hasPseudoFilter = True 

priorityRecord = inputRecord 

priorityFlagKeys = flagKeys 

break 

if hasPseudoFilter: 

break 

 

isBad = any(inputRecord.get(flag) for flag in self.badFlags) 

if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.instFluxErrKey) == 0: 

sn = 0. 

else: 

sn = inputRecord.get(self.instFluxKey)/inputRecord.get(self.instFluxErrKey) 

if numpy.isnan(sn) or sn < 0.: 

sn = 0. 

if (parent or child) and priorityRecord is None: 

priorityRecord = inputRecord 

priorityFlagKeys = flagKeys 

prioritySN = sn 

if sn > maxSN: 

maxSNRecord = inputRecord 

maxSNFlagKeys = flagKeys 

maxSN = sn 

 

# If the priority band has a low S/N we would like to choose the band with the highest S/N as 

# the reference band instead. However, we only want to choose the highest S/N band if it is 

# significantly better than the priority band. Therefore, to choose a band other than the 

# priority, we require that the priority S/N is below the minimum threshold and that the 

# difference between the priority and highest S/N is larger than the difference threshold. 

# 

# For pseudo code objects we always choose the first band in the priority list. 

bestRecord = None 

bestFlagKeys = None 

if hasPseudoFilter: 

bestRecord = priorityRecord 

bestFlagKeys = priorityFlagKeys 

elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and 

maxSNRecord is not None): 

bestRecord = maxSNRecord 

bestFlagKeys = maxSNFlagKeys 

elif priorityRecord is not None: 

bestRecord = priorityRecord 

bestFlagKeys = priorityFlagKeys 

 

if bestRecord is not None and bestFlagKeys is not None: 

outputRecord = mergedCatalog.addNew() 

outputRecord.assign(bestRecord, self.schemaMapper) 

outputRecord.set(bestFlagKeys.output, True) 

else: # if we didn't find any records 

raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" % 

inputRecord.getId()) 

 

# more checking for sane inputs, since zip silently iterates over the smallest sequence 

for inputCatalog in orderedCatalogs: 

if len(mergedCatalog) != len(inputCatalog): 

raise ValueError("Mismatch between catalog sizes: %s != %s" % 

(len(mergedCatalog), len(orderedCatalogs))) 

 

return mergedCatalog 

 

def write(self, patchRef, catalog): 

"""! 

@brief Write the output. 

 

@param[in] patchRef data reference for patch 

@param[in] catalog catalog 

 

We write as the dataset provided by the 'outputDataset' 

class variable. 

""" 

patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset) 

# since the filter isn't actually part of the data ID for the dataset we're saving, 

# it's confusing to see it in the log message, even if the butler simply ignores it. 

mergeDataId = patchRef.dataId.copy() 

del mergeDataId["filter"] 

self.log.info("Wrote merged catalog: %s" % (mergeDataId,))