lsst.pipe.tasks  19.0.0-14-gbb28fe44+1
ingest.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import os
23 import shutil
24 import sqlite3
25 import sys
26 import tempfile
27 from fnmatch import fnmatch
28 from glob import glob
29 from contextlib import contextmanager
30 
31 from lsst.pex.config import Config, Field, DictField, ListField, ConfigurableField
33 from lsst.afw.fits import readMetadata
34 from lsst.pipe.base import Task, InputOnlyArgumentParser
35 from lsst.afw.fits import DEFAULT_HDU
36 
37 
38 class IngestArgumentParser(InputOnlyArgumentParser):
39  """Argument parser to support ingesting images into the image repository"""
40 
41  def __init__(self, *args, **kwargs):
42  super(IngestArgumentParser, self).__init__(*args, **kwargs)
43  self.add_argument("-n", "--dry-run", dest="dryrun", action="store_true", default=False,
44  help="Don't perform any action?")
45  self.add_argument("--mode", choices=["move", "copy", "link", "skip"], default="link",
46  help="Mode of delivering the files to their destination")
47  self.add_argument("--create", action="store_true", help="Create new registry (clobber old)?")
48  self.add_argument("--ignore-ingested", dest="ignoreIngested", action="store_true",
49  help="Don't register files that have already been registered")
50  self.add_id_argument("--badId", "raw", "Data identifier for bad data", doMakeDataRefList=False)
51  self.add_argument("--badFile", nargs="*", default=[],
52  help="Names of bad files (no path; wildcards allowed)")
53  self.add_argument("files", nargs="+", help="Names of file")
54 
55 
56 class ParseConfig(Config):
57  """Configuration for ParseTask"""
58  translation = DictField(keytype=str, itemtype=str, default={},
59  doc="Translation table for property --> header")
60  translators = DictField(keytype=str, itemtype=str, default={},
61  doc="Properties and name of translator method")
62  defaults = DictField(keytype=str, itemtype=str, default={},
63  doc="Default values if header is not present")
64  hdu = Field(dtype=int, default=DEFAULT_HDU, doc="HDU to read for metadata")
65  extnames = ListField(dtype=str, default=[], doc="Extension names to search for")
66 
67 
68 class ParseTask(Task):
69  """Task that will parse the filename and/or its contents to get the required information
70  for putting the file in the correct location and populating the registry."""
71  ConfigClass = ParseConfig
72 
73  def getInfo(self, filename):
74  """Get information about the image from the filename and its contents
75 
76  Here, we open the image and parse the header, but one could also look at the filename itself
77  and derive information from that, or set values from the configuration.
78 
79  @param filename Name of file to inspect
80  @return File properties; list of file properties for each extension
81  """
82  md = readMetadata(filename, self.config.hdu)
83  phuInfo = self.getInfoFromMetadata(md)
84  if len(self.config.extnames) == 0:
85  # No extensions to worry about
86  return phuInfo, [phuInfo]
87  # Look in the provided extensions
88  extnames = set(self.config.extnames)
89  extnum = 0
90  infoList = []
91  while len(extnames) > 0:
92  extnum += 1
93  try:
94  md = readMetadata(filename, extnum)
95  except Exception as e:
96  self.log.warn("Error reading %s extensions %s: %s" % (filename, extnames, e))
97  break
98  ext = self.getExtensionName(md)
99  if ext in extnames:
100  hduInfo = self.getInfoFromMetadata(md, info=phuInfo.copy())
101  # We need the HDU number when registering MEF files.
102  hduInfo["hdu"] = extnum
103  infoList.append(hduInfo)
104  extnames.discard(ext)
105  return phuInfo, infoList
106 
107  @staticmethod
109  """ Get the name of an extension.
110  @param md: PropertySet like one obtained from lsst.afw.fits.readMetadata)
111  @return Name of the extension if it exists. None otherwise.
112  """
113  try:
114  # This returns a tuple
115  ext = md.getScalar("EXTNAME")
116  return ext[1]
118  return None
119 
120  def getInfoFromMetadata(self, md, info=None):
121  """Attempt to pull the desired information out of the header
122 
123  This is done through two mechanisms:
124  * translation: a property is set directly from the relevant header keyword
125  * translator: a property is set with the result of calling a method
126 
127  The translator methods receive the header metadata and should return the
128  appropriate value, or None if the value cannot be determined.
129 
130  @param md FITS header
131  @param info File properties, to be supplemented
132  @return info
133  """
134  if info is None:
135  info = {}
136  for p, h in self.config.translation.items():
137  value = md.get(h, None)
138  if value is not None:
139  if isinstance(value, str):
140  value = value.strip()
141  info[p] = value
142  elif p in self.config.defaults:
143  info[p] = self.config.defaults[p]
144  else:
145  self.log.warn("Unable to find value for %s (derived from %s)" % (p, h))
146  for p, t in self.config.translators.items():
147  func = getattr(self, t)
148  try:
149  value = func(md)
150  except Exception as e:
151  self.log.warn("%s failed to translate %s: %s", t, p, e)
152  value = None
153  if value is not None:
154  info[p] = value
155  return info
156 
157  def translate_date(self, md):
158  """Convert a full DATE-OBS to a mere date
159 
160  Besides being an example of a translator, this is also generally useful.
161  It will only be used if listed as a translator in the configuration.
162  """
163  date = md.getScalar("DATE-OBS").strip()
164  c = date.find("T")
165  if c > 0:
166  date = date[:c]
167  return date
168 
169  def translate_filter(self, md):
170  """Translate a full filter description into a mere filter name
171 
172  Besides being an example of a translator, this is also generally useful.
173  It will only be used if listed as a translator in the configuration.
174  """
175  filterName = md.getScalar("FILTER").strip()
176  filterName = filterName.strip()
177  c = filterName.find(" ")
178  if c > 0:
179  filterName = filterName[:c]
180  return filterName
181 
182  def getDestination(self, butler, info, filename):
183  """Get destination for the file
184 
185  @param butler Data butler
186  @param info File properties, used as dataId for the butler
187  @param filename Input filename
188  @return Destination filename
189  """
190  raw = butler.get("raw_filename", info)[0]
191  # Ensure filename is devoid of cfitsio directions about HDUs
192  c = raw.find("[")
193  if c > 0:
194  raw = raw[:c]
195  return raw
196 
197 
198 class RegisterConfig(Config):
199  """Configuration for the RegisterTask"""
200  table = Field(dtype=str, default="raw", doc="Name of table")
201  columns = DictField(keytype=str, itemtype=str, doc="List of columns for raw table, with their types",
202  itemCheck=lambda x: x in ("text", "int", "double"),
203  default={'object': 'text',
204  'visit': 'int',
205  'ccd': 'int',
206  'filter': 'text',
207  'date': 'text',
208  'taiObs': 'text',
209  'expTime': 'double',
210  },
211  )
212  unique = ListField(dtype=str, doc="List of columns to be declared unique for the table",
213  default=["visit", "ccd"])
214  visit = ListField(dtype=str, default=["visit", "object", "date", "filter"],
215  doc="List of columns for raw_visit table")
216  ignore = Field(dtype=bool, default=False, doc="Ignore duplicates in the table?")
217  permissions = Field(dtype=int, default=0o664, doc="Permissions mode for registry; 0o664 = rw-rw-r--")
218 
219 
221  """Context manager to provide a registry
222 
223  An existing registry is copied, so that it may continue
224  to be used while we add to this new registry. Finally,
225  the new registry is moved into the right place.
226  """
227 
228  def __init__(self, registryName, createTableFunc, forceCreateTables, permissions):
229  """Construct a context manager
230 
231  @param registryName: Name of registry file
232  @param createTableFunc: Function to create tables
233  @param forceCreateTables: Force the (re-)creation of tables?
234  @param permissions: Permissions to set on database file
235  """
236  self.registryName = registryName
237  self.permissions = permissions
238 
239  updateFile = tempfile.NamedTemporaryFile(prefix=registryName, dir=os.path.dirname(self.registryName),
240  delete=False)
241  self.updateName = updateFile.name
242 
243  if os.path.exists(registryName):
244  assertCanCopy(registryName, self.updateName)
245  os.chmod(self.updateName, os.stat(registryName).st_mode)
246  shutil.copyfile(registryName, self.updateName)
247 
248  self.conn = sqlite3.connect(self.updateName)
249  createTableFunc(self.conn, forceCreateTables=forceCreateTables)
250  os.chmod(self.updateName, self.permissions)
251 
252  def __enter__(self):
253  """Provide the 'as' value"""
254  return self.conn
255 
256  def __exit__(self, excType, excValue, traceback):
257  self.conn.commit()
258  self.conn.close()
259  if excType is None:
261  if os.path.exists(self.registryName):
262  os.unlink(self.registryName)
263  os.rename(self.updateName, self.registryName)
264  os.chmod(self.registryName, self.permissions)
265  return False # Don't suppress any exceptions
266 
267 
268 @contextmanager
270  """A context manager that doesn't provide any context
271 
272  Useful for dry runs where we don't want to actually do anything real.
273  """
274  yield
275 
276 
277 class RegisterTask(Task):
278  """Task that will generate the registry for the Mapper"""
279  ConfigClass = RegisterConfig
280  placeHolder = '?' # Placeholder for parameter substitution; this value suitable for sqlite3
281  typemap = {'text': str, 'int': int, 'double': float} # Mapping database type --> python type
282 
283  def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"):
284  """Open the registry and return the connection handle.
285 
286  @param directory Directory in which the registry file will be placed
287  @param create Clobber any existing registry and create a new one?
288  @param dryrun Don't do anything permanent?
289  @param name Filename of the registry
290  @return Database connection
291  """
292  if dryrun:
293  return fakeContext()
294 
295  registryName = os.path.join(directory, name)
296  context = RegistryContext(registryName, self.createTable, create, self.config.permissions)
297  return context
298 
299  def createTable(self, conn, table=None, forceCreateTables=False):
300  """Create the registry tables
301 
302  One table (typically 'raw') contains information on all files, and the
303  other (typically 'raw_visit') contains information on all visits.
304 
305  @param conn Database connection
306  @param table Name of table to create in database
307  """
308  cursor = conn.cursor()
309  if table is None:
310  table = self.config.table
311  cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % table
312  cursor.execute(cmd)
313  if cursor.fetchone() and not forceCreateTables: # Assume if we get an answer the table exists
314  self.log.info('Table "%s" exists. Skipping creation' % table)
315  return
316  else:
317  cmd = "drop table if exists %s" % table
318  cursor.execute(cmd)
319  cmd = "drop table if exists %s_visit" % table
320  cursor.execute(cmd)
321 
322  cmd = "create table %s (id integer primary key autoincrement, " % table
323  cmd += ",".join([("%s %s" % (col, colType)) for col, colType in self.config.columns.items()])
324  if len(self.config.unique) > 0:
325  cmd += ", unique(" + ",".join(self.config.unique) + ")"
326  cmd += ")"
327  cursor.execute(cmd)
328 
329  cmd = "create table %s_visit (" % table
330  cmd += ",".join([("%s %s" % (col, self.config.columns[col])) for col in self.config.visit])
331  cmd += ", unique(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
332  cmd += ")"
333  cursor.execute(cmd)
334 
335  conn.commit()
336 
337  def check(self, conn, info, table=None):
338  """Check for the presence of a row already
339 
340  Not sure this is required, given the 'ignore' configuration option.
341  """
342  if table is None:
343  table = self.config.table
344  if self.config.ignore or len(self.config.unique) == 0:
345  return False # Our entry could already be there, but we don't care
346  cursor = conn.cursor()
347  sql = "SELECT COUNT(*) FROM %s WHERE " % table
348  sql += " AND ".join(["%s = %s" % (col, self.placeHolder) for col in self.config.unique])
349  values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.unique]
350 
351  cursor.execute(sql, values)
352  if cursor.fetchone()[0] > 0:
353  return True
354  return False
355 
356  def addRow(self, conn, info, dryrun=False, create=False, table=None):
357  """Add a row to the file table (typically 'raw').
358 
359  @param conn Database connection
360  @param info File properties to add to database
361  @param table Name of table in database
362  """
363  if table is None:
364  table = self.config.table
365  ignoreClause = ""
366  if self.config.ignore:
367  ignoreClause = " OR IGNORE"
368  sql = "INSERT%s INTO %s (%s) VALUES (" % (ignoreClause, table, ",".join(self.config.columns))
369  sql += ",".join([self.placeHolder] * len(self.config.columns)) + ")"
370  values = [self.typemap[tt](info[col]) for col, tt in self.config.columns.items()]
371 
372  if dryrun:
373  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
374  else:
375  conn.cursor().execute(sql, values)
376 
377  sql = "INSERT OR IGNORE INTO %s_visit VALUES (" % table
378  sql += ",".join([self.placeHolder] * len(self.config.visit)) + ")"
379  values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.visit]
380 
381  if dryrun:
382  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
383  else:
384  conn.cursor().execute(sql, values)
385 
386 
387 class IngestConfig(Config):
388  """Configuration for IngestTask"""
389  parse = ConfigurableField(target=ParseTask, doc="File parsing")
390  register = ConfigurableField(target=RegisterTask, doc="Registry entry")
391  allowError = Field(dtype=bool, default=False, doc="Allow error in ingestion?")
392  clobber = Field(dtype=bool, default=False, doc="Clobber existing file?")
393 
394 
395 class IngestError(RuntimeError):
396  def __init__(self, message, pathname, position):
397  super().__init__(message)
398  self.pathname = pathname
399  self.position = position
400 
401 
402 class IngestTask(Task):
403  """Task that will ingest images into the data repository"""
404  ConfigClass = IngestConfig
405  ArgumentParser = IngestArgumentParser
406  _DefaultName = "ingest"
407 
408  def __init__(self, *args, **kwargs):
409  super(IngestTask, self).__init__(*args, **kwargs)
410  self.makeSubtask("parse")
411  self.makeSubtask("register")
412 
413  @classmethod
414  def _parse(cls):
415  """Parse the command-line arguments and return them along with a Task
416  instance."""
417  config = cls.ConfigClass()
418  parser = cls.ArgumentParser(name=cls._DefaultName)
419  args = parser.parse_args(config)
420  task = cls(config=args.config)
421  return task, args
422 
423  @classmethod
424  def parseAndRun(cls):
425  """Parse the command-line arguments and run the Task."""
426  task, args = cls._parse()
427  task.run(args)
428 
429  @classmethod
430  def prepareTask(cls, root=None, dryrun=False, mode="move", create=False,
431  ignoreIngested=False):
432  """Prepare for running the task repeatedly with `ingestFiles`.
433 
434  Saves the parsed arguments, including the Butler and log, as a
435  private instance variable.
436 
437  Parameters
438  ----------
439  root : `str`, optional
440  Repository root pathname. If None, run the Task using the
441  command line arguments, ignoring all other arguments below.
442  dryrun : `bool`, optional
443  If True, don't perform any action; log what would have happened.
444  mode : `str`, optional
445  How files are delivered to their destination. Default is "move",
446  unlike the command-line default of "link".
447  create : `bool`, optional
448  If True, create a new registry, clobbering any old one present.
449  ignoreIngested : `bool`, optional
450  If True, do not complain if the file is already present in the
451  registry (and do nothing else).
452 
453  Returns
454  -------
455  task : `IngestTask`
456  If `root` was provided, the IngestTask instance
457  """
458  sys.argv = ["IngestTask"]
459  sys.argv.append(root)
460  if dryrun:
461  sys.argv.append("--dry-run")
462  sys.argv.append("--mode")
463  sys.argv.append(mode)
464  if create:
465  sys.argv.append("--create")
466  if ignoreIngested:
467  sys.argv.append("--ignore-ingested")
468  sys.argv.append("__fakefile__") # needed for parsing, not used
469 
470  task, args = cls._parse()
471  task._args = args
472  return task
473 
474  def ingest(self, infile, outfile, mode="move", dryrun=False):
475  """Ingest a file into the image repository.
476 
477  @param infile Name of input file
478  @param outfile Name of output file (file in repository)
479  @param mode Mode of ingest (copy/link/move/skip)
480  @param dryrun Only report what would occur?
481  @param Success boolean
482  """
483  if mode == "skip":
484  return True
485  if dryrun:
486  self.log.info("Would %s from %s to %s" % (mode, infile, outfile))
487  return True
488  try:
489  outdir = os.path.dirname(outfile)
490  if not os.path.isdir(outdir):
491  try:
492  os.makedirs(outdir)
493  except OSError as exc:
494  # Silently ignore mkdir failures due to race conditions
495  if not os.path.isdir(outdir):
496  raise RuntimeError(f"Failed to create directory {outdir}") from exc
497  if os.path.lexists(outfile):
498  if self.config.clobber:
499  os.unlink(outfile)
500  else:
501  raise RuntimeError("File %s already exists; consider --config clobber=True" % outfile)
502 
503  if mode == "copy":
504  assertCanCopy(infile, outfile)
505  shutil.copyfile(infile, outfile)
506  elif mode == "link":
507  os.symlink(os.path.abspath(infile), outfile)
508  elif mode == "move":
509  assertCanCopy(infile, outfile)
510  shutil.move(infile, outfile)
511  else:
512  raise AssertionError("Unknown mode: %s" % mode)
513  self.log.info("%s --<%s>--> %s" % (infile, mode, outfile))
514  except Exception as e:
515  self.log.warn("Failed to %s %s to %s: %s" % (mode, infile, outfile, e))
516  if not self.config.allowError:
517  raise RuntimeError(f"Failed to {mode} {infile} to {outfile}") from e
518  return False
519  return True
520 
521  def isBadFile(self, filename, badFileList):
522  """Return whether the file qualifies as bad
523 
524  We match against the list of bad file patterns.
525  """
526  filename = os.path.basename(filename)
527  if not badFileList:
528  return False
529  for badFile in badFileList:
530  if fnmatch(filename, badFile):
531  return True
532  return False
533 
534  def isBadId(self, info, badIdList):
535  """Return whether the file information qualifies as bad
536 
537  We match against the list of bad data identifiers.
538  """
539  if not badIdList:
540  return False
541  for badId in badIdList:
542  if all(info[key] == value for key, value in badId.items()):
543  return True
544  return False
545 
546  def expandFiles(self, fileNameList):
547  """!Expand a set of filenames and globs, returning a list of filenames
548 
549  @param fileNameList A list of files and glob patterns
550 
551  N.b. globs obey Posix semantics, so a pattern that matches nothing is returned unchanged
552  """
553  filenameList = []
554  for globPattern in fileNameList:
555  files = glob(globPattern)
556 
557  if not files: # posix behaviour is to return pattern unchanged
558  self.log.warn("%s doesn't match any file" % globPattern)
559  continue
560 
561  filenameList.extend(files)
562 
563  return filenameList
564 
565  def runFile(self, infile, registry, args):
566  """!Examine and ingest a single file
567 
568  @param infile: File to process
569  @param args: Parsed command-line arguments
570  @return parsed information from FITS HDUs or None
571  """
572  if self.isBadFile(infile, args.badFile):
573  self.log.info("Skipping declared bad file %s" % infile)
574  return None
575  try:
576  fileInfo, hduInfoList = self.parse.getInfo(infile)
577  except Exception as e:
578  if not self.config.allowError:
579  raise RuntimeError(f"Error parsing {infile}") from e
580  self.log.warn("Error parsing %s (%s); skipping" % (infile, e))
581  return None
582  if self.isBadId(fileInfo, args.badId.idList):
583  self.log.info("Skipping declared bad file %s: %s" % (infile, fileInfo))
584  return
585  if registry is not None and self.register.check(registry, fileInfo):
586  if args.ignoreIngested:
587  return None
588  self.log.warn("%s: already ingested: %s" % (infile, fileInfo))
589  outfile = self.parse.getDestination(args.butler, fileInfo, infile)
590  if not self.ingest(infile, outfile, mode=args.mode, dryrun=args.dryrun):
591  return None
592  return hduInfoList
593 
594  def run(self, args):
595  """Ingest all specified files and add them to the registry"""
596  filenameList = self.expandFiles(args.files)
597  root = args.input
598  context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
599  with context as registry:
600  for pos in range(len(filenameList)):
601  infile = filenameList[pos]
602  try:
603  hduInfoList = self.runFile(infile, registry, args)
604  except Exception as exc:
605  self.log.warn("Failed to ingest file %s: %s", infile, exc)
606  if not self.config.allowError:
607  raise IngestError(f"Failed to ingest file {infile}", infile, pos) from exc
608  continue
609  if hduInfoList is None:
610  continue
611  for info in hduInfoList:
612  try:
613  self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
614  except Exception as exc:
615  raise IngestError(f"Failed to register file {infile}", infile, pos) from exc
616 
617  def ingestFiles(self, fileList):
618  """Ingest specified file or list of files and add them to the registry.
619 
620  This method can only be called if `prepareTask` was used.
621 
622  Parameters
623  ----------
624  fileList : `str` or `list` [`str`]
625  Pathname or list of pathnames of files to ingest.
626  """
627  if not hasattr(self, "_args"):
628  raise RuntimeError("Task not created with prepareTask")
629  if isinstance(fileList, str):
630  fileList = [fileList]
631  self._args.files = fileList
632  self.run(self._args)
633 
634 
635 def assertCanCopy(fromPath, toPath):
636  """Can I copy a file? Raise an exception is space constraints not met.
637 
638  @param fromPath Path from which the file is being copied
639  @param toPath Path to which the file is being copied
640  """
641  req = os.stat(fromPath).st_size
642  st = os.statvfs(os.path.dirname(toPath))
643  avail = st.f_bavail * st.f_frsize
644  if avail < req:
645  raise RuntimeError("Insufficient space: %d vs %d" % (req, avail))
def ingest(self, infile, outfile, mode="move", dryrun=False)
Definition: ingest.py:474
def ingestFiles(self, fileList)
Definition: ingest.py:617
def __init__(self, message, pathname, position)
Definition: ingest.py:396
def translate_filter(self, md)
Definition: ingest.py:169
def expandFiles(self, fileNameList)
Expand a set of filenames and globs, returning a list of filenames.
Definition: ingest.py:546
def translate_date(self, md)
Definition: ingest.py:157
def __exit__(self, excType, excValue, traceback)
Definition: ingest.py:256
def getInfo(self, filename)
Definition: ingest.py:73
def getInfoFromMetadata(self, md, info=None)
Definition: ingest.py:120
def getDestination(self, butler, info, filename)
Definition: ingest.py:182
def runFile(self, infile, registry, args)
Examine and ingest a single file.
Definition: ingest.py:565
def isBadFile(self, filename, badFileList)
Definition: ingest.py:521
def __init__(self, registryName, createTableFunc, forceCreateTables, permissions)
Definition: ingest.py:228
def createTable(self, conn, table=None, forceCreateTables=False)
Definition: ingest.py:299
def assertCanCopy(fromPath, toPath)
Definition: ingest.py:635
def check(self, conn, info, table=None)
Definition: ingest.py:337
def __init__(self, args, kwargs)
Definition: ingest.py:41
def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3")
Definition: ingest.py:283
def prepareTask(cls, root=None, dryrun=False, mode="move", create=False, ignoreIngested=False)
Definition: ingest.py:431
def __init__(self, args, kwargs)
Definition: ingest.py:408
def addRow(self, conn, info, dryrun=False, create=False, table=None)
Definition: ingest.py:356
def isBadId(self, info, badIdList)
Definition: ingest.py:534