lsst.pipe.tasks 21.0.0-178-g80f1dd77+d3f2e76fa8
ingest.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import os
23import shutil
24import sqlite3
25import sys
26from fnmatch import fnmatch
27from glob import glob
28from contextlib import contextmanager
29
30from astro_metadata_translator import fix_header
31from lsst.pex.config import Config, Field, DictField, ListField, ConfigurableField
32from lsst.afw.fits import readMetadata
33from lsst.pipe.base import Task, InputOnlyArgumentParser
34from lsst.afw.fits import DEFAULT_HDU
35
36
37class IngestArgumentParser(InputOnlyArgumentParser):
38 """Argument parser to support ingesting images into the image repository"""
39
40 def __init__(self, *args, **kwargs):
41 super(IngestArgumentParser, self).__init__(*args, **kwargs)
42 self.add_argument("-n", "--dry-run", dest="dryrun", action="store_true", default=False,
43 help="Don't perform any action?")
44 self.add_argument("--mode", choices=["move", "copy", "link", "skip"], default="link",
45 help="Mode of delivering the files to their destination")
46 self.add_argument("--create", action="store_true", help="Create new registry (clobber old)?")
47 self.add_argument("--ignore-ingested", dest="ignoreIngested", action="store_true",
48 help="Don't register files that have already been registered")
49 self.add_id_argument("--badId", "raw", "Data identifier for bad data", doMakeDataRefList=False)
50 self.add_argument("--badFile", nargs="*", default=[],
51 help="Names of bad files (no path; wildcards allowed)")
52 self.add_argument("files", nargs="+", help="Names of file")
53
54
55class ParseConfig(Config):
56 """Configuration for ParseTask"""
57 translation = DictField(keytype=str, itemtype=str, default={},
58 doc="Translation table for property --> header")
59 translators = DictField(keytype=str, itemtype=str, default={},
60 doc="Properties and name of translator method")
61 defaults = DictField(keytype=str, itemtype=str, default={},
62 doc="Default values if header is not present")
63 hdu = Field(dtype=int, default=DEFAULT_HDU, doc="HDU to read for metadata")
64 extnames = ListField(dtype=str, default=[], doc="Extension names to search for")
65
66
67class ParseTask(Task):
68 """Task that will parse the filename and/or its contents to get the required information
69 for putting the file in the correct location and populating the registry."""
70 ConfigClass = ParseConfig
71 translator_class = None
72 """Metadata translation support (astro_metadata_translator.MetadataTranslator).
73
74 Notes
75 -----
76 The default of `None` will attempt to guess the correct translator,
77 but specifying one (e.g., in obs-package specific ingest) will be faster.
78 """
79
80 def getInfo(self, filename):
81 """Get information about the image from the filename and its contents
82
83 Here, we open the image and parse the header, but one could also look at the filename itself
84 and derive information from that, or set values from the configuration.
85
86 Parameters
87 ----------
88 filename : `str`
89 Name of file to inspect
90
91 Returns
92 -------
93 phuInfo : `dict`
94 File properties
95 infoList : `list`
96 List of file properties for each extension
97 """
98 md = readMetadata(filename, self.config.hdu)
99 fix_header(md, translator_class=self.translator_classtranslator_class)
100 phuInfo = self.getInfoFromMetadatagetInfoFromMetadata(md)
101 if len(self.config.extnames) == 0:
102 # No extensions to worry about
103 return phuInfo, [phuInfo]
104 # Look in the provided extensions
105 extnames = set(self.config.extnames)
106 extnum = 0
107 infoList = []
108 while len(extnames) > 0:
109 extnum += 1
110 try:
111 md = readMetadata(filename, extnum)
112 fix_header(md, translator_class=self.translator_classtranslator_class)
113 except Exception as e:
114 self.log.warning("Error reading %s extensions %s: %s", filename, extnames, e)
115 break
116 ext = self.getExtensionNamegetExtensionName(md)
117 if ext in extnames:
118 hduInfo = self.getInfoFromMetadatagetInfoFromMetadata(md, info=phuInfo.copy())
119 # We need the HDU number when registering MEF files.
120 hduInfo["hdu"] = extnum
121 infoList.append(hduInfo)
122 extnames.discard(ext)
123 return phuInfo, infoList
124
125 @staticmethod
127 """ Get the name of a FITS extension.
128
129 Parameters
130 ----------
132 FITS header metadata.
133
134 Returns
135 -------
136 result : `str` or `None`
137 The string from the EXTNAME header card if it exists. None otherwise.
138 """
139 try:
140 ext = md["EXTNAME"]
141 except KeyError:
142 ext = None
143 return ext
144
145 def getInfoFromMetadata(self, md, info=None):
146 """Attempt to pull the desired information out of the header
147
148 This is done through two mechanisms:
149 * translation: a property is set directly from the relevant header keyword
150 * translator: a property is set with the result of calling a method
151
152 The translator methods receive the header metadata and should return the
153 appropriate value, or None if the value cannot be determined.
154
155 @param md FITS header
156 @param info File properties, to be supplemented
157 @return info
158 """
159 if info is None:
160 info = {}
161 for p, h in self.config.translation.items():
162 value = md.get(h, None)
163 if value is not None:
164 if isinstance(value, str):
165 value = value.strip()
166 info[p] = value
167 elif p in self.config.defaults:
168 info[p] = self.config.defaults[p]
169 else:
170 self.log.warning("Unable to find value for %s (derived from %s)", p, h)
171 for p, t in self.config.translators.items():
172 func = getattr(self, t)
173 try:
174 value = func(md)
175 except Exception as e:
176 self.log.warning("%s failed to translate %s: %s", t, p, e)
177 value = None
178 if value is not None:
179 info[p] = value
180 return info
181
182 def translate_date(self, md):
183 """Convert a full DATE-OBS to a mere date
184
185 Besides being an example of a translator, this is also generally useful.
186 It will only be used if listed as a translator in the configuration.
187 """
188 date = md.getScalar("DATE-OBS").strip()
189 c = date.find("T")
190 if c > 0:
191 date = date[:c]
192 return date
193
194 def translate_filter(self, md):
195 """Translate a full filter description into a mere filter name
196
197 Besides being an example of a translator, this is also generally useful.
198 It will only be used if listed as a translator in the configuration.
199 """
200 filterName = md.getScalar("FILTER").strip()
201 filterName = filterName.strip()
202 c = filterName.find(" ")
203 if c > 0:
204 filterName = filterName[:c]
205 return filterName
206
207 def getDestination(self, butler, info, filename):
208 """Get destination for the file
209
210 @param butler Data butler
211 @param info File properties, used as dataId for the butler
212 @param filename Input filename
213 @return Destination filename
214 """
215 raw = butler.get("raw_filename", info)[0]
216 # Ensure filename is devoid of cfitsio directions about HDUs
217 c = raw.find("[")
218 if c > 0:
219 raw = raw[:c]
220 return raw
221
222
223class RegisterConfig(Config):
224 """Configuration for the RegisterTask"""
225 table = Field(dtype=str, default="raw", doc="Name of table")
226 columns = DictField(keytype=str, itemtype=str, doc="List of columns for raw table, with their types",
227 itemCheck=lambda x: x in ("text", "int", "double"),
228 default={'object': 'text',
229 'visit': 'int',
230 'ccd': 'int',
231 'filter': 'text',
232 'date': 'text',
233 'taiObs': 'text',
234 'expTime': 'double',
235 },
236 )
237 unique = ListField(dtype=str, doc="List of columns to be declared unique for the table",
238 default=["visit", "ccd"])
239 visit = ListField(dtype=str, default=["visit", "object", "date", "filter"],
240 doc="List of columns for raw_visit table")
241 ignore = Field(dtype=bool, default=False, doc="Ignore duplicates in the table?")
242 permissions = Field(dtype=int, default=0o664, doc="Permissions mode for registry; 0o664 = rw-rw-r--")
243
244
246 """Context manager to provide a registry
247 """
248
249 def __init__(self, registryName, createTableFunc, forceCreateTables, permissions):
250 """Construct a context manager
251
252 @param registryName: Name of registry file
253 @param createTableFunc: Function to create tables
254 @param forceCreateTables: Force the (re-)creation of tables?
255 @param permissions: Permissions to set on database file
256 """
257 self.connconn = sqlite3.connect(registryName)
258 os.chmod(registryName, permissions)
259 createTableFunc(self.connconn, forceCreateTables=forceCreateTables)
260
261 def __enter__(self):
262 """Provide the 'as' value"""
263 return self.connconn
264
265 def __exit__(self, excType, excValue, traceback):
266 self.connconn.commit()
267 self.connconn.close()
268 return False # Don't suppress any exceptions
269
270
271@contextmanager
273 """A context manager that doesn't provide any context
274
275 Useful for dry runs where we don't want to actually do anything real.
276 """
277 yield
278
279
280class RegisterTask(Task):
281 """Task that will generate the registry for the Mapper"""
282 ConfigClass = RegisterConfig
283 placeHolder = '?' # Placeholder for parameter substitution; this value suitable for sqlite3
284 typemap = {'text': str, 'int': int, 'double': float} # Mapping database type --> python type
285
286 def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"):
287 """Open the registry and return the connection handle.
288
289 @param directory Directory in which the registry file will be placed
290 @param create Clobber any existing registry and create a new one?
291 @param dryrun Don't do anything permanent?
292 @param name Filename of the registry
293 @return Database connection
294 """
295 if dryrun:
296 return fakeContext()
297
298 registryName = os.path.join(directory, name)
299 context = RegistryContext(registryName, self.createTablecreateTable, create, self.config.permissions)
300 return context
301
302 def createTable(self, conn, table=None, forceCreateTables=False):
303 """Create the registry tables
304
305 One table (typically 'raw') contains information on all files, and the
306 other (typically 'raw_visit') contains information on all visits.
307
308 @param conn Database connection
309 @param table Name of table to create in database
310 """
311 cursor = conn.cursor()
312 if table is None:
313 table = self.config.table
314 cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % table
315 cursor.execute(cmd)
316 if cursor.fetchone() and not forceCreateTables: # Assume if we get an answer the table exists
317 self.log.info('Table "%s" exists. Skipping creation', table)
318 return
319 else:
320 cmd = "drop table if exists %s" % table
321 cursor.execute(cmd)
322 cmd = "drop table if exists %s_visit" % table
323 cursor.execute(cmd)
324
325 cmd = "create table %s (id integer primary key autoincrement, " % table
326 cmd += ",".join([("%s %s" % (col, colType)) for col, colType in self.config.columns.items()])
327 if len(self.config.unique) > 0:
328 cmd += ", unique(" + ",".join(self.config.unique) + ")"
329 cmd += ")"
330 cursor.execute(cmd)
331
332 cmd = "create table %s_visit (" % table
333 cmd += ",".join([("%s %s" % (col, self.config.columns[col])) for col in self.config.visit])
334 cmd += ", unique(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
335 cmd += ")"
336 cursor.execute(cmd)
337
338 conn.commit()
339
340 def check(self, conn, info, table=None):
341 """Check for the presence of a row already
342
343 Not sure this is required, given the 'ignore' configuration option.
344 """
345 if table is None:
346 table = self.config.table
347 if self.config.ignore or len(self.config.unique) == 0:
348 return False # Our entry could already be there, but we don't care
349 cursor = conn.cursor()
350 sql = "SELECT COUNT(*) FROM %s WHERE " % table
351 sql += " AND ".join(["%s = %s" % (col, self.placeHolderplaceHolder) for col in self.config.unique])
352 values = [self.typemaptypemap[self.config.columns[col]](info[col]) for col in self.config.unique]
353
354 cursor.execute(sql, values)
355 if cursor.fetchone()[0] > 0:
356 return True
357 return False
358
359 def addRow(self, conn, info, dryrun=False, create=False, table=None):
360 """Add a row to the file table (typically 'raw').
361
362 @param conn Database connection
363 @param info File properties to add to database
364 @param table Name of table in database
365 """
366 with conn:
367 if table is None:
368 table = self.config.table
369 ignoreClause = ""
370 if self.config.ignore:
371 ignoreClause = " OR IGNORE"
372 sql = "INSERT%s INTO %s (%s) VALUES (" % (ignoreClause, table, ",".join(self.config.columns))
373 sql += ",".join([self.placeHolderplaceHolder] * len(self.config.columns)) + ")"
374 values = [self.typemaptypemap[tt](info[col]) for col, tt in self.config.columns.items()]
375
376 if dryrun:
377 print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
378 else:
379 conn.cursor().execute(sql, values)
380
381 sql = "INSERT OR IGNORE INTO %s_visit VALUES (" % table
382 sql += ",".join([self.placeHolderplaceHolder] * len(self.config.visit)) + ")"
383 values = [self.typemaptypemap[self.config.columns[col]](info[col]) for col in self.config.visit]
384
385 if dryrun:
386 print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
387 else:
388 conn.cursor().execute(sql, values)
389
390
391class IngestConfig(Config):
392 """Configuration for IngestTask"""
393 parse = ConfigurableField(target=ParseTask, doc="File parsing")
394 register = ConfigurableField(target=RegisterTask, doc="Registry entry")
395 allowError = Field(dtype=bool, default=False, doc="Allow error in ingestion?")
396 clobber = Field(dtype=bool, default=False, doc="Clobber existing file?")
397
398
399class IngestError(RuntimeError):
400 def __init__(self, message, pathname, position):
401 super().__init__(message)
402 self.pathnamepathname = pathname
403 self.positionposition = position
404
405
406class IngestTask(Task):
407 """Task that will ingest images into the data repository"""
408 ConfigClass = IngestConfig
409 ArgumentParser = IngestArgumentParser
410 _DefaultName = "ingest"
411
412 def __init__(self, *args, **kwargs):
413 super(IngestTask, self).__init__(*args, **kwargs)
414 self.makeSubtask("parse")
415 self.makeSubtask("register")
416
417 @classmethod
418 def _parse(cls):
419 """Parse the command-line arguments and return them along with a Task
420 instance."""
421 config = cls.ConfigClassConfigClass()
422 parser = cls.ArgumentParserArgumentParser(name=cls._DefaultName_DefaultName)
423 args = parser.parse_args(config)
424 task = cls(config=args.config)
425 return task, args
426
427 @classmethod
428 def parseAndRun(cls):
429 """Parse the command-line arguments and run the Task."""
430 task, args = cls._parse_parse()
431 task.run(args)
432
433 @classmethod
434 def prepareTask(cls, root=None, dryrun=False, mode="move", create=False,
435 ignoreIngested=False):
436 """Prepare for running the task repeatedly with `ingestFiles`.
437
438 Saves the parsed arguments, including the Butler and log, as a
439 private instance variable.
440
441 Parameters
442 ----------
443 root : `str`, optional
444 Repository root pathname. If None, run the Task using the
445 command line arguments, ignoring all other arguments below.
446 dryrun : `bool`, optional
447 If True, don't perform any action; log what would have happened.
448 mode : `str`, optional
449 How files are delivered to their destination. Default is "move",
450 unlike the command-line default of "link".
451 create : `bool`, optional
452 If True, create a new registry, clobbering any old one present.
453 ignoreIngested : `bool`, optional
454 If True, do not complain if the file is already present in the
455 registry (and do nothing else).
456
457 Returns
458 -------
459 task : `IngestTask`
460 If `root` was provided, the IngestTask instance
461 """
462 sys.argv = ["IngestTask"]
463 sys.argv.append(root)
464 if dryrun:
465 sys.argv.append("--dry-run")
466 sys.argv.append("--mode")
467 sys.argv.append(mode)
468 if create:
469 sys.argv.append("--create")
470 if ignoreIngested:
471 sys.argv.append("--ignore-ingested")
472 sys.argv.append("__fakefile__") # needed for parsing, not used
473
474 task, args = cls._parse_parse()
475 task._args = args
476 return task
477
478 def ingest(self, infile, outfile, mode="move", dryrun=False):
479 """Ingest a file into the image repository.
480
481 @param infile Name of input file
482 @param outfile Name of output file (file in repository)
483 @param mode Mode of ingest (copy/link/move/skip)
484 @param dryrun Only report what would occur?
485 @param Success boolean
486 """
487 if mode == "skip":
488 return True
489 if dryrun:
490 self.log.info("Would %s from %s to %s", mode, infile, outfile)
491 return True
492 try:
493 outdir = os.path.dirname(outfile)
494 if not os.path.isdir(outdir):
495 try:
496 os.makedirs(outdir)
497 except OSError as exc:
498 # Silently ignore mkdir failures due to race conditions
499 if not os.path.isdir(outdir):
500 raise RuntimeError(f"Failed to create directory {outdir}") from exc
501 if os.path.lexists(outfile):
502 if self.config.clobber:
503 os.unlink(outfile)
504 else:
505 raise RuntimeError("File %s already exists; consider --config clobber=True" % outfile)
506
507 if mode == "copy":
508 assertCanCopy(infile, outfile)
509 shutil.copyfile(infile, outfile)
510 elif mode == "link":
511 if os.path.exists(outfile):
512 if os.path.samefile(infile, outfile):
513 self.log.debug("Already linked %s to %s: ignoring", infile, outfile)
514 else:
515 self.log.warning("%s already has a file at the target location (%s): ignoring "
516 "(set clobber=True to overwrite)", infile, outfile)
517 return False
518 os.symlink(os.path.abspath(infile), outfile)
519 elif mode == "move":
520 assertCanCopy(infile, outfile)
521 shutil.move(infile, outfile)
522 else:
523 raise AssertionError("Unknown mode: %s" % mode)
524 self.log.info("%s --<%s>--> %s", infile, mode, outfile)
525 except Exception as e:
526 self.log.warning("Failed to %s %s to %s: %s", mode, infile, outfile, e)
527 if not self.config.allowError:
528 raise RuntimeError(f"Failed to {mode} {infile} to {outfile}") from e
529 return False
530 return True
531
532 def isBadFile(self, filename, badFileList):
533 """Return whether the file qualifies as bad
534
535 We match against the list of bad file patterns.
536 """
537 filename = os.path.basename(filename)
538 if not badFileList:
539 return False
540 for badFile in badFileList:
541 if fnmatch(filename, badFile):
542 return True
543 return False
544
545 def isBadId(self, info, badIdList):
546 """Return whether the file information qualifies as bad
547
548 We match against the list of bad data identifiers.
549 """
550 if not badIdList:
551 return False
552 for badId in badIdList:
553 if all(info[key] == value for key, value in badId.items()):
554 return True
555 return False
556
557 def expandFiles(self, fileNameList):
558 """!Expand a set of filenames and globs, returning a list of filenames
559
560 @param fileNameList A list of files and glob patterns
561
562 N.b. globs obey Posix semantics, so a pattern that matches nothing is returned unchanged
563 """
564 filenameList = []
565 for globPattern in fileNameList:
566 files = glob(globPattern)
567
568 if not files: # posix behaviour is to return pattern unchanged
569 self.log.warning("%s doesn't match any file", globPattern)
570 continue
571
572 filenameList.extend(files)
573
574 return filenameList
575
576 def runFile(self, infile, registry, args, pos):
577 """!Examine and ingest a single file
578
579 @param infile: File to process
580 @param registry: Registry into which to insert Butler metadata, or None
581 @param args: Parsed command-line arguments
582 @param pos: Position number of this file in the input list
583 @return parsed information from FITS HDUs if registry is None; or None
584 """
585 if self.isBadFileisBadFile(infile, args.badFile):
586 self.log.info("Skipping declared bad file %s", infile)
587 return None
588 try:
589 fileInfo, hduInfoList = self.parse.getInfo(infile)
590 except Exception as e:
591 if not self.config.allowError:
592 raise RuntimeError(f"Error parsing {infile}") from e
593 self.log.warning("Error parsing %s (%s); skipping", infile, e)
594 return None
595 if self.isBadIdisBadId(fileInfo, args.badId.idList):
596 self.log.info("Skipping declared bad file %s: %s", infile, fileInfo)
597 return None
598 if registry is not None and self.register.check(registry, fileInfo):
599 if args.ignoreIngested:
600 return None
601 self.log.warning("%s: already ingested: %s", infile, fileInfo)
602 outfile = self.parse.getDestination(args.butler, fileInfo, infile)
603 if not self.ingestingest(infile, outfile, mode=args.mode, dryrun=args.dryrun):
604 return None
605 if hduInfoList is None:
606 return None
607 if registry is None:
608 return hduInfoList
609 for info in hduInfoList:
610 try:
611 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
612 except Exception as exc:
613 raise IngestError(f"Failed to register file {infile}", infile, pos) from exc
614 return None # No further registration should be performed
615
616 def run(self, args):
617 """Ingest all specified files and add them to the registry"""
618 filenameList = self.expandFilesexpandFiles(args.files)
619 root = args.input
620 context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
621 with context as registry:
622 for pos in range(len(filenameList)):
623 infile = filenameList[pos]
624 try:
625 self.runFilerunFile(infile, registry, args, pos)
626 except Exception as exc:
627 self.log.warning("Failed to ingest file %s: %s", infile, exc)
628 if not self.config.allowError:
629 raise IngestError(f"Failed to ingest file {infile}", infile, pos) from exc
630 continue
631
632 def ingestFiles(self, fileList):
633 """Ingest specified file or list of files and add them to the registry.
634
635 This method can only be called if `prepareTask` was used.
636
637 Parameters
638 ----------
639 fileList : `str` or `list` [`str`]
640 Pathname or list of pathnames of files to ingest.
641 """
642 if not hasattr(self, "_args"):
643 raise RuntimeError("Task not created with prepareTask")
644 if isinstance(fileList, str):
645 fileList = [fileList]
646 self._args.files = fileList
647 self.runrun(self._args)
648
649
650def assertCanCopy(fromPath, toPath):
651 """Can I copy a file? Raise an exception is space constraints not met.
652
653 @param fromPath Path from which the file is being copied
654 @param toPath Path to which the file is being copied
655 """
656 req = os.stat(fromPath).st_size
657 st = os.statvfs(os.path.dirname(toPath))
658 avail = st.f_bavail * st.f_frsize
659 if avail < req:
660 raise RuntimeError("Insufficient space: %d vs %d" % (req, avail))
def __init__(self, *args, **kwargs)
Definition: ingest.py:40
def __init__(self, message, pathname, position)
Definition: ingest.py:400
def expandFiles(self, fileNameList)
Expand a set of filenames and globs, returning a list of filenames.
Definition: ingest.py:557
def runFile(self, infile, registry, args, pos)
Examine and ingest a single file.
Definition: ingest.py:576
def ingestFiles(self, fileList)
Definition: ingest.py:632
def __init__(self, *args, **kwargs)
Definition: ingest.py:412
def prepareTask(cls, root=None, dryrun=False, mode="move", create=False, ignoreIngested=False)
Definition: ingest.py:435
def isBadFile(self, filename, badFileList)
Definition: ingest.py:532
def isBadId(self, info, badIdList)
Definition: ingest.py:545
def ingest(self, infile, outfile, mode="move", dryrun=False)
Definition: ingest.py:478
def getInfoFromMetadata(self, md, info=None)
Definition: ingest.py:145
def translate_date(self, md)
Definition: ingest.py:182
def getDestination(self, butler, info, filename)
Definition: ingest.py:207
def getInfo(self, filename)
Definition: ingest.py:80
def translate_filter(self, md)
Definition: ingest.py:194
def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3")
Definition: ingest.py:286
def check(self, conn, info, table=None)
Definition: ingest.py:340
def addRow(self, conn, info, dryrun=False, create=False, table=None)
Definition: ingest.py:359
def createTable(self, conn, table=None, forceCreateTables=False)
Definition: ingest.py:302
def __exit__(self, excType, excValue, traceback)
Definition: ingest.py:265
def __init__(self, registryName, createTableFunc, forceCreateTables, permissions)
Definition: ingest.py:249
def assertCanCopy(fromPath, toPath)
Definition: ingest.py:650