Tryag File Manager - Edit By Mr.SaHr

//lib/python2.7/site-packages/createrepo/__init__.py

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2009  Red Hat, Inc -
# written by seth vidal skvidal at fedoraproject.org

import os
import sys
import fnmatch
import time
import yumbased
import shutil
from  bz2 import BZ2File
from urlgrabber import grabber
import tempfile
import stat
import fcntl
import subprocess
from select import select

from yum import misc, Errors
from yum.repoMDObject import RepoMD, RepoData
from yum.sqlutils import executeSQL
from yum.packageSack import MetaSack
from yum.packages import YumAvailablePackage

import rpmUtils.transaction
from utils import _, errorprint, MDError, lzma, _available_compression
import readMetadata
try:
    import sqlite3 as sqlite
except ImportError:
    import sqlite

try:
    import sqlitecachec
except ImportError:
    pass

from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \
                  checksum_and_rename, split_list_into_equal_chunks
from utils import num_cpus_online
import deltarpms

__version__ = '0.9.9'

class MetaDataConfig(object):
    def __init__(self):
        self.quiet = False
        self.verbose = False
        self.profile = False
        self.excludes = []
        self.baseurl = None
        self.groupfile = None
        self.sumtype = 'sha256'
        self.pretty = False
        self.cachedir = None
        self.use_cache = False
        self.basedir = os.getcwd()
        self.checkts = False
        self.split = False
        self.update = False
        self.deltas = False # do the deltarpm thing
        # where to put the .drpms - defaults to 'drpms' inside 'repodata'
        self.deltadir = None
        self.delta_relative = 'drpms/'
        self.oldpackage_paths = [] # where to look for the old packages -
        self.deltafile = 'prestodelta.xml'
        self.num_deltas = 1 # number of older versions to delta (max)
        self.max_delta_rpm_size = 100000000
        self.update_md_path = None
        self.skip_stat = False
        self.database = True
        self.outputdir = None
        self.file_patterns = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
        self.dir_patterns = ['.*bin\/.*', '^\/etc\/.*']
        self.skip_symlinks = False
        self.pkglist = []
        self.database_only = False
        self.primaryfile = 'primary.xml'
        self.filelistsfile = 'filelists.xml'
        self.otherfile = 'other.xml'
        self.repomdfile = 'repomd.xml'
        self.tempdir = '.repodata'
        self.finaldir = 'repodata'
        self.olddir = '.olddata'
        self.mdtimestamp = 0
        self.directory = None
        self.directories = []
        self.changelog_limit = 10 # needs to be an int or None
        self.unique_md_filenames = True
        self.additional_metadata = {} # dict of 'type':'filename'
        self.revision = str(int(time.time()))
        self.content_tags = [] # flat list of strings (like web 2.0 tags)
        self.distro_tags = []# [(cpeid(None allowed), human-readable-string)]
        self.repo_tags = []# strings, forwhatever they are worth
        self.read_pkgs_list = None # filepath/name to write out list of pkgs
                                   # read in this run of createrepo
        self.collapse_glibc_requires = True
        self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
        self.worker_cmd = '/usr/share/createrepo/worker.py'
        #self.worker_cmd = './worker.py' # helpful when testing
        self.retain_old_md = 0
        self.compress_type = 'compat'

class SimpleMDCallBack(object):
    def errorlog(self, thing):
        print >> sys.stderr, thing

def log(self, thing):
        print thing

def progress(self, item, current, total):
        sys.stdout.write('\r' + ' ' * 80)
        sys.stdout.write("\r%d/%d - %s" % (current, total, item))
        sys.stdout.flush()

class MetaDataGenerator:
    def __init__(self, config_obj=None, callback=None):
        self.conf = config_obj
        if config_obj == None:
            self.conf = MetaDataConfig()
        if not callback:
            self.callback = SimpleMDCallBack()
        else:
            self.callback = callback

self.ts = rpmUtils.transaction.initReadOnlyTransaction()
        self.pkgcount = 0
        self.current_pkg = 0
        self.files = []
        self.rpmlib_reqs = {}
        self.read_pkgs = []
        self.compat_compress = False

if not self.conf.directory and not self.conf.directories:
            raise MDError, "No directory given on which to run."
        
        if self.conf.compress_type == 'compat':
            self.compat_compress = True
            self.conf.compress_type = None
            
        if not self.conf.compress_type:
            self.conf.compress_type = 'gz'
        
        if self.conf.compress_type not in utils._available_compression:
            raise MDError, "Compression %s not available: Please choose from: %s" \
                 % (self.conf.compress_type, ', '.join(utils._available_compression))
            
            
        if not self.conf.directories: # just makes things easier later
            self.conf.directories = [self.conf.directory]
        if not self.conf.directory: # ensure we have both in the config object
            self.conf.directory = self.conf.directories[0]

# the cachedir thing:
        if self.conf.cachedir:
            self.conf.use_cache = True

# this does the dir setup we need done
        self._parse_directory()
        self._test_setup_dirs()

def _parse_directory(self):
        """pick up the first directory given to us and make sure we know
           where things should go"""
        if os.path.isabs(self.conf.directory):
            self.conf.basedir = os.path.dirname(self.conf.directory)
            self.conf.relative_dir = os.path.basename(self.conf.directory)
        else:
            self.conf.basedir = os.path.realpath(self.conf.basedir)
            self.conf.relative_dir = self.conf.directory

self.package_dir = os.path.join(self.conf.basedir,
                                        self.conf.relative_dir)

if not self.conf.outputdir:
            self.conf.outputdir = os.path.join(self.conf.basedir,
                                               self.conf.relative_dir)

def _test_setup_dirs(self):
        # start the sanity/stupidity checks
        for mydir in self.conf.directories:
            if os.path.isabs(mydir):
                testdir = mydir
            else:
                if mydir.startswith('../'):
                    testdir = os.path.realpath(mydir)
                else:
                    testdir = os.path.join(self.conf.basedir, mydir)

if not os.path.exists(testdir):
                raise MDError, _('Directory %s must exist') % mydir

if not os.path.isdir(testdir):
                raise MDError, _('%s must be a directory') % mydir

if not os.access(self.conf.outputdir, os.W_OK):
            raise MDError, _('Directory %s must be writable.') % self.conf.outputdir

temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir)
        if not checkAndMakeDir(temp_output):
            raise MDError, _('Cannot create/verify %s') % temp_output

temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir)
        if not checkAndMakeDir(temp_final):
            raise MDError, _('Cannot create/verify %s') % temp_final

if self.conf.database:
            # do flock test on temp_final, temp_output
            # if it fails raise MDError
            for direc in [temp_final, temp_output]:
                f = open(direc + '/locktest', 'w')
                try:
                    fcntl.flock(f.fileno(), fcntl.LOCK_EX)
                except (OSError, IOError), e:
                    raise MDError, _("Could not create exclusive lock in %s and sqlite database generation enabled. Is this path on nfs? Is your lockd running?") % direc
                else:
                    f.close()
                    os.unlink(direc + '/locktest')
                
        if self.conf.deltas:
            temp_delta = os.path.join(self.conf.outputdir,
                                      self.conf.delta_relative)
            if not checkAndMakeDir(temp_delta):
                raise MDError, _('Cannot create/verify %s') % temp_delta
            self.conf.deltadir = temp_delta

if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)):
            raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir

# make sure we can write to where we want to write to:
        # and pickup the mdtimestamps while we're at it
        direcs = ['tempdir' , 'finaldir']
        if self.conf.deltas:
            direcs.append('deltadir')

for direc in direcs:
            filepath = os.path.join(self.conf.outputdir, getattr(self.conf,
                                                                 direc))
            if os.path.exists(filepath):
                if not os.access(filepath, os.W_OK):
                    raise MDError, _('error in must be able to write to metadata dir:\n  -> %s') % filepath

if self.conf.checkts:
                    # checking for repodata/repomd.xml - not just the data dir
                    rxml = filepath + '/repomd.xml'
                    if os.path.exists(rxml):
                        timestamp = os.path.getctime(rxml)
                        if timestamp > self.conf.mdtimestamp:
                            self.conf.mdtimestamp = timestamp

if self.conf.groupfile:
            a = self.conf.groupfile
            if self.conf.split:
                a = os.path.join(self.package_dir, self.conf.groupfile)
            elif not os.path.isabs(a):
                a = os.path.join(self.package_dir, self.conf.groupfile)

if not os.path.exists(a):
                raise MDError, _('Error: groupfile %s cannot be found.' % a)

self.conf.groupfile = a

if self.conf.cachedir:
            a = self.conf.cachedir
            if not os.path.isabs(a):
                a = os.path.join(self.conf.outputdir, a)
            if not checkAndMakeDir(a):
                raise MDError, _('Error: cannot open/write to cache dir %s' % a)

self.conf.cachedir = a

def _os_path_walk(self, top, func, arg):
        """Directory tree walk with callback function.
         copy of os.path.walk, fixes the link/stating problem
         """

try:
            names = os.listdir(top)
        except os.error:
            return
        func(arg, top, names)
        for name in names:
            name = os.path.join(top, name)
            if os.path.isdir(name):
                self._os_path_walk(name, func, arg)
    def getFileList(self, directory, ext):
        """Return all files in path matching ext, store them in filelist,
        recurse dirs. Returns a list object"""

extlen = len(ext)

def extension_visitor(filelist, dirname, names):
            for fn in names:
                fn = os.path.join(dirname, fn)
                if os.path.isdir(fn):
                    continue
                if self.conf.skip_symlinks and os.path.islink(fn):
                    continue
                elif fn[-extlen:].lower() == '%s' % (ext):
                    filelist.append(fn[len(startdir):])

filelist = []
        startdir = directory + '/'
        self._os_path_walk(startdir, extension_visitor, filelist)
        return filelist

def errorlog(self, thing):
        """subclass this if you want something different...."""
        errorprint(thing)

def checkTimeStamps(self):
        """check the timestamp of our target dir. If it is not newer than
           the repodata return False, else True"""
        if self.conf.checkts and self.conf.mdtimestamp:
            dn = os.path.join(self.conf.basedir, self.conf.directory)
            files = self.getFileList(dn, '.rpm')
            files = self.trimRpms(files)
            for f in files:
                fn = os.path.join(self.conf.basedir, self.conf.directory, f)
                if not os.path.exists(fn):
                    self.callback.errorlog(_('cannot get to file: %s') % fn)
                if os.path.getctime(fn) > self.conf.mdtimestamp:
                    return False

return True

return False

def trimRpms(self, files):
        badrpms = []
        for rpm_file in files:
            for glob in self.conf.excludes:
                if fnmatch.fnmatch(rpm_file, glob):
                    if rpm_file not in badrpms:
                        badrpms.append(rpm_file)
        for rpm_file in badrpms:
            if rpm_file in files:
                files.remove(rpm_file)
        return files

def _setup_old_metadata_lookup(self):
        """sets up the .oldData object for handling the --update call. Speeds
           up generating updates for new metadata"""
        #FIXME - this only actually works for single dirs. It will only
        # function for the first dir passed to --split, not all of them
        # this needs to be fixed by some magic in readMetadata.py
        # using opts.pkgdirs as a list, I think.
        if self.conf.update:
            #build the paths
            opts = {
                'verbose' : self.conf.verbose,
                'pkgdir'  : os.path.normpath(self.package_dir)
            }

if self.conf.skip_stat:
                opts['do_stat'] = False

if self.conf.update_md_path:
                norm_u_md_path = os.path.normpath(self.conf.update_md_path)
                u_md_repodata_path  = norm_u_md_path + '/repodata'
                if not os.path.exists(u_md_repodata_path):
                    msg = _('Warning: could not open update_md_path: %s') %  u_md_repodata_path
                    self.callback.errorlog(msg)
                old_repo_path = os.path.normpath(norm_u_md_path)
            else:
                old_repo_path = self.conf.outputdir

#and scan the old repo
            self.oldData = readMetadata.MetadataIndex(old_repo_path, opts)

def _setup_grabber(self):
        if not hasattr(self, '_grabber'):
            self._grabber = grabber.URLGrabber()

return self._grabber

grabber = property(fget = lambda self: self._setup_grabber())

def doPkgMetadata(self):
        """all the heavy lifting for the package metadata"""
        if self.conf.update:
            self._setup_old_metadata_lookup()
        # rpms we're going to be dealing with
        if isinstance(self.conf.pkglist, MetaSack):
            packages = self.conf.pkglist
        elif self.conf.pkglist:
            packages = []
            for pkg in self.conf.pkglist:
                if '://' in pkg: # remote
                    packages.append(pkg)
                    continue
                path = os.path.join(self.conf.basedir, self.conf.directory, pkg)
                if os.access(path, os.R_OK):
                    packages.append(pkg)
                    continue
                # not fatal, yet
                self.callback.errorlog('Cannot read file: %s' % path)
        else:
            packages = self.getFileList(self.package_dir, '.rpm')

if not isinstance(packages, MetaSack):
            packages = self.trimRpms(packages)
        self.pkgcount = len(packages)
        try:
            self.openMetadataDocs()
            self.writeMetadataDocs(packages)
            self.closeMetadataDocs()
        except (IOError, OSError), e:
            raise MDError, _('Cannot access/write repodata files: %s') % e

def openMetadataDocs(self):
        if self.conf.database_only:
            self.setup_sqlite_dbs()
        else:
            self.primaryfile = self._setupPrimary()
            self.flfile = self._setupFilelists()
            self.otherfile = self._setupOther()
        if self.conf.deltas:
            self.deltafile = self._setupDelta()

def _setupPrimary(self):
        # setup the primary metadata file
        fpz = self.conf.primaryfile + '.' + self.conf.compress_type
        primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
                                       fpz)
        fo = compressOpen(primaryfilepath, 'w', self.conf.compress_type)
        fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
            ' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' %
                       self.pkgcount)
        return fo

def _setupFilelists(self):
        # setup the filelist file
        fpz = self.conf.filelistsfile + '.' + self.conf.compress_type
        filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir,
                                    fpz)
        fo = compressOpen(filelistpath, 'w', self.conf.compress_type)
        fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \
                 ' packages="%s">' % self.pkgcount)
        return fo

def _setupOther(self):
        # setup the other file
        fpz = self.conf.otherfile + '.' + self.conf.compress_type
        otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
                                     fpz)
        fo = compressOpen(otherfilepath, 'w', self.conf.compress_type)
        fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \
                 ' packages="%s">' %
                       self.pkgcount)
        return fo

def _setupDelta(self):
        # setup the other file
        fpz = self.conf.deltafile + '.' + self.conf.compress_type        
        deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
                                     fpz)
        fo = compressOpen(deltafilepath, 'w', self.conf.compress_type)
        fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        fo.write('<prestodelta>\n')
        return fo

def read_in_package(self, rpmfile, pkgpath=None, reldir=None):
        """rpmfile == relative path to file from self.packge_dir"""
        baseurl = self.conf.baseurl

if not pkgpath:
            pkgpath = self.package_dir

if not rpmfile.strip():
            raise MDError, "Blank filename passed in, skipping"

if rpmfile.find("://") != -1:

if not hasattr(self, 'tempdir'):
                self.tempdir = tempfile.mkdtemp()

pkgname = os.path.basename(rpmfile)
            baseurl = os.path.dirname(rpmfile)
            reldir = self.tempdir
            dest = os.path.join(self.tempdir, pkgname)
            if not self.conf.quiet:
                self.callback.log('\nDownloading %s' % rpmfile)
            try:
                rpmfile = self.grabber.urlgrab(rpmfile, dest)
            except grabber.URLGrabError, e:
                raise MDError, "Unable to retrieve remote package %s: %s" % (
                                                                     rpmfile, e)

else:
            rpmfile = '%s/%s' % (pkgpath, rpmfile)

external_data = { '_cachedir': self.conf.cachedir,
                          '_baseurl': baseurl,
                          '_reldir': reldir,
                          '_packagenumber': self.current_pkg,
                          '_collapse_libc_requires':self.conf.collapse_glibc_requires,
                          }
                        
        try:
            po = yumbased.CreateRepoPackage(self.ts, rpmfile,
                                            sumtype=self.conf.sumtype,
                                            external_data = external_data)
        except Errors.MiscError, e:
            raise MDError, "Unable to open package: %s" % e

for r in po.requires_print:
            if r.startswith('rpmlib('):
                self.rpmlib_reqs[r] = 1

if po.checksum in (None, ""):
            raise MDError, "No Package ID found for package %s, not going to" \
                           " add it" % po

return po

def writeMetadataDocs(self, pkglist=[], pkgpath=None):

if not pkglist:
            pkglist = self.conf.pkglist

if not pkgpath:
            directory = self.conf.directory
        else:
            directory = pkgpath

# for worker/forked model
        # iterate the pkglist - see which ones are handled by --update and let them
        # go on their merry way
        
        newpkgs = []
        keptpkgs = []
        if self.conf.update:
            # if we're in --update mode then only act on the new/changed pkgs
            for pkg in pkglist:
                self.current_pkg += 1

#see if we can pull the nodes from the old repo
                #print self.oldData.basenodes.keys()
                old_pkg = pkg
                if pkg.find("://") != -1:
                    old_pkg = os.path.basename(pkg)
                old_po = self.oldData.getNodes(old_pkg)
                if old_po: # we have a match in the old metadata
                    if self.conf.verbose:
                        self.callback.log(_("Using data from old metadata for %s")
                                            % pkg)
                    keptpkgs.append((pkg, old_po))

#FIXME - if we're in update and we have deltas enabled
                    # check the presto data for this pkg and write its info back out
                    # to our deltafile
                    continue
                else:
                    newpkgs.append(pkg)
        else:
            newpkgs = pkglist

# setup our reldir
        if not pkgpath:
            reldir = os.path.join(self.conf.basedir, directory)
        else:
            reldir = pkgpath

# filter out those pkgs which are not files - but are pkgobjects
        pkgfiles = []
        for pkg in newpkgs:
            po = None
            if isinstance(pkg, YumAvailablePackage):
                po = pkg
                self.read_pkgs.append(po.localPkg())

# if we're dealing with remote pkgs - pitch it over to doing
            # them one at a time, for now. 
            elif pkg.find('://') != -1:
                po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir)
                self.read_pkgs.append(pkg)
            
            if po:
                keptpkgs.append((pkg, po))
                continue
                
            pkgfiles.append(pkg)

keptpkgs.sort(reverse=True)
        # keptkgs is a list of (filename, po), pkgfiles is a list if filenames.
        # Need to write them in sorted(filename) order.  We loop over pkgfiles,
        # inserting keptpkgs in right spots (using the upto argument).
        def save_keptpkgs(upto):
            while keptpkgs and (upto is None or keptpkgs[-1][0] < upto):
                filename, po = keptpkgs.pop()
                # reset baseurl in the old pkg
                po.basepath = self.conf.baseurl
                self.primaryfile.write(po.xml_dump_primary_metadata())
                self.flfile.write(po.xml_dump_filelists_metadata())
                self.otherfile.write(po.xml_dump_other_metadata(
                    clog_limit=self.conf.changelog_limit))

if pkgfiles:
            # divide that list by the number of workers and fork off that many
            # workers to tmpdirs
            # waitfor the workers to finish and as each one comes in
            # open the files they created and write them out to our metadata
            # add up the total pkg counts and return that value
            self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later
            if self.conf.workers < 1:
                self.conf.workers = num_cpus_online()
            pkgfiles.sort()
            worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers)
            worker_cmd_dict = {}
            worker_jobs = {}
            base_worker_cmdline = [self.conf.worker_cmd, 
                    '--pkgoptions=_reldir=%s' % reldir,
                    '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires, 
                    '--pkgoptions=_cachedir=%s' % self.conf.cachedir,
                    '--pkgoptions=_baseurl=%s' % self.conf.baseurl,
                    '--globalopts=clog_limit=%s' % self.conf.changelog_limit,
                    '--globalopts=sumtype=%s' % self.conf.sumtype, ]
            
            if self.conf.quiet:
                base_worker_cmdline.append('--quiet')
            
            if self.conf.verbose:
                base_worker_cmdline.append('--verbose')
                
            for worker_num in range(self.conf.workers):
                pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num
                f = open(pkl, 'w') 
                f.write('\n'.join(worker_chunks[worker_num]))
                f.close()
                
                workercmdline = []
                workercmdline.extend(base_worker_cmdline)
                workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num))
                worker_cmd_dict[worker_num] = workercmdline

for (num, cmdline) in worker_cmd_dict.items():
                if not self.conf.quiet:
                    self.callback.log("Spawning worker %s with %s pkgs" % (num, 
                                                      len(worker_chunks[num])))
                job = subprocess.Popen(cmdline, stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
                worker_jobs[num] = job
            
            files = self.primaryfile, self.flfile, self.otherfile
            def log_messages(num):
                job = worker_jobs[num]
                while True:
                    # check stdout and stderr
                    for stream in select((job.stdout, job.stderr), (), ())[0]:
                        line = stream.readline()
                        if line: break
                    else:
                        return # EOF, EOF
                    if stream is job.stdout:
                        if line.startswith('*** '):
                            if line == '*** \n':
                                return True
                            # get data, save to local files
                            for out, size in zip(files, line[4:].split()):
                                out.write(stream.read(int(size)))
                            return
                        self.callback.log('Worker %s: %s' % (num, line.rstrip()))
                    else:
                        self.callback.errorlog('Worker %s: %s' % (num, line.rstrip()))

err = 0
            for i, pkg in enumerate(pkgfiles):
                # insert cached packages
                save_keptpkgs(pkg)

# save output to local files
                if log_messages(i % self.conf.workers):
                    err += 1

for (num, job) in worker_jobs.items():
                # process remaining messages on stderr
                log_messages(num)

if job.wait() != 0:
                    msg = "Worker exited with non-zero value: %s. Fatal." % job.returncode
                    self.callback.errorlog(msg)
                    raise MDError, msg
                    
            if not self.conf.quiet:
                self.callback.log("Workers Finished")
                    
            if err:
                raise MDError, "Failed to process %d package(s)." % err

for pkgfile in pkgfiles:
                if self.conf.deltas:
                    try:
                        po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir)
                        self._do_delta_rpm_package(po)
                    except MDError, e:
                        errorprint(e)
                        continue
                self.read_pkgs.append(pkgfile)

save_keptpkgs(None) # append anything left
        return self.current_pkg

def closeMetadataDocs(self):
        # save them up to the tmp locations:
        if not self.conf.quiet:
            self.callback.log(_('Saving Primary metadata'))
        if self.conf.database_only:
            self.md_sqlite.pri_cx.close()
        else:
            self.primaryfile.write('\n</metadata>')
            self.primaryfile.close()

if not self.conf.quiet:
            self.callback.log(_('Saving file lists metadata'))
        if self.conf.database_only:
            self.md_sqlite.file_cx.close()
        else:
            self.flfile.write('\n</filelists>')
            self.flfile.close()

if not self.conf.quiet:
            self.callback.log(_('Saving other metadata'))
        if self.conf.database_only:
            self.md_sqlite.other_cx.close()
        else:
            self.otherfile.write('\n</otherdata>')
            self.otherfile.close()

if self.conf.deltas:
            deltam_st = time.time()
            if not self.conf.quiet:
                self.callback.log(_('Saving delta metadata'))
            self.deltafile.write(self.generate_delta_xml())
            self.deltafile.write('\n</prestodelta>')
            self.deltafile.close()
            if self.conf.profile:
                self.callback.log('deltam time: %0.3f' % (time.time() - deltam_st))

def _do_delta_rpm_package(self, pkg):
        """makes the drpms, if possible, for this package object.
           returns the presto/delta xml metadata as a string
        """
        drpm_pkg_time = time.time()
        # duck and cover if the pkg.size is > whatever
        if int(pkg.size) > self.conf.max_delta_rpm_size:
            if not self.conf.quiet:
                self.callback.log("Skipping %s package " \
                                    "that is > max_delta_rpm_size"  % pkg)
            return

# generate a list of all the potential 'old rpms'
        opd = self._get_old_package_dict()
        # for each of our old_package_paths -
        # make a drpm from the newest of that pkg
        # get list of potential candidates which are likely to match
        for d in self.conf.oldpackage_paths:
            pot_cand = []
            if d not in opd:
                continue
            for fn in opd[d]:
                if os.path.basename(fn).startswith(pkg.name):
                    pot_cand.append(fn)

candidates = []
            for fn in pot_cand:
                try:
                    thispo = yumbased.CreateRepoPackage(self.ts, fn,
                                                     sumtype=self.conf.sumtype)
                except Errors.MiscError, e:
                    continue
                if (thispo.name, thispo.arch) != (pkg.name, pkg.arch):
                    # not the same, doesn't matter
                    continue
                if thispo == pkg: #exactly the same, doesn't matter
                    continue
                if thispo.EVR >= pkg.EVR: # greater or equal, doesn't matter
                    continue
                candidates.append(thispo)
                candidates.sort()
                candidates.reverse()

for delta_p in candidates[0:self.conf.num_deltas]:
                #make drpm of pkg and delta_p
                dt_st = time.time()
                drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir)
                if not self.conf.quiet or self.conf.profile:
                    self.callback.log('created drpm from %s to %s: %s in %0.3f' % (
                        delta_p, pkg, drpmfn, (time.time() - dt_st)))
        if self.conf.profile:
            self.callback.log('total drpm time for %s: %0.3f' % (pkg,
                                                 (time.time() - drpm_pkg_time)))

def _get_old_package_dict(self):
        if hasattr(self, '_old_package_dict'):
            return self._old_package_dict

self._old_package_dict = {}
        for d in self.conf.oldpackage_paths:
            for f in self.getFileList(d, '.rpm'):
                fp = d + '/' + f
                fpstat = os.stat(fp)
                if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size:
                    self.callback.log("Skipping %s package " \
                                      "that is > max_delta_rpm_size"  % f)
                    continue
                if not self._old_package_dict.has_key(d):
                    self._old_package_dict[d] = []
                self._old_package_dict[d].append(d + '/' + f)

return self._old_package_dict

def generate_delta_xml(self):
        """take the delta rpm output dir, process all the drpm files
           produce the text output for the presto/delta xml metadata"""
        # go through the drpm dir
        # for each file -store the drpm info in a dict based on its target. Just
        # appending the output. for each of the keys in the dict, return
        # the tag for the target + each of the drpm infos + closure for the target
        # tag
        targets = {}
        results = []
        for drpm_fn in self.getFileList(self.conf.deltadir, '.drpm'):
            drpm_rel_fn = os.path.normpath(self.conf.delta_relative +
                                           '/' + drpm_fn) # this is annoying
            drpm_po = yumbased.CreateRepoPackage(self.ts,
                 self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype)

drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir,
                                             drpm_rel_fn)
            if not targets.has_key(drpm_po.pkgtup):
                targets[drpm_po.pkgtup] = []
            targets[drpm_po.pkgtup].append(drpm.xml_dump_metadata())

for (n, a, e, v, r) in targets.keys():
            results.append("""  <newpackage name="%s" epoch="%s" version="%s" release="%s" arch="%s">\n""" % (
                    n, e, v, r, a))
            results.extend(targets[(n,a,e,v,r)])
#            for src in targets[(n, a, e, v, r)]:
#                results.append(src)

results.append("   </newpackage>\n")

return ' '.join(results)

def _createRepoDataObject(self, mdfile, mdtype, compress=True, 
                              compress_type=None, attribs={}):
        """return random metadata as RepoData object to be  added to RepoMD
           mdfile = complete path to file
           mdtype = the metadata type to use
           compress = compress the file before including it
        """
        # copy the file over here
        sfile = os.path.basename(mdfile)
        fo = open(mdfile, 'r')
        outdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
        if not compress_type:
            compress_type = self.conf.compress_type
        if compress:
            sfile = '%s.%s' % (sfile, compress_type)
            outfn = os.path.join(outdir, sfile)
            output = compressOpen(outfn, mode='wb', compress_type=compress_type)
                
        else:
            outfn  = os.path.join(outdir, sfile)
            output = open(outfn, 'w')

output.write(fo.read())
        output.close()
        fo.seek(0)
        open_csum = misc.checksum(self.conf.sumtype, fo)
        fo.close()

if self.conf.unique_md_filenames:
            (csum, outfn) = checksum_and_rename(outfn, self.conf.sumtype)
            sfile = os.path.basename(outfn)
        else:
            if compress:
                csum = misc.checksum(self.conf.sumtype, outfn)
            else:
                csum = open_csum

thisdata = RepoData()
        thisdata.type = mdtype
        thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile))
        thisdata.checksum = (self.conf.sumtype, csum)
        if compress:
            thisdata.openchecksum  = (self.conf.sumtype, open_csum)
        
        thisdata.size = str(os.stat(outfn).st_size)
        thisdata.timestamp = str(int(os.stat(outfn).st_mtime))
        for (k, v) in attribs.items():
            setattr(thisdata, k, str(v))
        
        return thisdata

def doRepoMetadata(self):
        """wrapper to generate the repomd.xml file that stores the info
           on the other files"""
        
        repomd = RepoMD('repoid')
        repomd.revision = self.conf.revision

repopath = os.path.join(self.conf.outputdir, self.conf.tempdir)
        repofilepath = os.path.join(repopath, self.conf.repomdfile)

if self.conf.content_tags:
            repomd.tags['content'] = self.conf.content_tags
        if self.conf.distro_tags:
            repomd.tags['distro'] = self.conf.distro_tags
            # NOTE - test out the cpeid silliness here
        if self.conf.repo_tags:
            repomd.tags['repo'] = self.conf.repo_tags

sumtype = self.conf.sumtype
        workfiles = [(self.conf.otherfile, 'other',),
                     (self.conf.filelistsfile, 'filelists'),
                     (self.conf.primaryfile, 'primary')]

if self.conf.deltas:
            workfiles.append((self.conf.deltafile, 'prestodelta'))
        
        if self.conf.database:
            if not self.conf.quiet: self.callback.log('Generating sqlite DBs')
            try:
                dbversion = str(sqlitecachec.DBVERSION)
            except AttributeError:
                dbversion = '9'
            #FIXME - in theory some sort of try/except  here
            rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None)

for (rpm_file, ftype) in workfiles:
            unpath = os.path.join(repopath, rpm_file)
            if (ftype in ('other', 'filelists', 'primary')
                    or (rpm_file.find('.') != -1 and rpm_file.split('.')[-1]
                        not in _available_compression)):
                rpm_file = rpm_file + '.' + self.conf.compress_type
            complete_path = os.path.join(repopath, rpm_file)
            zfo = compressOpen(complete_path)
            dfo = None
            if (self.conf.compress_type == 'bz2' and self.conf.database and
                    ftype in ('other', 'filelists', 'primary')):
                # yum-metadata-parser doesn't understand bz2 so let's write the
                # decompressed data to a file and pass that via gen_func
                # instead of the compressed version
                dfo = open(unpath, 'w')
            # This is misc.checksum() done locally so we can get the size too.
            data = misc.Checksums([sumtype])
            while True:
                chunk = data.read(zfo, 2**16)
                if not chunk:
                    break
                if dfo is not None:
                    dfo.write(chunk)
            uncsum = data.hexdigest(sumtype)
            unsize = len(data)
            zfo.close()
            if dfo is not None:
                dfo.close()

csum = misc.checksum(sumtype, complete_path)
            timestamp = os.stat(complete_path)[8]

db_csums = {}
            db_compressed_sums = {}

if self.conf.database:
                if ftype in ['primary', 'filelists', 'other']:
                    if self.conf.verbose:
                        self.callback.log("Starting %s db creation: %s" % (ftype,
                                                                  time.ctime()))

gen_func = None
                if ftype == 'primary':
                    gen_func = rp.getPrimary
                elif ftype == 'filelists':
                    gen_func = rp.getFilelists
                elif ftype == 'other':
                    gen_func = rp.getOtherdata
                if gen_func is not None:
                    if dfo is None:
                        #FIXME - in theory some sort of try/except  here
                        # TypeError appears to be raised, sometimes :(
                        gen_func(complete_path, csum)
                    else:
                        #FIXME and here
                        gen_func(unpath, uncsum)
                        os.unlink(unpath)

if ftype in ['primary', 'filelists', 'other']:
                    if dfo is None:
                        compress_ext = '.%s' % self.conf.compress_type
                    else:
                        compress_ext = ''
                    tmp_result_name = '%s.xml%s.sqlite' % (ftype, compress_ext)
                    tmp_result_path = os.path.join(repopath, tmp_result_name)
                    good_name = '%s.sqlite' % ftype
                    resultpath = os.path.join(repopath, good_name)

# compat compression for rhel5 compatibility from fedora :(
                    compress_type = self.conf.compress_type
                    if self.compat_compress:
                        compress_type = 'bz2'
                        
                    # rename from silly name to not silly name
                    os.rename(tmp_result_path, resultpath)
                    compressed_name = '%s.%s' % (good_name, compress_type)
                    result_compressed = os.path.join(repopath, compressed_name)
                    db_csums[ftype] = misc.checksum(sumtype, resultpath)

# compress the files

compressFile(resultpath, result_compressed, compress_type)
                    # csum the compressed file
                    db_compressed_sums[ftype] = misc.checksum(sumtype,
                                                             result_compressed)
                    # timestamp+size the uncompressed file
                    un_stat = os.stat(resultpath)
                    # remove the uncompressed file
                    os.unlink(resultpath)

if self.conf.unique_md_filenames:
                        csum_compressed_name = '%s-%s.%s' % (
                                           db_compressed_sums[ftype], good_name, compress_type)
                        csum_result_compressed =  os.path.join(repopath,
                                                           csum_compressed_name)
                        os.rename(result_compressed, csum_result_compressed)
                        result_compressed = csum_result_compressed
                        compressed_name = csum_compressed_name

# timestamp+size the compressed file
                    db_stat = os.stat(result_compressed)

# add this data as a section to the repomdxml
                    db_data_type = '%s_db' % ftype
                    data = RepoData()
                    data.type = db_data_type
                    data.location = (self.conf.baseurl, 
                              os.path.join(self.conf.finaldir, compressed_name))
                    data.checksum = (sumtype, db_compressed_sums[ftype])
                    data.timestamp = str(int(db_stat.st_mtime))
                    data.size = str(db_stat.st_size)
                    data.opensize = str(un_stat.st_size)
                    data.openchecksum = (sumtype, db_csums[ftype])
                    data.dbversion = dbversion
                    if self.conf.verbose:
                        self.callback.log("Ending %s db creation: %s" % (ftype,
                                                                  time.ctime()))
                    repomd.repoData[data.type] = data
                    
            data = RepoData()
            data.type = ftype
            data.checksum = (sumtype, csum)
            data.timestamp = str(timestamp)
            data.size = str(os.stat(os.path.join(repopath, rpm_file)).st_size)
            data.opensize = str(unsize)
            data.openchecksum = (sumtype, uncsum)

if self.conf.unique_md_filenames:
                main_name = '.'.join(rpm_file.split('.')[:-1])
                res_file = '%s-%s.%s' % (csum, main_name, self.conf.compress_type)
                orig_file = os.path.join(repopath, rpm_file)
                dest_file = os.path.join(repopath, res_file)
                os.rename(orig_file, dest_file)
            else:
                res_file = rpm_file
            rpm_file = res_file
            href = os.path.join(self.conf.finaldir, rpm_file)

data.location = (self.conf.baseurl, href)
            repomd.repoData[data.type] = data

if not self.conf.quiet and self.conf.database:
            self.callback.log('Sqlite DBs complete')

if self.conf.groupfile is not None:
            mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group_gz')
            repomd.repoData[mdcontent.type] = mdcontent
            
            mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group',
                              compress=False)
            repomd.repoData[mdcontent.type] = mdcontent

if self.conf.additional_metadata:
            for md_type, md_file in self.conf.additional_metadata.items():
                mdcontent = self._createRepoDataObject(md_file, md_type)
                repomd.repoData[mdcontent.type] = mdcontent

# FIXME - disabled until we decide how best to use this
        #if self.rpmlib_reqs:
        #    rpmlib = reporoot.newChild(rpmns, 'lib', None)
        #    for r in self.rpmlib_reqs.keys():
        #        req  = rpmlib.newChild(rpmns, 'requires', r)

# save it down
        try:
            fo = open(repofilepath, 'w')
            fo.write(repomd.dump_xml())
            fo.close()
        except (IOError, OSError, TypeError), e:
            self.callback.errorlog(
                  _('Error saving temp file for repomd.xml: %s') % repofilepath)
            self.callback.errorlog('Error was: %s') % str(e)
            fo.close()
            raise MDError, 'Could not save temp file: %s' % repofilepath

def doFinalMove(self):
        """move the just-created repodata from .repodata to repodata
           also make sure to preserve any files we didn't mess with in the
           metadata dir"""

output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir)
        output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)

if os.path.exists(output_final_dir):
            try:
                os.rename(output_final_dir, output_old_dir)
            except:
                raise MDError, _('Error moving final %s to old dir %s' % (
                                 output_final_dir, output_old_dir))

output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)

try:
            os.rename(output_temp_dir, output_final_dir)
        except:
            # put the old stuff back
            os.rename(output_old_dir, output_final_dir)
            raise MDError, _('Error moving final metadata into place')

for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile',
                 'groupfile']:
            if getattr(self.conf, f):
                fn = os.path.basename(getattr(self.conf, f))
            else:
                continue
            oldfile = os.path.join(output_old_dir, fn)

if os.path.exists(oldfile):
                try:
                    os.remove(oldfile)
                except OSError, e:
                    raise MDError, _(
                    'Could not remove old metadata file: %s: %s') % (oldfile, e)

old_to_remove = []
        old_pr = []
        old_fl = []
        old_ot = []
        old_pr_db = []
        old_fl_db = []
        old_ot_db = []
        for f in os.listdir(output_old_dir):
            oldfile = os.path.join(output_old_dir, f)
            finalfile = os.path.join(output_final_dir, f)

for (end,lst) in (('-primary.sqlite', old_pr_db), ('-primary.xml', old_pr),
                           ('-filelists.sqlite', old_fl_db), ('-filelists.xml', old_fl),
                           ('-other.sqlite', old_ot_db), ('-other.xml', old_ot)):
                fn = '.'.join(f.split('.')[:-1])
                if fn.endswith(end):
                    lst.append(oldfile)
                    break

# make a list of the old metadata files we don't want to remove.
        for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db):
            sortlst = sorted(lst, key=lambda x: os.path.getmtime(x),
                             reverse=True)
            for thisf in sortlst[self.conf.retain_old_md:]:
                old_to_remove.append(thisf)

for f in os.listdir(output_old_dir):
            oldfile = os.path.join(output_old_dir, f)
            finalfile = os.path.join(output_final_dir, f)
            fn = '.'.join(f.split('.')[:-1])
            if fn in ('filelists.sqlite', 'other.sqlite',
                     'primary.sqlite') or oldfile in old_to_remove:
                try:
                    os.remove(oldfile)
                except (OSError, IOError), e:
                    raise MDError, _(
                    'Could not remove old metadata file: %s: %s') % (oldfile, e)
                continue

if os.path.exists(finalfile):
                # Hmph?  Just leave it alone, then.
                try:
                    if os.path.isdir(oldfile):
                        shutil.rmtree(oldfile)
                    else:
                        os.remove(oldfile)
                except OSError, e:
                    raise MDError, _(
                    'Could not remove old metadata file: %s: %s') % (oldfile, e)
            else:
                try:
                    os.rename(oldfile, finalfile)
                except OSError, e:
                    msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile)
                    msg += _('Error was %s') % e
                    raise MDError, msg
        self._write_out_read_pkgs_list()

def cleanup(self):
        self._cleanup_tmp_repodata_dir()
        self._cleanup_update_tmp_dir()

def _cleanup_update_tmp_dir(self):
        if self.conf.update and hasattr(self, 'oldData'):
            self.oldData.cleanup()

def _write_out_read_pkgs_list(self):
        # write out the read_pkgs_list file with self.read_pkgs
        if self.conf.read_pkgs_list:
            try:
                fo = open(self.conf.read_pkgs_list, 'w')
                fo.write('\n'.join(self.read_pkgs))
                fo.flush()
                fo.close()
            except (OSError, IOError), e:
                self.errorlog(_('Could not write out readpkgs list: %s')
                              % self.conf.read_pkgs_list)
                self.errorlog(_('Error was %s') % e)

def _cleanup_tmp_repodata_dir(self):
        output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
        output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
        for dirbase in (self.conf.olddir, self.conf.tempdir):
            dirpath = os.path.join(self.conf.outputdir, dirbase)
            if os.path.exists(dirpath):
                try:
                    os.rmdir(dirpath)
                except OSError, e:
                    self.errorlog(_('Could not remove  temp metadata dir: %s')
                                  % dirbase)
                    self.errorlog(_('Error was %s') % e)
                    self.errorlog(_('Please clean up this directory manually.'))
        # our worker tmp path
        if hasattr(self, '_worker_tmp_path') and os.path.exists(self._worker_tmp_path):
            shutil.rmtree(self._worker_tmp_path, ignore_errors=True)
        
    def setup_sqlite_dbs(self, initdb=True):
        """sets up the sqlite dbs w/table schemas and db_infos"""
        destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
        try:
            self.md_sqlite = MetaDataSqlite(destdir)
        except sqlite.OperationalError, e:
            raise MDError, _('Cannot create sqlite databases: %s.\n'\
                'Maybe you need to clean up a .repodata dir?') % e

class SplitMetaDataGenerator(MetaDataGenerator):
    """takes a series of dirs and creates repodata for all of them
       most commonly used with -u media:// - if no outputdir is specified
       it will create the repodata in the first dir in the list of dirs
       """
    def __init__(self, config_obj=None, callback=None):
        MetaDataGenerator.__init__(self, config_obj=config_obj, callback=None)

def _getFragmentUrl(self, url, fragment):
        import urlparse
        urlparse.uses_fragment.append('media')
        if not url:
            return url
        (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url)
        return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment)))

def doPkgMetadata(self):
        """all the heavy lifting for the package metadata"""
        if len(self.conf.directories) == 1:
            MetaDataGenerator.doPkgMetadata(self)
            return

if self.conf.update:
            self._setup_old_metadata_lookup()

filematrix = {}
        for mydir in self.conf.directories:
            if os.path.isabs(mydir):
                thisdir = mydir
            else:
                if mydir.startswith('../'):
                    thisdir = os.path.realpath(mydir)
                else:
                    thisdir = os.path.join(self.conf.basedir, mydir)

filematrix[mydir] = self.getFileList(thisdir, '.rpm')

#  pkglist is a bit different for split media, as we have to know
            # which dir. it belongs to. So we walk the dir. and then filter.
            # We could be faster by not walking the dir. ... but meh.
            if self.conf.pkglist:
                pkglist = set(self.conf.pkglist)
                pkgs = []
                for fname in filematrix[mydir]:
                    if fname not in pkglist:
                        continue
                    pkgs.append(fname)
                filematrix[mydir] = pkgs

self.trimRpms(filematrix[mydir])
            self.pkgcount += len(filematrix[mydir])

mediano = 1
        self.current_pkg = 0
        self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
        try:
            self.openMetadataDocs()
            for mydir in self.conf.directories:
                self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
                self.writeMetadataDocs(filematrix[mydir], mydir)
                mediano += 1
            self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, 1)
            self.closeMetadataDocs()
        except (IOError, OSError), e:
            raise MDError, _('Cannot access/write repodata files: %s') % e

class MetaDataSqlite(object):
    def __init__(self, destdir):
        self.pri_sqlite_file = os.path.join(destdir, 'primary.sqlite')
        self.pri_cx = sqlite.Connection(self.pri_sqlite_file)
        self.file_sqlite_file = os.path.join(destdir, 'filelists.sqlite')
        self.file_cx = sqlite.Connection(self.file_sqlite_file)
        self.other_sqlite_file = os.path.join(destdir, 'other.sqlite')
        self.other_cx = sqlite.Connection(self.other_sqlite_file)
        self.primary_cursor = self.pri_cx.cursor()

self.filelists_cursor = self.file_cx.cursor()

self.other_cursor = self.other_cx.cursor()

self.create_primary_db()
        self.create_filelists_db()
        self.create_other_db()

def create_primary_db(self):
        # make the tables
        schema = [
        """PRAGMA synchronous="OFF";""",
        """pragma locking_mode="EXCLUSIVE";""",
        """CREATE TABLE conflicts (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );""",
        """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
        """CREATE TABLE files (  name TEXT,  type TEXT,  pkgKey INTEGER);""",
        """CREATE TABLE obsoletes (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );""",
        """CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT,  name TEXT,  arch TEXT,  version TEXT,  epoch TEXT,  release TEXT,  summary TEXT,  description TEXT,  url TEXT,  time_file INTEGER,  time_build INTEGER,  rpm_license TEXT,  rpm_vendor TEXT,  rpm_group TEXT,  rpm_buildhost TEXT,  rpm_sourcerpm TEXT,  rpm_header_start INTEGER,  rpm_header_end INTEGER,  rpm_packager TEXT,  size_package INTEGER,  size_installed INTEGER,  size_archive INTEGER,  location_href TEXT,  location_base TEXT,  checksum_type TEXT);""",
        """CREATE TABLE provides (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );""",
        """CREATE TABLE requires (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER , pre BOOL DEFAULT FALSE);""",
        """CREATE INDEX filenames ON files (name);""",
        """CREATE INDEX packageId ON packages (pkgId);""",
        """CREATE INDEX packagename ON packages (name);""",
        """CREATE INDEX pkgconflicts on conflicts (pkgKey);""",
        """CREATE INDEX pkgobsoletes on obsoletes (pkgKey);""",
        """CREATE INDEX pkgprovides on provides (pkgKey);""",
        """CREATE INDEX pkgrequires on requires (pkgKey);""",
        """CREATE INDEX providesname ON provides (name);""",
        """CREATE INDEX requiresname ON requires (name);""",
        """CREATE TRIGGER removals AFTER DELETE ON packages
             BEGIN
             DELETE FROM files WHERE pkgKey = old.pkgKey;
             DELETE FROM requires WHERE pkgKey = old.pkgKey;
             DELETE FROM provides WHERE pkgKey = old.pkgKey;
             DELETE FROM conflicts WHERE pkgKey = old.pkgKey;
             DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;
             END;""",
         """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
             ]

for cmd in schema:
            executeSQL(self.primary_cursor, cmd)

def create_filelists_db(self):
        schema = [
            """PRAGMA synchronous="OFF";""",
            """pragma locking_mode="EXCLUSIVE";""",
            """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
            """CREATE TABLE filelist (  pkgKey INTEGER,  dirname TEXT,  filenames TEXT,  filetypes TEXT);""",
            """CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);""",
            """CREATE INDEX dirnames ON filelist (dirname);""",
            """CREATE INDEX keyfile ON filelist (pkgKey);""",
            """CREATE INDEX pkgId ON packages (pkgId);""",
            """CREATE TRIGGER remove_filelist AFTER DELETE ON packages
                   BEGIN
                   DELETE FROM filelist WHERE pkgKey = old.pkgKey;
                   END;""",
         """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
            ]
        for cmd in schema:
            executeSQL(self.filelists_cursor, cmd)

def create_other_db(self):
        schema = [
            """PRAGMA synchronous="OFF";""",
            """pragma locking_mode="EXCLUSIVE";""",
            """CREATE TABLE changelog (  pkgKey INTEGER,  author TEXT,  date INTEGER,  changelog TEXT);""",
            """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
            """CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);""",
            """CREATE INDEX keychange ON changelog (pkgKey);""",
            """CREATE INDEX pkgId ON packages (pkgId);""",
            """CREATE TRIGGER remove_changelogs AFTER DELETE ON packages
                 BEGIN
                 DELETE FROM changelog WHERE pkgKey = old.pkgKey;
                 END;""",
         """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
            ]

for cmd in schema:
            executeSQL(self.other_cursor, cmd)