From 5eede10a5202932c969e1eaf480dc465b829aa1b Mon Sep 17 00:00:00 2001
From: David Lehman <dlehman@redhat.com>
Date: Fri, 17 Feb 2012 19:51:10 -0600
Subject: [PATCH 1/2] Add new packaging module.

This will eventually replace backend.py, livecd.py, and yuminstall.py.
---
 pyanaconda/packaging.py |  757 +++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 757 insertions(+), 0 deletions(-)
 create mode 100644 pyanaconda/packaging.py

diff --git a/pyanaconda/packaging.py b/pyanaconda/packaging.py
new file mode 100644
index 0000000..f013947
--- /dev/null
+++ b/pyanaconda/packaging.py
@@ -0,0 +1,757 @@
+#!/usr/bin/python
+
+"""
+    TODO
+        - error handling!!!
+        - see what from YumPayload can be moved up into parent classes
+        - document all methods
+        - YumPayload
+            - preupgrade
+            - sort out dependency resolution
+            - callbacks
+                - dependency resolution?
+                - package install
+            - rpm macros
+                - __file_context_path
+                - _excludedocs
+        - LiveImagePayload
+            - register the live image, either via self.data.method or in setup
+              using storage
+
+"""
+
+from pyanaconda.constants import *
+
+from urlgrabber.grabber import URLGrabber
+from urlgrabber.grabber import URLGrabError
+import ConfigParser
+
+# hate.
+# this can probably move down into __main__
+from pyanaconda import anaconda_log
+anaconda_log.init()
+
+try:
+    import tarfile
+except ImportError:
+    pass
+
+from pyanaconda.flags import flags
+from pyanaconda.network import hasActiveNetDev
+
+from pyanaconda.images import opticalInstallMedia
+from pyanaconda.images import mountImmage
+from pyanaconda.images import findFirstIsoImage
+
+from pykickstart.parser import Group
+from pykickstart.version import makeVersion
+
+import logging
+log = logging.getLogger("anaconda")
+
+from pyanaconda.errors import *
+
+###
+### ERROR HANDLING
+###
+class PayloadError(Exception):
+    pass
+
+class MetadataError(PayloadError):
+    pass
+
+class NoNetworkError(PayloadError):
+    pass
+
+class NoSuchGroup(PayloadError):
+    pass
+
+class NoSuchPackage(PayloadError):
+    pass
+
+class PayloadSetupError(PayloadError):
+    pass
+
+class DependencyError(PayloadError):
+    pass
+
+class PayloadInstallError(PayloadError):
+    pass
+
+class Payload(object):
+    """ Payload is an abstract class for OS install delivery methods. """
+    def __init__(self, data):
+        self.data = data
+
+    def setup(self, storage):
+        """ Do any payload-specific setup. """
+        raise NotImplementedError()
+
+    ###
+    ### METHODS FOR WORKING WITH REPOSITORIES
+    ###
+    @property
+    def repos(self):
+        """Return a list of repo identifiers, not objects themselves."""
+        raise NotImplementedError()
+
+    def addRepo(self, newrepo):
+        """Add the repo given by the pykickstart Repo object newrepo to the
+           system.  The repo will be automatically enabled and its metadata
+           fetched.
+
+           Duplicate repos will not raise an error.  They should just silently
+           take the place of the previous value.
+        """
+        # Add the repo to the ksdata so it'll appear in the output ks file.
+        self.data.repo.dataList().append(newrepo)
+
+    def removeRepo(self, repo_id):
+        # XXX can we manipulate self.data.repo.repoList directly for this
+        #     purpose or should we do all that one time at the end of the
+        #     configuration step
+        raise NotImplementedError()
+
+    def enableRepo(self, repo_id):
+        raise NotImplementedError()
+
+    def disableRepo(self, repo_id):
+        raise NotImplementedError()
+
+    ###
+    ### METHODS FOR WORKING WITH GROUPS
+    ###
+    @property
+    def groups(self):
+        raise NotImplementedError()
+
+    def description(self, groupid):
+        raise NotImplementedError()
+
+    def selectGroup(self, groupid):
+        grp = Group(groupid)
+
+        if grp in self.data.groupList:
+            return
+
+        if grp in self.data.excludedGroupList:
+            self.data.excludedGroupList.remove(grp)
+
+        self.groupList.append(grp)
+
+    def deselectGroup(self, groupid):
+        grp = Group(groupid)
+
+        if grp in self.data.excludedGroupList:
+            return
+
+        if grp in self.data.groupList:
+            self.data.groupList.remove(grp)
+
+        self.excludedGroupList.append(grp)
+
+    ###
+    ### METHODS FOR WORKING WITH PACKAGES
+    ###
+    @property
+    def packages(self):
+        raise NotImplementedError()
+
+    def selectPackage(self, pkgid):
+        """Mark a package for installation.
+
+           pkgid - The name of a package to be installed.  This could include
+                   a version or architecture component.
+        """
+        if pkgid in self.data.packageList:
+            return
+
+        if pkgid in self.data.excludedList:
+            self.data.excludedList.remove(pkgid)
+
+        self.data.packageList.append(pkgid)
+
+    def deselectPackage(self, pkgid):
+        """Mark a package to be excluded from installation.
+
+           pkgid - The name of a package to be excluded.  This could include
+                   a version or architecture component.
+        """
+        if pkgid in self.data.excludedList:
+            return
+
+        if pkgid in self.data.packageList:
+            self.data.packageList.remove(pkgid)
+
+        self.data.excludedList.append(pkgid)
+
+    ###
+    ### METHODS FOR QUERYING STATE
+    ###
+    @property
+    def spaceRequired(self):
+        raise NotImplementedError()
+
+    @property
+    def kernelVersionList(self):
+        raise NotImplementedError()
+
+    ###
+    ### METHODS FOR MEDIA HANDLING
+    ###
+    
+
+    ###
+    ### METHODS FOR INSTALLING THE PAYLOAD
+    ###
+    def preInstall(self):
+        """ Perform pre-installation tasks.
+
+            PRE: filesystems are mounted
+
+            Callbacks take an Exception as sole argument and return True to
+            indicate that the error was fatal or False otherwise.
+        """
+        pass
+
+        # start logger
+
+    def install(self):
+        """ Install the payload.
+
+            Callbacks take an Exception as sole argument and return True to
+            indicate that the error was fatal or False otherwise.
+        """
+        raise NotImplementedError()
+
+    def postInstall(self):
+        """ Perform post-installation tasks.
+
+            Callbacks take an Exception as sole argument and return True to
+            indicate that the error was fatal or False otherwise.
+        """
+        pass
+
+        # set default runlevel/target (?)
+        # write out static config (storage, modprobe, keyboard, ??)
+        #   kickstart should handle this before we get here
+        # copy firmware
+        # recreate initrd
+        #   postInstall or bootloader.install
+        # copy dd rpms (yum/rpm only?)
+        #   kickstart
+        # copy dd modules and firmware (yum/rpm only?)
+        #   kickstart
+        # write escrow packets
+        # stop logger
+
+class ImagePayload(Payload):
+    """ An ImagePayload installs an OS image to the target system. """
+    def __init__(self, data):
+        super(ImagePayload, self).__init__(data)
+        self.image_file = None
+
+    def setup(self, storage):
+        if not self.image_file:
+            exn = PayloadSetupError("image file not set")
+            if errorHandler(exn) == ERROR_RAISE:
+                raise exn
+
+class LiveImagePayload(ImagePayload):
+    """ A LivePayload copies the source image onto the target system. """
+    def setup(self, storage):
+        super(LiveImagePayload, self).setup()
+        if not stat.S_ISBLK(os.stat(self.image_file)[stat.ST_MODE]):
+            raise PayloadSetupError("unable to find image")
+
+    def install(self):
+        """ Install the payload. """
+        cmd = "rsync"
+        args = ["-rlptgoDHAXv", self.os_image, ROOT_PATH]
+        try:
+            rc = iutil.execWithRedirect(cmd, args,
+                                        stderr="/dev/tty5", stdout="/dev/tty5")
+        except (OSError, RuntimeError) as e:
+            err = str(e)
+        else:
+            err = None
+            if rc != 0:
+                err = "%s exited with code %d" % (cmd, rc)
+
+        if err:
+            exn = PayloadInstallError(err)
+            if errorHandler(exn) == ERROR_RAISE:
+                raise exn
+
+    @property
+    def spaceRequired(self):
+        
+
+class ArchivePayload(ImagePayload):
+    """ An ArchivePayload unpacks source archives onto the target system. """
+    pass
+
+class TarPayload(ArchivePayload):
+    """ A TarPayload unpacks tar archives onto the target system. """
+    def __init__(self, data):
+        if 'tarfile' not in locals():
+            raise PayloadError("unsupported payload type")
+
+        super(TarPayload, self).__init__(data)
+        self.archive = None
+
+    def setup(self, storage):
+        super(TarPayload, self).setup()
+
+        try:
+            self.archive = tarfile.open(self.image_file)
+        except (tarfile.ReadError, tarfile.CompressionError) as e:
+            # maybe we only need to catch ReadError and CompressionError here
+            log.error("opening tar archive %s: %s" % (self.image_file, e))
+            raise PayloadError("invalid payload format")
+
+    @property
+    def requiredSpace(self):
+        byte_count = sum([m.size for m in self.archive.getmembers()])
+        return byte_count / (1024.0 * 1024.0)   # FIXME: Size
+
+    @property
+    def kernelVersionList(self):
+        names = self.archive.getnames()
+        kernels = [n for n in names if "boot/vmlinuz-" in n]
+
+    def install(self):
+        try:
+            selfarchive.extractall(path=ROOT_PATH)
+        except (tarfile.ExtractError, tarfile.CompressionError) as e:
+            log.error("extracting tar archive %s: %s" % (self.image_file, e))
+
+class PackagePayload(Payload):
+    """ A PackagePayload installs a set of packages onto the target system. """
+    pass
+
+class YumPayload(PackagePayload):
+    """ A YumPayload installs packages onto the target system using yum. """
+    def __init__(self, data):
+        PackagePayload.__init__(self, data)
+
+        from yum import YumBase
+
+        self._groups = []
+        self._packages = []
+
+        self.install_tree = "/mnt/install/source"   # move to constants.py?
+        self.iso_dir = "/mnt/install/isodir"
+        self.install_device = None
+        self.proxy = None                           # global proxy
+
+        self._yum = YumBase()
+
+        # Set some configuration parameters that don't get set through a config
+        # file.  yum will know what to do with these.
+        # XXX We have to try to set releasever before we trigger a read of the
+        #     repo config files. We do that from setup before adding any repos.
+        self._yum.preconf.enabled_plugins = ["blacklist", "whiteout"]
+        self._yum.preconf.fn = "/tmp/anaconda-yum.conf"
+        self._yum.preconf.root = ROOT_PATH
+
+    def setup(self, storage, proxy=None):
+        buf = """
+[main]
+installroot=%s
+cachedir=/tmp/cache/yum
+keepcache=0
+logfile=/tmp/yum.log
+metadata_expire=0
+pluginpath=/usr/lib/yum-plugins,/tmp/updates/yum-plugins
+pluginconfpath=/etc/yum/pluginconf.d,/tmp/updates/pluginconf.d
+plugins=1
+reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/tmp/product/anaconda.repos.d
+""" % ROOT_PATH
+
+        if proxy:
+            buf += "proxy=%s" % proxy
+
+        fd = open("/tmp/anaconda-yum.conf", "w")
+        fd.write(buf)
+        fd.close()
+
+        self.proxy = proxy
+        self._configureMethod(storage)
+        self._configureRepos(storage)
+
+    ###
+    ### METHODS FOR WORKING WITH REPOSITORIES
+    ###
+    @property
+    def repos(self):
+        return self._yum.repos.repos.keys()
+
+    def _repoNeedsNetwork(self, repo):
+        urls = [repo.baseurl] + repo.mirrorlist
+        network_protocols = ["http:", "ftp:", "nfs:", "nfsiso:"]
+        for url in urls:
+            if any([url.startswith(p) for p in network_protocols]):
+                return True
+
+        return False
+
+    def _configureRepos(self, storage):
+        # FIXME: driverdisk
+
+        # add/enable the repos anaconda knows about
+        # identify repos based on ksdata. 
+        for repo in self.data.repo.dataList():
+            self._configureKSRepo(storage, repo)
+
+        # remove/disable repos that don't make sense during system install.
+        # If a method was given, disable any repos that aren't in ksdata.
+        for repo in self._yum.repos.repos.values():
+            if "-source" in repo.id or "-debuginfo" in repo.id:
+                self.removeRepo(repo.id)
+            elif isFinal and ("rawhide" in repo.id or "development" in repo.id):
+                self.removeRepo(repo.id)
+            elif not isFinal and not repo.enabled:
+                self.removeRepo(repo.id)
+            elif self.data.method.method and repo.id not in self.repos:
+                self._yum.disableRepo(repo.id)
+
+    def _configureMethod(self, storage):
+        # set up the main repo specified by method=, repo=, or ks method
+        # XXX this repo does not go into ksdata -- only yum
+        method = self.data.method
+        url = "file://" + self.install_tree
+        noverifyssl = False
+        proxy = self.proxy
+
+        if method.method == "cdrom":
+            devices = opticalInstallMedia(storage.devicetree)
+            if devices:
+                self._setupCD(devices[0])
+                self.install_device = devices[0]
+        elif method.method == "harddrive":
+            if method.biospart:
+                devspec = method.biospart
+            else:
+                devspec = method.partition
+
+            # FIXME: teach DeviceTree.resolveDevice about biospart
+            device = storage.devicetree.resolveDevice(devspec)
+            self._setupHD(device, method.dir)
+            self.install_device = device
+        elif method.method == "nfs":
+            # XXX this may require some additional tweaking for nfsiso, eg:
+            #     when the path specifies an iso image instead of a directory
+            self._setupNFS(method.server, method.dir, method.opts)
+        elif method.method == "url":
+            url = method.url
+            # is this redundant?
+            noverifyssl = method.noverifyssl
+            proxy = method.proxy
+
+        self._yum.preconf.releasever = self._getReleaseVer(url)
+        self._addYumRepo("Installation Repo", url,
+                         proxy=proxy, sslverify=not nosslverify)
+
+    def _configureKSRepo(self, storage, repo):
+        url = getattr(repo, "baseurl", repo.mirrorlist)
+        if url.startswith("nfs:"):
+            (opts, server, path) = iutil.parseNfsUrl(url)
+            self._setupNFS(server, path, opts)
+        else:
+            # check for media, fall back to default repo
+            devices = opticalInstallMedia(storage.devicetree)
+            if devices:
+                self._setupCD(devices[0])
+                self.install_device = devices[0]
+
+        if self._repoNeedsNetwork(repo) and not hasActiveNetDev():
+            raise NoNetworkError
+
+        proxy = repo.proxy or None  # FIXME: what about global/cmdline proxy?
+        sslverify = not (flags.noverifyssl or repo.noverifyssl)
+
+        self.addYumRepo(repo.id, repo.baseurl, repo.mirrorlist, cost=repo.cost,
+                        exclude=repo.excludepkgs, includepkgs=repo.includepkgs,
+                        proxy=proxy, sslverify=sslverify)
+
+        # TODO: enable addons
+
+    def _setupCD(self, device):
+        device.setup()
+        device.format.setup(mountpoint=self.install_tree)
+
+    def _setupNFS(self, server, path, options):
+        # mount the specified directory
+        url = "%s:%s" % (server, path)
+        isys.mount(url, self.install_tree, options=options)
+
+        # check for ISO if base
+        image = findFirstIsoImage(self.install_tree)
+        if image:
+            isys.umount(self.install_tree)
+            self._setupNFSISO(server, path, options)
+
+    def _setupNFSISO(self, server, path, options):
+        url = "%s:%s" % (server, path)
+        isys.mount(url, self.iso_dir, options=options)
+
+        # mount the ISO
+        mountImage(self.iso_dir, self.install_tree)
+
+    def _setupHD(self, device, path):
+        # mount the filesystem on the specified device
+        device.setup()
+        device.format.setup(mountpoint=self.iso_dir)
+
+        # mount the image (currently, first one with matching arch in
+        # .discinfo)
+        mountImage(self.iso_dir, self.install_tree)
+
+    def _addYumRepo(self, name, baseurl, mirrorlist=None, **kwargs):
+        from yum.Errors import RepoError, RepoMDError
+
+        # First, delete any pre-existing repo with the same name.
+        if name in self._yum.repos.repos:
+            self._yum.repos.delete(name)
+
+        # Replace anything other than HTTP/FTP with file://
+        if baseurl and \
+           not baseurl.startswith("http:") and \
+           not baseurl.startswith("ftp:"):
+            baseurl = "file:/" + self.install_tree
+
+        # Then add it to yum's internal structures.
+        obj = self._yum.add_enable_repo(name,
+                                        baseurl=[baseurl],
+                                        mirrorlist=mirrorlist,
+                                        **kwargs)
+
+        # And try to grab its metadata.  We do this here so it can be done
+        # on a per-repo basis, so we can then get some finer grained error
+        # handling and recovery.
+        try:
+            obj.getPrimaryXML()
+            obj.getOtherXML()
+        except RepoError as e:
+            raise MetadataError(e.value)
+
+        # Not getting group info is bad, but doesn't seem like a fatal error.
+        # At the worst, it just means the groups won't be displayed in the UI
+        # which isn't too bad, because you may be doing a kickstart install and
+        # picking packages instead.
+        try:
+            obj.getGroups()
+        except RepoMDError:
+            pass
+
+        # Adding a new repo means the cached packages and groups lists
+        # are out of date.  Clear them out now so the next reference to
+        # either will cause it to be regenerated.
+        self._groups = []
+        self._packages = []
+
+    def addRepo(self, newrepo):
+        self._addYumRepo(newrepo)
+        super(YumRepo, self).addRepo(newrepo)
+
+    def removeRepo(self, repo_id):
+        if repo_id in self.repos:
+            self._yum.repos.delete(repo_id)
+
+        # XXX remove from self.data.repos.repoList too?
+
+    def enableRepo(self, repo_id):
+        if repo_id in self.repos:
+            self._yum.repos.enableRepo(repo_id)
+
+    def disableRepo(self, repo_id):
+        if repo_id in self.repos:
+            self._yum.repos.disableRepo(repo_id)
+
+    def _getTreeInfo(self, url, sslverify, proxy_dict):
+        """ Retrieve treeinfo and return the path to the local file. """
+        if not url:
+            return None
+
+        ug = URLGrabber()
+        try:
+            treeinfo = ug.urlgrab("%s/.treeinfo" % url,
+                                  "/tmp/.treeinfo", copy_local=True,
+                                  proxies=proxies, **ugopts)
+        except URLGrabError as e:
+            try:
+                treeinfo = ug.urlgrab("%s/treeinfo" % eurl,
+                                      "/tmp/.treeinfo", copy_local=True,
+                                      proxies=proxies, **ugopts)
+            except URLGrabError as e:
+                log.info("Error downloading treeinfo: %s" % e)
+
+        return treeinfo
+        
+    def _getReleaseVersion(self, url):
+        """ Return the release version of the tree at the specified URL. """
+        version = productVersion.split("-")[0]
+
+        ugopts = {"ssl_verify_peer": not flags.noverifyssl,
+                  "ssl_verify_host": not flags.noverifyssl}
+
+        proxies = {}
+        if self.proxy:
+            proxies = {"http": self.proxy,
+                       "https": self.proxy}
+
+        treeinfo = self._getTreeInfo(url, sslverify, proxies)
+        if treeinfo:
+            c = ConfigParser.ConfigParser()
+            c.read(treeinfo)
+            try:
+                # Trim off any -Alpha or -Beta
+                version = c.get("general", "version").split("-")[0]
+            except ConfigParser.Error:
+                pass
+
+        return version
+
+    ###
+    ### METHODS FOR WORKING WITH GROUPS
+    ###
+    @property
+    def groups(self):
+        from yum.Errors import RepoError
+
+        if not self._groups:
+            if not hasActiveNetDev():
+                raise NoNetworkError
+
+            try:
+                self._groups = self._yum.comps
+            except RepoError as e:
+                raise MetadataError(e.value)
+
+        return map(lambda grp: grp.groupid, self._groups.get_groups())
+
+    def description(self, groupid):
+        if not self._groups.has_group(groupid):
+            raise NoSuchGroup(groupid)
+
+        group = self._groups.return_group(groupid)
+
+        return (group.ui_name, group.ui_description)
+
+    ###
+    ### METHODS FOR WORKING WITH PACKAGES
+    ###
+    @property
+    def packages(self):
+        from yum.Errors import RepoError
+
+        if not self._packages:
+            if not hasActiveNetDev():
+                raise NoNetworkError
+
+            try:
+                self._packages = self._yum.pkgSack.returnPackages()
+            except RepoError as e:
+                raise MetadataError(e.value)
+
+        return map(lambda pkg: pkg.name, self._packages)
+
+    ###
+    ### METHODS FOR INSTALLING THE PAYLOAD
+    ###
+    def checkSoftwareSelection(self):
+        # TODO: propagate ksdata selections down into yum
+
+        # doPostSelection
+        # select kernel packages
+        # select packages needed for storage, bootloader
+        # check dependencies
+
+        while True:
+            (code, msgs) = self._yum.resolveDeps()
+            
+            if code == 0:
+                # empty transaction?
+                break
+            elif code == 2:
+                # success
+                break
+            elif self.data.packages.handleMissing == KS_MISSING_IGNORE:
+                break
+            elif self.data.upgrade.upgrade:
+                break
+
+            for msg in msgs:
+                log.warning(msg)
+
+            exn = DependencyError(msgs)
+            rc = errorHandler(exn)
+            if rc == ERROR_RAISE:
+                raise exn
+            elif rc == ERROR_RETRY:
+                # FIXME: figure out how to allow modification of software set
+                return False
+
+        # check free space (?)
+
+
+    def preInstall(self):
+        """ Perform pre-installation tasks. """
+        pass
+        # doPreInstall
+        # create a bunch of directories like /var, /var/lib/rpm, /root, &c (?)
+        # create mountpoints for protected device mountpoints (?)
+        # initialize the backend logger
+        # write static configs (storage, modprobe.d/anaconda.conf, network, keyboard)
+        #   on upgrade, just make sure /etc/mtab is a symlink to /proc/self/mounts
+
+    def install(self):
+        """ Install the payload. """
+        # This basically replaces YumBase._doTransaction.
+        self._yum.initActionTs()
+        self._yum.populateTs(keepold=0)
+        self._yum.ts.check()
+        self._yum.ts.order()
+
+        # XXX we could probably get away with setting cb=self if we defined a
+        #     callback method
+        self._yum.runTransaction(cb=cb)
+
+    def postInstall(self):
+        """ Perform post-installation tasks. """
+        self._yum.close()
+
+        # clean up repo tmpdirs
+        # clean the yum cache
+        # on preupgrade, remove the preupgrade dir
+
+###
+### MAIN
+###
+ksdata = makeVersion()
+obj = YumPayload(ksdata)
+obj.setup()
+
+repo = ksdata.RepoData(name="anaconda", baseurl="http://cannonball/install/rawhide/os/")
+obj.addRepo(repo)
+
+desktops = []
+addons = []
+
+for grp in obj.groups:
+    if not desktops and not addons:
+        print dir(grp)
+    if grp.endswith("-desktop"):
+        desktops.append(obj.description(grp))
+    elif not grp.endswith("-support"):
+        addons.append(obj.description(grp))
+
+import pprint
+
+print "==== DESKTOPS ===="
+pprint.pprint(desktops)
+print "==== ADDONS ===="
+pprint.pprint(addons)
+
+print obj.groups
-- 
1.7.6

