From 1c51976278af139ad12a6027b2a5144d1847d394 Mon Sep 17 00:00:00 2001 From: David Lehman Date: Thu, 8 Mar 2012 09:13:45 -0600 Subject: [PATCH 1/2] Allow changing the base repo. Additional improvements to media/repo handling: - Make setup and updateMethod (almost?) completely non-interactive. - Clear out package and group lists when disabling or removing a repo. - Use repo-specific mount directories for NFS addon repos. - For NFS base repo, mount the NFS share on ISO_DIR in case it ends up being nfsiso. If it doesn't, make a symlink at INSTALL_TREE that points to ISO_DIR. - Remove fallback-to-optical-media for addons since that makes no sense. - Don't check for an NFS mirrorlist URL since that's not possible. --- pyanaconda/constants.py | 5 +- pyanaconda/image.py | 2 - pyanaconda/packaging/__init__.py | 53 ++++++- pyanaconda/packaging/yumpayload.py | 259 +++++++++++++++++++++++++----------- 4 files changed, 231 insertions(+), 88 deletions(-) diff --git a/pyanaconda/constants.py b/pyanaconda/constants.py index efbfddd..6757a87 100644 --- a/pyanaconda/constants.py +++ b/pyanaconda/constants.py @@ -71,6 +71,7 @@ TRANSLATIONS_UPDATE_DIR="/tmp/updates/po" ANACONDA_CLEANUP = "anaconda-cleanup" ROOT_PATH = "/mnt/sysimage" -ISO_DIR = "/mnt/install/isodir" -INSTALL_TREE = "/mnt/install/source" +MOUNT_DIR = "/mnt/install" +ISO_DIR = MOUNT_DIR + "/isodir" +INSTALL_TREE = MOUNT_DIR + "/source" BASE_REPO_NAME = "Installation Repo" diff --git a/pyanaconda/image.py b/pyanaconda/image.py index 7bfdb52..8c7597b 100644 --- a/pyanaconda/image.py +++ b/pyanaconda/image.py @@ -153,8 +153,6 @@ def mountImageDirectory(method, storage): raise exn def mountImage(isodir, tree, messageWindow): - if os.path.ismount(tree): - raise SystemError, "trying to mount already-mounted iso image!" while True: image = findFirstIsoImage(isodir, messageWindow) if image is None: diff --git a/pyanaconda/packaging/__init__.py b/pyanaconda/packaging/__init__.py index 2f7d59a..6bd8c65 100644 --- a/pyanaconda/packaging/__init__.py +++ b/pyanaconda/packaging/__init__.py @@ -96,6 +96,30 @@ class PayloadInstallError(PayloadError): pass +def get_mount_device(mountpoint): + import re + mounts = open("/proc/mounts").readlines() + mount_device = None + for mount in mounts: + try: + (device, path, rest) = mount.split(None, 2) + except ValueError: + continue + + if path == mountpoint: + mount_device = device + break + + if mount_device and re.match(r'/dev/loop\d+$', mount_device): + from pyanaconda.storage.devicelibs import loop + loop_name = os.path.basename(mount_device) + mount_device = loop.get_backing_file(loop_name) + log.debug("found backing file %s for loop device %s" % (mount_device, + loop_name)) + + log.debug("%s is mounted on %s" % (mount_device, mountpoint)) + return mount_device + class Payload(object): """ Payload is an abstract class for OS install delivery methods. """ def __init__(self, data): @@ -113,6 +137,15 @@ class Payload(object): """Return a list of repo identifiers, not objects themselves.""" raise NotImplementedError() + def getRepo(self, repo_id): + repo = None + for r in self.data.repo.dataList(): + if r.name == repo_id: + repo = r + break + + return repo + def addRepo(self, newrepo): """Add the repo given by the pykickstart Repo object newrepo to the system. The repo will be automatically enabled and its metadata @@ -237,7 +270,7 @@ class Payload(object): if not url: return None - log.debug("retrieving treeinfo from %s (proxies: %s ; sslverify: %s" + log.debug("retrieving treeinfo from %s (proxies: %s ; sslverify: %s)" % (url, proxies, sslverify)) ugopts = {"ssl_verify_peer": sslverify, @@ -292,15 +325,21 @@ class Payload(object): log.info("setting up device %s and mounting on %s" % (device.name, mountpoint)) if os.path.ismount(mountpoint): - log.debug("%s already has something mounted on it" % mountpoint) - return + mdev = get_mount_device(mountpoint) + log.warning("%s is already mounted on %s" % (mdev, mountpoint)) + if mdev == device.path: + return + else: + log.info("mounting on top of it") try: device.setup() device.format.setup(mountpoint=mountpoint) except StorageError as e: + log.error("mount failed: %s" % e) exn = PayloadSetupError(str(e)) if errorHandler.cb(exn) == ERROR_RAISE: + device.teardown(recursive=True) raise exn def _setupNFS(self, mountpoint, server, path, options): @@ -316,11 +355,11 @@ class Payload(object): try: isys.mount(url, mountpoint, options=options) except SystemError as e: + log.error("mount failed: %s" % e) exn = PayloadSetupError(str(e)) if errorHandler.cb(exn) == ERROR_RAISE: raise exn - ### ### METHODS FOR INSTALLING THE PAYLOAD ### @@ -450,13 +489,11 @@ if __name__ == "__main__": # set some things specially since we're just testing flags.testing = True - global ROOT_PATH - ROOT_PATH = "/tmp/test-root" # set up ksdata ksdata = makeVersion() - ksdata.method.method = "url" - ksdata.method.url = "http://husky/install/f17/os/" + #ksdata.method.method = "url" + #ksdata.method.url = "http://husky/install/f17/os/" #ksdata.method.url = "http://dl.fedoraproject.org/pub/fedora/linux/development/17/x86_64/os/" # set up storage diff --git a/pyanaconda/packaging/yumpayload.py b/pyanaconda/packaging/yumpayload.py index 03085e3..b69924f 100644 --- a/pyanaconda/packaging/yumpayload.py +++ b/pyanaconda/packaging/yumpayload.py @@ -23,11 +23,9 @@ """ TODO - - error handling!!! - document all methods - YumPayload - preupgrade - - clean up use of flags.testing - write test cases - more logging in key methods - rpm macros @@ -63,6 +61,7 @@ from pyanaconda.constants import * from pyanaconda.flags import flags from pyanaconda import iutil +from pyanaconda import isys from pyanaconda.network import hasActiveNetDev from pyanaconda.image import opticalInstallMedia @@ -78,19 +77,50 @@ log = logging.getLogger("anaconda") from pyanaconda.errors import * #from pyanaconda.progress import progress +default_repos = [productName.lower(), "rawhide"] + class YumPayload(PackagePayload): - """ A YumPayload installs packages onto the target system using yum. """ + """ A YumPayload installs packages onto the target system using yum. + + User-defined (aka: addon) repos exist both in ksdata and in yum. They + are the only repos in ksdata.repo. The repos we find in the yum config + only exist in yum. Lastly, the base repo exists in yum and in + ksdata.method. + """ def __init__(self, data): if rpm is None or yum is None: raise PayloadError("unsupported payload type") PackagePayload.__init__(self, data) - self._groups = [] - self._packages = [] - self.install_device = None self.proxy = None # global proxy + self._cache_dir = "/var/cache/yum" + self._yum = None + + self.reset() + + def reset(self): + if self._yum: + self._yum.close() + del self._yum + + if os.path.ismount(INSTALL_TREE) and not flags.testing: + isys.umount(INSTALL_TREE) + + if os.path.islink(INSTALL_TREE): + os.unlink(INSTALL_TREE) + + if os.path.ismount(ISO_DIR) and not flags.testing: + isys.umount(INSTALL_TREE) + + if self.install_device: + self.install_device.teardown(recursive=True) + + self.install_device = None + + self._groups = [] + self._packages = [] self._yum = yum.YumBase() @@ -98,12 +128,12 @@ class YumPayload(PackagePayload): # Set some configuration parameters that don't get set through a config # file. yum will know what to do with these. - # XXX We have to try to set releasever before we trigger a read of the - # repo config files. We do that from setup before adding any repos. self._yum.preconf.enabled_plugins = ["blacklist", "whiteout"] self._yum.preconf.fn = "/tmp/anaconda-yum.conf" self._yum.preconf.root = ROOT_PATH - self._cache_dir = "/var/cache/yum" + # set this now to the best default we've got ; we'll update it if/when + # we get a base repo set up + self._yum.preconf.releasever = self._getReleaseVersion(None) def setup(self, storage, proxy=None): buf = """ @@ -128,16 +158,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t fd.close() self.proxy = proxy - self._configureMethod(storage) - self._configureRepos(storage) - if flags.testing: - self._yum.setCacheDir() - - # go ahead and get metadata for all enabled repos now - for repoid in self.repos: - repo = self._yum.repos.getRepo(repoid) - if repo.enabled: - self._getRepoMetadata(repo) + self.updateMethod(storage) ### ### METHODS FOR WORKING WITH REPOSITORIES @@ -147,8 +168,12 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t return self._yum.repos.repos.keys() @property + def addOns(self): + return [r.name for r in self.data.repo.dataList()] + + @property def baseRepo(self): - repo_names = [BASE_REPO_NAME, productName.lower(), "rawhide"] + repo_names = [BASE_REPO_NAME] + default_repos base_repo_name = None for repo_name in repo_names: if repo_name in self.repos and \ @@ -168,35 +193,96 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t return False - def _configureRepos(self, storage): - """ Configure the initial repository set. """ - log.info("configuring repos") - # FIXME: driverdisk support + def _resetMethod(self): + self.data.method.method = "" + self.data.method.url = None + self.data.method.server = None + self.data.method.dir = None + self.data.method.partition = None + self.data.method.biospart = None + self.data.method.noverifyssl = False + self.data.method.proxy = "" + self.data.method.opts = None + + def updateMethod(self, storage): + """ Update the base repo based on self.data.method. + + - Tear down any previous base repo devices, symlinks, &c. + - Reset the YumBase instance. + - Try to convert the new method to a base repo. + - If that fails, we'll use whatever repos yum finds in the config. + - Set up addon repos. + - Filter out repos that don't make sense to have around. + - Get metadata for all enabled repos, disabling those for which the + retrieval fails. + """ + log.info("updating base repo") + # start with a fresh YumBase instance + self.reset() - # add/enable the repos anaconda knows about - # identify repos based on ksdata. - for repo in self.data.repo.dataList(): - self._configureKSRepo(storage, repo) + # see if we can get a usable base repo from self.data.method + try: + self._configureBaseRepo(storage) + except PayloadError as e: + log.error("failed to set up base repo: %s" % e) - # remove/disable repos that don't make sense during system install. - # If a method was given, disable any repos that aren't in ksdata. + if BASE_REPO_NAME not in self._yum.repos.repos.keys(): + log.info("using default repos from local yum configuration") + self._resetMethod() + + # set up addon repos + # FIXME: driverdisk support + for repo in self.data.repo.dataList(): + try: + self._configureAddOnRepo(storage, repo) + except NoNetworkError as e: + log.error("repo %s needs an active network connection" + % repo.name) + self.removeRepo(repo.name) + except PayloadError as e: + log.error("repo %s setup failed: %s" % (repo.name, e)) + self.removeRepo(repo.name) + + # now disable and/or remove any repos that don't make sense for repo in self._yum.repos.repos.values(): + """ Rules for which repos to enable/disable/remove + + - always remove + - source, debuginfo + - remove if isFinal + - rawhide, development + - remove any repo when not isFinal and repo not enabled + - if a base repo is defined, disable any repo not defined by + the user that is not the base repo + + """ + if repo.id in self.addOns: + continue + if "-source" in repo.id or "-debuginfo" in repo.id: - log.info("excluding source or debug repo %s" % repo.id) - self.removeRepo(repo.id) + self._removeYumRepo(repo.id) elif isFinal and ("rawhide" in repo.id or "development" in repo.id): - log.info("excluding devel repo %s for non-devel anaconda" % repo.id) - self.removeRepo(repo.id) + self._removeYumRepo(repo.id) elif not isFinal and not repo.enabled: - log.info("excluding disabled repo %s for prerelease" % repo.id) - self.removeRepo(repo.id) + self._removeYumRepo(repo.id) elif self.data.method.method and \ repo.id != BASE_REPO_NAME and \ repo.id not in [r.name for r in self.data.repo.dataList()]: - log.info("disabling repo %s" % repo.id) + # if a method/repo was given, disable all default repos self.disableRepo(repo.id) - def _configureMethod(self, storage): + # now go through and get metadata for all enabled repos + for repo_id in self.repos: + repo = self._yum.repos.getRepo(repo_id) + if repo.enabled: + try: + self._getRepoMetadata(repo) + except PayloadError as e: + log.error("failed to grab repo metadata for %s: %s" + % (repo_id, e)) + self.disableRepo(repo_id) + + def _configureBaseRepo(self, storage): """ Configure the base repo. """ log.info("configuring base repo") # set up the main repo specified by method=, repo=, or ks method @@ -225,9 +311,8 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t image = findFirstIsoImage(path) if not image: - exn = PayloadSetupError("failed to find valid iso image") - if errorHandler.cb(exn) == ERROR_RAISE: - raise exn + device.teardown(recursive=True) + raise PayloadSetupError("failed to find valid iso image") if path.endswith(".iso"): path = os.path.dirname(path) @@ -239,10 +324,9 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t self.install_device = device url = "file://" + INSTALL_TREE elif method.method == "nfs": - # XXX what if we mount it on ISO_DIR and then create a symlink - # if there are no isos instead of the remount? - self._setupNFS(INSTALL_TREE, method.server, method.dir, - method.opts) + # Mount the NFS share on ISO_DIR. If it ends up not being nfsiso we + # will create a symlink at INSTALL_TREE pointing to ISO_DIR. + self._setupNFS(ISO_DIR, method.server, method.dir, method.opts) # check for ISO images in the newly mounted dir path = ISO_DIR @@ -256,13 +340,19 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t # it appears there are ISO images in the dir, so assume they want to # install from one of them if image: - isys.umount(INSTALL_TREE) - self._setupNFS(ISO_DIR, method.server, method.path, - method.options) - # mount the ISO on a loop image = os.path.normpath("%s/%s" % (ISO_DIR, image)) mountImage(image, INSTALL_TREE) + else: + # create a symlink at INSTALL_TREE that points to ISO_DIR + try: + if os.path.exists(INSTALL_TREE): + os.unlink(INSTALL_TREE) + os.symlink(os.path.basename(ISO_DIR), INSTALL_TREE) + except OSError as e: + log.error("failed to update %s symlink: %s" + % (INSTALL_TREE, e)) + raise PayloadSetupError(str(e)) url = "file://" + INSTALL_TREE elif method.method == "url": @@ -270,33 +360,31 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t sslverify = not (method.noverifyssl or flags.noverifyssl) proxy = method.proxy or self.proxy elif method.method == "cdrom" or not method.method: + # cdrom or no method specified -- check for media device = opticalInstallMedia(storage.devicetree) if device: self.install_device = device url = "file://" + INSTALL_TREE if not method.method: method.method = "cdrom" - - self._yum.preconf.releasever = self._getReleaseVersion(url) + elif method.method == "cdrom": + # no cdrom was found, so update method.method + method.method = "" if method.method: - # FIXME: handle MetadataError + self._yum.preconf.releasever = self._getReleaseVersion(url) self._addYumRepo(BASE_REPO_NAME, url, proxy=proxy, sslverify=sslverify) - def _configureKSRepo(self, storage, repo): + def _configureAddOnRepo(self, storage, repo): """ Configure a single ksdata repo. """ - url = getattr(repo, "baseurl", repo.mirrorlist) - if url.startswith("nfs:"): - # FIXME: create a directory other than INSTALL_TREE based on - # the repo's id/name to avoid crashes if the base repo is NFS + url = repo.baseurl + if url and url.startswith("nfs:"): (opts, server, path) = iutil.parseNfsUrl(url) - self._setupNFS(INSTALL_TREE, server, path, opts) - else: - # check for media, fall back to default repo - device = opticalInstallMedia(storage.devicetree) - if device: - self.install_device = device + mountpoint = "%s/%s.nfs" % (MOUNT_DIR, repo.name) + self._setupNFS(mountpoint, server, path, opts) + + url = "file://" + mountpoint if self._repoNeedsNetwork(repo) and not hasActiveNetDev(): raise NoNetworkError @@ -304,12 +392,12 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t proxy = repo.proxy or self.proxy sslverify = not (flags.noverifyssl or repo.noverifyssl) - # this repo does not go into ksdata -- only yum - self.addYumRepo(repo.id, repo.baseurl, repo.mirrorlist, cost=repo.cost, - exclude=repo.excludepkgs, includepkgs=repo.includepkgs, - proxy=proxy, sslverify=sslverify) + # this repo is already in ksdata, so we only add it to yum here + self._addYumRepo(repo.name, url, repo.mirrorlist, cost=repo.cost, + exclude=repo.excludepkgs, includepkgs=repo.includepkgs, + proxy=proxy, sslverify=sslverify) - # TODO: enable addons + # TODO: enable addons via treeinfo def _getRepoMetadata(self, yumrepo): """ Retrieve repo metadata if we don't already have it. """ @@ -318,6 +406,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t # And try to grab its metadata. We do this here so it can be done # on a per-repo basis, so we can then get some finer grained error # handling and recovery. + log.debug("getting repo metadata for %s" % yumrepo.id) try: yumrepo.getPrimaryXML() yumrepo.getOtherXML() @@ -328,6 +417,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t # At the worst, it just means the groups won't be displayed in the UI # which isn't too bad, because you may be doing a kickstart install and # picking packages instead. + log.debug("getting group info for %s" % yumrepo.id) try: yumrepo.getGroups() except RepoMDError: @@ -339,12 +429,6 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t if name in self._yum.repos.repos: self._yum.repos.delete(name) - # Replace anything other than HTTP/FTP with file:// - if baseurl and \ - not baseurl.startswith("http:") and \ - not baseurl.startswith("ftp:"): - baseurl = "file://" + INSTALL_TREE - log.debug("adding yum repo %s with baseurl %s and mirrorlist %s" % (name, baseurl, mirrorlist)) # Then add it to yum's internal structures. @@ -367,14 +451,34 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t self._addYumRepo(newrepo) # FIXME: handle MetadataError super(YumRepo, self).addRepo(newrepo) + def _removeYumRepo(self, repo_id): + if repo_id in self.repos: + self._yum.repos.delete(repo_id) + + self._groups = [] + self._packages = [] + def removeRepo(self, repo_id): """ Remove a repo as specified by id. """ log.debug("removing repo %s" % repo_id) - if repo_id in self.repos: - self._yum.repos.delete(repo_id) + # if this is an NFS repo, we'll want to unmount the NFS mount after + # removing the repo + mountpoint = None + yum_repo = self._yum.repos.getRepo(repo_id) + ks_repo = self.getRepo(repo_id) + if yum_repo and ks_repo and ks_repo.baseurl.startswith("nfs:"): + mountpoint = yum_repo.baseurl[0][7:] # strip leading "file://" + + self._removeYumRepo(repo_id) super(YumPayload, self).removeRepo(repo_id) + if mountpoint and os.path.ismount(mountpoint): + try: + isys.umount(mountpoint) + except SystemError as e: + log.error("failed to unmount nfs repo %s: %s" % (mountpoint, e)) + def enableRepo(self, repo_id): """ Enable a repo as specified by id. """ log.debug("enabling repo %s" % repo_id) @@ -387,6 +491,9 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t if repo_id in self.repos: self._yum.repos.disableRepo(repo_id) + self._groups = [] + self._packages = [] + ### ### METHODS FOR WORKING WITH GROUPS ### -- 1.7.9.1