diff --git a/pyanaconda/packaging.py b/pyanaconda/packaging.py index 9238f9b..ce7f42e 100644 --- a/pyanaconda/packaging.py +++ b/pyanaconda/packaging.py @@ -18,6 +18,7 @@ - passed to anaconda as --proxy, --proxyUsername, and --proxyPassword - drop the use of a file for proxy and ftp auth info + - tell wwoods. he probably hasn't done that yet. - specified via KS as a URL - LiveImagePayload - register the live image, either via self.data.method or in setup @@ -28,6 +29,7 @@ from urlgrabber.grabber import URLGrabber from urlgrabber.grabber import URLGrabError import ConfigParser +import shutil from pyanaconda import anaconda_log anaconda_log.init() @@ -35,10 +37,22 @@ anaconda_log.init() try: import tarfile except ImportError: - pass + tarfile = None + +try: + import rpm +except ImportError: + rpm = None + +try: + import yum +except ImportError: + yum = None from pyanaconda.constants import * from pyanaconda.flags import flags + +from pyanaconda import iutil from pyanaconda.network import hasActiveNetDev from pyanaconda.image import opticalInstallMedia @@ -51,7 +65,10 @@ from pykickstart.version import makeVersion import logging log = logging.getLogger("anaconda") +from pyanaconda.backend_log import log as instlog + from pyanaconda.errors import * +#from pyanaconda.progress import progress ### ### ERROR HANDLING @@ -140,27 +157,39 @@ class Payload(object): def description(self, groupid): raise NotImplementedError() - def selectGroup(self, groupid): - grp = Group(groupid) + def selectGroup(self, groupid, default=True, optional=False): + if optional: + include = GROUP_ALL + elif default: + include = GROUP_DEFAULT + else: + include = GROUP_REQUIRED + + grp = Group(groupid, include=include) + + if grp in self.data.packages.groupList: + # I'm not sure this would ever happen, but ensure that re-selecting + # a group with a different types set works as expected. + if grp.include != include: + grp.include = include - if grp in self.data.groupList: return - if grp in self.data.excludedGroupList: - self.data.excludedGroupList.remove(grp) + if grp in self.data.packages.excludedGroupList: + self.data.packages.excludedGroupList.remove(grp) - self.groupList.append(grp) + self.data.packages.groupList.append(grp) def deselectGroup(self, groupid): grp = Group(groupid) - if grp in self.data.excludedGroupList: + if grp in self.data.packages.excludedGroupList: return - if grp in self.data.groupList: - self.data.groupList.remove(grp) + if grp in self.data.packages.groupList: + self.data.packages.groupList.remove(grp) - self.excludedGroupList.append(grp) + self.data.packages.excludedGroupList.append(grp) ### ### METHODS FOR WORKING WITH PACKAGES @@ -175,13 +204,13 @@ class Payload(object): pkgid - The name of a package to be installed. This could include a version or architecture component. """ - if pkgid in self.data.packageList: + if pkgid in self.data.packages.packageList: return - if pkgid in self.data.excludedList: - self.data.excludedList.remove(pkgid) + if pkgid in self.data.packages.excludedList: + self.data.packages.excludedList.remove(pkgid) - self.data.packageList.append(pkgid) + self.data.packages.packageList.append(pkgid) def deselectPackage(self, pkgid): """Mark a package to be excluded from installation. @@ -189,13 +218,13 @@ class Payload(object): pkgid - The name of a package to be excluded. This could include a version or architecture component. """ - if pkgid in self.data.excludedList: + if pkgid in self.data.packages.excludedList: return - if pkgid in self.data.packageList: - self.data.packageList.remove(pkgid) + if pkgid in self.data.packages.packageList: + self.data.packages.packageList.remove(pkgid) - self.data.excludedList.append(pkgid) + self.data.packages.excludedList.append(pkgid) ### ### METHODS FOR QUERYING STATE @@ -208,6 +237,60 @@ class Payload(object): def kernelVersionList(self): raise NotImplementedError() + ## + ## METHODS FOR TREE VERIFICATION + ## + def _getTreeInfo(self, url, sslverify, proxies): + """ Retrieve treeinfo and return the path to the local file. """ + if not url: + return None + + log.debug("retrieving treeinfo from %s (proxies: %s ; sslverify: %s" + % (url, proxies, sslverify)) + + ugopts = {"ssl_verify_peer": sslverify, + "ssl_verify_host": sslverify} + + ug = URLGrabber() + try: + treeinfo = ug.urlgrab("%s/.treeinfo" % url, + "/tmp/.treeinfo", copy_local=True, + proxies=proxies, **ugopts) + except URLGrabError as e: + try: + treeinfo = ug.urlgrab("%s/treeinfo" % url, + "/tmp/.treeinfo", copy_local=True, + proxies=proxies, **ugopts) + except URLGrabError as e: + log.info("Error downloading treeinfo: %s" % e) + + return treeinfo + + def _getReleaseVersion(self, url): + """ Return the release version of the tree at the specified URL. """ + version = productVersion.split("-")[0] + + log.debug("getting release version from tree at %s (%s)" % (url, + version)) + + proxies = {} + if self.proxy: + proxies = {"http": self.proxy, + "https": self.proxy} + + treeinfo = self._getTreeInfo(url, not flags.noverifyssl, proxies) + if treeinfo: + c = ConfigParser.ConfigParser() + c.read(treeinfo) + try: + # Trim off any -Alpha or -Beta + version = c.get("general", "version").split("-")[0] + except ConfigParser.Error: + pass + + log.debug("got a release version of %s" % version) + return version + ### ### METHODS FOR MEDIA HANDLING ### @@ -216,31 +299,37 @@ class Payload(object): ### METHODS FOR INSTALLING THE PAYLOAD ### def preInstall(self): - """ Perform pre-installation tasks. + """ Perform pre-installation tasks. """ + # XXX this should be handled already + iutil.mkdirChain(ROOT_PATH + "/root") - PRE: filesystems are mounted + if self.data.upgrade.upgrade: + mode = "upgrade" + else: + mode = "install" - Callbacks take an Exception as sole argument and return True to - indicate that the error was fatal or False otherwise. - """ - pass + log_file_name = "%s.log" % mode + log_file_path = "%s/root/%s" % (ROOT_PATH, log_file_name) + try: + shutil.rmtree (log_file_path) + except OSError: + pass - # start logger + self.install_log = open(log_file_path, "w+") - def install(self): - """ Install the payload. + syslogname = "%s%s.syslog" % log_file_path + try: + shutil.rmtree (syslogname) + except OSError: + pass + instlog.start(ROOT_PATH, syslogname) - Callbacks take an Exception as sole argument and return True to - indicate that the error was fatal or False otherwise. - """ + def install(self): + """ Install the payload. """ raise NotImplementedError() def postInstall(self): - """ Perform post-installation tasks. - - Callbacks take an Exception as sole argument and return True to - indicate that the error was fatal or False otherwise. - """ + """ Perform post-installation tasks. """ pass # set default runlevel/target (?) @@ -302,7 +391,7 @@ class ArchivePayload(ImagePayload): class TarPayload(ArchivePayload): """ A TarPayload unpacks tar archives onto the target system. """ def __init__(self, data): - if 'tarfile' not in locals(): + if tarfile is None: raise PayloadError("unsupported payload type") super(TarPayload, self).__init__(data) @@ -340,10 +429,14 @@ class PackagePayload(Payload): class YumPayload(PackagePayload): """ A YumPayload installs packages onto the target system using yum. """ + BASE_REPO_NAME = "Installation Repo" + def __init__(self, data): - PackagePayload.__init__(self, data) + if rpm is None or yum is None: + print _locals + raise PayloadError("unsupported payload type") - from yum import YumBase + PackagePayload.__init__(self, data) self._groups = [] self._packages = [] @@ -351,7 +444,7 @@ class YumPayload(PackagePayload): self.install_device = None self.proxy = None # global proxy - self._yum = YumBase() + self._yum = yum.YumBase() # Set some configuration parameters that don't get set through a config # file. yum will know what to do with these. @@ -368,7 +461,7 @@ installroot=%s cachedir=/tmp/cache/yum keepcache=0 logfile=/tmp/yum.log -metadata_expire=0 +metadata_expire=never pluginpath=/usr/lib/yum-plugins,/tmp/updates/yum-plugins pluginconfpath=/etc/yum/pluginconf.d,/tmp/updates/pluginconf.d plugins=1 @@ -376,6 +469,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t """ % ROOT_PATH if proxy: + # FIXME: include proxy_username, proxy_password buf += "proxy=%s" % proxy fd = open("/tmp/anaconda-yum.conf", "w") @@ -393,6 +487,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t ### @property def repos(self): + # FIXME: should this return pykickstart Repo or YumRepo? return self._yum.repos.repos.keys() def _repoNeedsNetwork(self, repo): @@ -435,8 +530,8 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t """ Configure the base repo. """ log.info("configuring base repo") # set up the main repo specified by method=, repo=, or ks method - # XXX this repo does not go into ksdata -- only yum # XXX FIXME: does this need to handle whatever was set up by dracut? + # XXX FIXME: most of this probably belongs up in Payload method = self.data.method sslverify = True @@ -505,7 +600,6 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t url = "file://" + INSTALL_TREE elif method.method == "url": - # FIXME: handle a directory containing ISO images url = method.url sslverify = not (method.noverifyssl or flags.noverifyssl) proxy = method.proxy or self.proxy @@ -513,7 +607,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t self._yum.preconf.releasever = self._getReleaseVersion(url) if method.method: - self._addYumRepo("Installation Repo", url, + self._addYumRepo(self.BASE_REPO_NAME, url, proxy=proxy, sslverify=sslverify) def _configureKSRepo(self, storage, repo): @@ -537,63 +631,13 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t proxy = repo.proxy or self.proxy sslverify = not (flags.noverifyssl or repo.noverifyssl) + # this repo does not go into ksdata -- only yum self.addYumRepo(repo.id, repo.baseurl, repo.mirrorlist, cost=repo.cost, exclude=repo.excludepkgs, includepkgs=repo.includepkgs, proxy=proxy, sslverify=sslverify) # TODO: enable addons - def _getTreeInfo(self, url, sslverify, proxies): - """ Retrieve treeinfo and return the path to the local file. """ - if not url: - return None - - log.debug("retrieving treeinfo from %s (proxies: %s ; sslverify: %s" - % (url, proxies, sslverify)) - - ugopts = {"ssl_verify_peer": sslverify, - "ssl_verify_host": sslverify} - - ug = URLGrabber() - try: - treeinfo = ug.urlgrab("%s/.treeinfo" % url, - "/tmp/.treeinfo", copy_local=True, - proxies=proxies, **ugopts) - except URLGrabError as e: - try: - treeinfo = ug.urlgrab("%s/treeinfo" % url, - "/tmp/.treeinfo", copy_local=True, - proxies=proxies, **ugopts) - except URLGrabError as e: - log.info("Error downloading treeinfo: %s" % e) - - return treeinfo - - def _getReleaseVersion(self, url): - """ Return the release version of the tree at the specified URL. """ - version = productVersion.split("-")[0] - - log.debug("getting release version from tree at %s (%s)" % (url, - version)) - - proxies = {} - if self.proxy: - proxies = {"http": self.proxy, - "https": self.proxy} - - treeinfo = self._getTreeInfo(url, not flags.noverifyssl, proxies) - if treeinfo: - c = ConfigParser.ConfigParser() - c.read(treeinfo) - try: - # Trim off any -Alpha or -Beta - version = c.get("general", "version").split("-")[0] - except ConfigParser.Error: - pass - - log.debug("got a release version of %s" % version) - return version - def _addYumRepo(self, name, baseurl, mirrorlist=None, **kwargs): """ Add a yum repo to the YumBase instance. """ from yum.Errors import RepoError, RepoMDError @@ -632,7 +676,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t try: obj.getGroups() except RepoMDError: - pass + log.error("failed to get groups for repo %s" % repo.id) # Adding a new repo means the cached packages and groups lists # are out of date. Clear them out now so the next reference to @@ -667,7 +711,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t self._yum.repos.disableRepo(repo_id) ## - ## METHODS FOR MEDIA MANAGEMENT + ## METHODS FOR MEDIA MANAGEMENT (XXX should these go in another module?) ## def _setupDevice(self, device, mountpoint): """ Prepare an install CD/DVD for use as a package source. """ @@ -718,7 +762,7 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t except RepoError as e: raise MetadataError(e.value) - return map(lambda grp: grp.groupid, self._groups.get_groups()) + return [g.groupid for g in self._groups.get_groups()] def description(self, groupid): """ Return name/description tuple for the group specified by id. """ @@ -745,31 +789,81 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t except RepoError as e: raise MetadataError(e.value) - return map(lambda pkg: pkg.name, self._packages) + return self._packages ### ### METHODS FOR INSTALLING THE PAYLOAD ### + def _setYumSelections(self): + # propagate group and package selections into yum + # XXX FIXME: Things that got pulled in as deps by a previous group + # selection are not getting removed when that group is de- + # selected. + self._yum._undoDepInstalls() + + # select packages + for package in self.data.packages.packageList: + log.debug("select package %s" % package) + try: + mbrs = self._yum.install(pattern=package) + except yum.Errors.InstallError: + log.error("no package matching %s" % package) + + # select groups + for group in self.data.packages.groupList: + pkg_types = ['mandatory'] + if group.include != GROUP_REQUIRED: + pkg_types.append("default") + + if group.include == GROUP_ALL: + pkg_types.append("optional") + + log.debug("select group %s" % group.name) + try: + self._yum.selectGroup(group.name, group_package_types=pkg_types) + except yum.Errors.GroupsError: + log.error("no such group: %s" % group.name) + + # deselect packages + for package in self.data.packages.excludedList: + log.debug("deselect package %s" % package) + self._yum.tsInfo.deselect(package) + + # deselect groups + for group in self.data.packages.excludedGroupList: + log.debug("deselect group %s" % group.name) + try: + self._yum.deselectGroup(group.name, force=True) + except yum.Errors.GroupsError: + log.error("no such group: %s" % group.name) + def checkSoftwareSelection(self): - # TODO: propagate ksdata selections down into yum + log.info("checking software selection") + self._setYumSelections() # doPostSelection # select kernel packages # select packages needed for storage, bootloader - # check dependencies + # check dependencies + # XXX FIXME: set self._yum.dsCallback before entering this loop while True: - (code, msgs) = self._yum.resolveDeps() + log.info("checking dependencies") + (code, msgs) = self._yum.buildTransaction(unfinished_transactions_check=False) if code == 0: # empty transaction? + log.debug("empty transaction") break elif code == 2: # success + log.debug("success") break elif self.data.packages.handleMissing == KS_MISSING_IGNORE: + log.debug("ignoring missing due to ks config") break elif self.data.upgrade.upgrade: + log.debug("ignoring unresolved deps on upgrade") break for msg in msgs: @@ -788,7 +882,8 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t def preInstall(self): """ Perform pre-installation tasks. """ - pass + super(YumPayload, self).preInstall() + # doPreInstall # create a bunch of directories like /var, /var/lib/rpm, /root, &c (?) # create mountpoints for protected device mountpoints (?) @@ -798,25 +893,210 @@ reposdir=/etc/yum.repos.d,/etc/anaconda.repos.d,/tmp/updates/anaconda.repos.d,/t def install(self): """ Install the payload. """ - # This basically replaces YumBase._doTransaction. + log.info("preparing transaction") self._yum.initActionTs() - self._yum.populateTs(keepold=0) + + try: + # uses dsCallback.transactionPopulation + self._yum.populateTs(keepold=0) + except RepoError as e: + log.error("error populating transaction: %s" % e) + exn = PayloadInstallError(str(e)) + if errorHandler(exn) == ERROR_RAISE: + raise exn + self._yum.ts.check() self._yum.ts.order() - # XXX we could probably get away with setting cb=self if we defined a - # callback method - self._yum.runTransaction(cb=cb) + # set up rpm logging to go to our log + self._yum.ts.ts.scriptFd = self.install_log.fileno() + rpm.setLogFile(self.install_log) + + # create the install callback + rpmcb = RPMCallback(self._yum, self.install_log, + upgrade=self.data.upgrade.upgrade) + + if flags.testing: + #self._yum.ts.setFlags(rpm.RPMTRANS_FLAG_TEST) + return + + log.info("running transaction") + try: + self._yum.runTransaction(cb=rpmcb) + except PackageSackError as e: + log.error("error running transaction: %s" % e) + exn = PayloadInstallError(str(e)) + if errorHandler(exn) == ERROR_RAISE: + raise exn + except YumRPMTransError as e: + log.error("error running transaction: %s" % e) + for error in e.errors: + log.error(e[0]) + exn = PayloadInstallError(str(e)) + if errorHandler(exn) == ERROR_RAISE: + raise exn + except YumBaseError as e: + log.error("error [2] running transaction: %s" % e) + for error in e.errors: + log.error("%s" % e[0]) + exn = PayloadInstallError(str(e)) + if errorHandler(exn) == ERROR_RAISE: + raise exn + finally: + self._yum.ts.close() + iutil.resetRpmDb() def postInstall(self): """ Perform post-installation tasks. """ self._yum.close() # clean up repo tmpdirs - # clean the yum cache - # on preupgrade, remove the preupgrade dir + self._yum.cleanPackages() + self._yum.cleanHeaders() + + # remove cache dirs of install-specific repos + for repo in self._yum.repos.listEnabled(): + if repo.name == self.BASE_REPO_NAME or \ + repo.id.startswith("anaconda-"): + shutil.rmtree(repo.cachedir) + + # clean the yum cache on upgrade + if self.data.upgrade.upgrade: + self._yum.cleanMetadata() + + # TODO: on preupgrade, remove the preupgrade dir + +class RPMCallback(object): + def __init__(self, yb, log, upgrade): + self._yum = yb # yum.YumBase + self.install_log = log # file instance + self.upgrade = upgrade # boolean + + self.package_file = None # file instance (package file management) + + def _get_txmbr(self, key): + """ Return a (name, TransactionMember) tuple from cb key. """ + if hasattr(key, "po"): + # New-style callback, key is a TransactionMember + txmbr = key.po + name = key.name + elif isinstance(key, tuple): + # Old-style (hdr, path) callback + h = key[0] + name = h['name'] + epoch = '0' + if h['epoch'] is not None: + epoch = str(h['epoch']) + pkgtup = (h['name'], h['arch'], epoch, h['version'], h['release']) + txmbrs = self._yum.tsInfo.getMembers(pkgtup=pkgtup) + if len(txmbrs) != 1: + log.error("unable to find package %s" % pkgtup) + exn = PayloadInstallError("failed to find package") + if errorHandler(exn, pkgtup) == ERROR_RAISE: + raise exn + txmbr = txmbrs[0] + else: + # cleanup/remove error + name = key + txmbr = None + + return (name, txmbr) + def callback(self, what, amount, total, h, user): + """ Yum install callback. """ + if what == rpm.RPMCALLBACK_TRANS_START: + pass + elif what == rpm.RPMCALLBACK_TRANS_PROGRESS: + # amount / total complete + pass + elif what == rpm.RPMCALLBACK_TRANS_STOP: + # we are done + pass + elif what == rpm.RPMCALLBACK_INST_OPEN_FILE: + # update status that we're installing/upgrading %h + # return an open fd to the file + txmbr = self._get_txmbr(h)[1] + + if self.upgrade: + mode = _("Upgrading") + else: + mode = _("Installing") + + self.install_log.write("%s %s %s" % (time.strftime("%H:%M:%S"), + mode, txmbr.po)) + self.install_log.flush() + + self.package_file = None + repo = self._yum.repos.getRepo(po.repoid) + + while self.package_file is None: + try: + package_path = repo.getPackage(po) + except (yum.Errors.NoMoreMirrorsRepoError, IOError): + exn = PayloadInstallError("failed to open package") + if errorHandler(exn, po) == ERROR_RAISE: + raise exn + except yum.Errors.RepoError: + continue + + self.package_file = open(package_path) + + return self.package_file.fileno() + elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE: + # close and remove the last opened file + # update count of installed/upgraded packages + package_path = self.package_file.name + self.package_file.close() + self.package_file = None + + if package_path.startswith("%s/var/cache/yum/" % ROOT_PATH): + try: + os.unlink(package_file) + except OSError as e: + log.debug("unable to remove file %s" % e.strerror) + elif what == rpm.RPMCALLBACK_UNINST_START: + # update status that we're cleaning up %h + #progress.set_text(_("Cleaning up %s" % h)) + pass + elif what in (rpm.RPMCALLBACK_CPIO_ERROR, + rpm.RPMCALLBACK_UNPACK_ERROR, + rpm.RPMCALLBACK_SCRIPT_ERROR): + name = self._get_txmbr(h)[0] + + # Script errors store whether or not they're fatal in "total". So, + # we should only error out for fatal script errors or the cpio and + # unpack problems. + if what != rpm.RPMCALLBACK_SCRIPT_ERROR or total: + exn = PayloadInstallError("cpio, unpack, or fatal script error") + if errorHandler(exn, name) == ERROR_RAISE: + raise exn + + +class YumDepsolveCallback(object): + def __init__(self): + pass + + def transactionPopulation(self): + pass + + def pkgAdded(self, pkgtup, state): + pass + + def tscheck(self): + pass + + def downloadHeader(self, name): + pass + + def procReq(self, name, need): + pass + + def procConflict(self, name, need): + pass + + def end(self): + pass def show_groups(): ksdata = makeVersion() @@ -846,6 +1126,23 @@ def show_groups(): print obj.groups +def print_txmbrs(payload, f=None): + if f is None: + f = sys.stdout + + print >> f, "###########" + for txmbr in payload._yum.tsInfo.getMembers(): + print >> f, txmbr + print >> f, "###########" + +def write_txmbrs(payload, filename): + if os.path.exists(filename): + os.unlink(filename) + + f = open(filename, 'w') + print_txmbrs(payload, f) + f.close() + ### ### MAIN ### @@ -867,6 +1164,30 @@ if __name__ == "__main__": payload = YumPayload(ksdata) payload.setup(storage) + import sys + import os + payload.install_log = sys.stdout for repo in payload._yum.repos.repos.values(): print repo.name, repo.enabled - print payload.groups + #print payload.groups + #sys.exit(0) + for group in payload.groups: + payload.deselectGroup(group) + + payload.selectGroup("core") + payload.selectGroup("base") + payload.checkSoftwareSelection() + write_txmbrs(payload, "tx.1") + + payload.selectGroup("development-tools") + payload.selectGroup("development-libs") + payload.checkSoftwareSelection() + write_txmbrs(payload, "tx.2") + + payload.deselectGroup("development-tools") + payload.deselectGroup("development-libs") + payload.checkSoftwareSelection() + write_txmbrs(payload, "tx.3") + + payload.install() + write_txmbrs(payload, "tx.4") diff --git a/pyanaconda/storage/devicetree.py b/pyanaconda/storage/devicetree.py index dc31302..1a76080 100644 --- a/pyanaconda/storage/devicetree.py +++ b/pyanaconda/storage/devicetree.py @@ -753,8 +753,8 @@ class DeviceTree(object): # something must be wrong -- if all of the slaves we in # the tree, this device should be as well if device is None: - raise DeviceTreeError("MD RAID device %s not in devicetree after " - "scanning all slaves" % name) + pass + return device def addUdevPartitionDevice(self, info, disk=None):