From 7da6a8d568e8b014dacc6c68f961cc802bc24b7e Mon Sep 17 00:00:00 2001
From: David Lehman <dlehman@redhat.com>
Date: Thu, 17 Apr 2014 16:29:44 -0500
Subject: [PATCH 12/12] Start a new implementation of partition allocation.

Start with a list of every possible arrangement of partition requests
across the available free space regions on the available disks. Filter
the list of layouts using progressively more stringent (and costly)
evaulations until the best layout is identified.

XXX It would probably be worthwhile to pre-process situations like
      a) one request per disk across a large number of disks, and
      b) more identical disks than there are requests
    to reduce the mathematical workload of generating the layouts.
---
 blivet/autopart.py | 518 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 518 insertions(+)
 create mode 100644 blivet/autopart.py

diff --git a/blivet/autopart.py b/blivet/autopart.py
new file mode 100644
index 0000000..811a6d6
--- /dev/null
+++ b/blivet/autopart.py
@@ -0,0 +1,518 @@
+import parted
+from itertools import product, ifilter, imap, izip
+
+from blivet.devices import StorageDevice, PartitionDevice
+from blivet.partitioning import PartitionRequest, DiskChunk, ChunkSet
+from blivet.partitioning import sectorsToSize, sizeToSectors
+from blivet.partitioning import getFreeRegions, addPartition
+from blivet.size import Size
+from blivet.errors import StorageError, PartitioningError
+
+import logging
+log = logging.getLogger("blivet")
+
+"""
+    General approach
+        - generate all possible layout of the requests over the chunks
+        - evaluate each permutation
+            - does it fit?
+            - does it meet all of the requests' requirements?
+            - to what extent are the requests' weights honored?
+            - how much growth does it allow?
+                - how do we evaluate this?
+                    - total growth?
+                    - some sort of equality?
+
+    Considerations
+        - size sets
+        - disk preference (soft)
+        - boot disk
+            - are there any platforms/bootloaders for which the boot request(s)
+              absolutely must be on the boot disk?
+                - how prevalent is that assumption throughout blivet/anaconda?
+
+    TODO
+        - partition type handling
+        - account for size sets in growth estimate?
+        - defuse combinatorial bombs
+        - other optimizations
+            - skip growth code when no requests are growable
+        - actually grow growable partitions
+            - could probably use a lightly modified version of
+              partitioning.growPartitions
+
+"""
+class PartitionAllocator(object):
+    def __init__(self, disks, partitions, size_sets=None, boot_disk=None):
+        """
+            :param disks: usable (partitioned) disks
+            :type disks: sorted list of :class:`~.devices.StorageDevice`
+            :param partitions: requested partitions
+            :type partitions: list of :class:`~.devices.PartitionDevice`
+            :keyword size_sets: list of related requests
+            :type size_sets: list of :class:`~.partitioning.SizeSet`
+            :keyword boot_disk: boot disk
+            :type boot_disk: :class:`~.devices.StorageDevice`
+        """
+        if not all(d.partitioned for d in disks):
+            raise ValueError("all disks must be partitioned")
+
+        if not all(isinstance(p, PartitionDevice) for p in partitions):
+            raise ValueError("partitions must be PartitionDevice instances")
+
+        if boot_disk and not isinstance(boot_disk, StorageDevice):
+            raise ValueError("boot_disk must be StorageDevice instance or None")
+
+        self.disks = disks[:]
+        self.partitions = partitions
+        self.size_sets = size_sets or []
+        self.boot_disk = boot_disk
+
+        self.disk_dict = dict((d.path, d) for d in self.disks)
+        """ device path keys and :class:`~.devices.StorageDevice` values """
+
+        self.chunks = []
+        """ list of :class:`~.partitioning.DiskChunk` """
+
+        self.requests = []
+        """ list of :class:`~.partitioning.PartitionRequest` """
+
+        self.layout_partitions = []
+        """ (disk, start, end, type) for each request for each layout """
+
+        self.layout_growth = []
+        """ growth in sectors for each request in each layout """
+
+    def defuse_combinatorial_bombs(self):
+        # 1. If there are multiple identical requests, each constrained to a
+        #    different disk, include only enough of those disks/requests to
+        #    accommodate the other requests.
+
+        # 2. If there are more identical disks than there are requests, drop the
+        #    extras.
+        pass
+
+    def set_up_chunks(self):
+        """ Create DiskChunk instances representing free space on disks.
+
+            The same set of DiskChunk and PartitionRequest instances are reused
+            as needed. They should be set up again when either the request set
+            or the disks change.
+        """
+        free = getFreeRegions(self.disks)
+        self.chunks = [DiskChunk(f) for f in free]
+        self.requests = [PartitionRequest(p) for p in self.partitions]
+
+    def reset_chunks(self):
+        """ Reset all chunks and requests for reuse with another layout. """
+        log.debug("resetting chunks")
+        map(lambda d: d.format.partedDisk.deleteAllPartitions(), self.disks)
+
+        for request in self.requests:
+            request.device.disk = None
+            request.partedPartition = None
+            request.reset()
+
+        # reset the chunks so they're ready to use again
+        map(lambda c: c.reset(), self.chunks)
+
+        # reset the size sets
+        map(lambda s: s.reset(), self.size_sets)
+
+    def check_layout_fit(self, layout):
+        """ Return True if the given layout fits (before any growth).
+
+            The purpose of this method is to filter out layouts that do not fit
+            to whittle down the set of possible layouts so that the more
+            expensive evaluations can be run on as few layouts as possible. It
+            is okay if it lets through some layouts that don't actually fit due
+            to partition types, alignment, &c.
+        """
+        chunk_free = [c.lengthToSize(c.length) for c in self.chunks]
+        request_idx = 0
+        for chunk_idx in layout:
+            log.debug("check_layout_fit: %s", layout)
+            chunk = self.chunks[chunk_idx]
+            chunk_free[chunk_idx] -= self.requests[request_idx].device.req_base_size
+            if chunk_free[chunk_idx] < 0:
+                return False
+
+            request_idx += 1
+
+        return True
+
+    def check_layout_requirements(self, layout):
+        """ Return True if the layout meets all requests' requirements. """
+        """
+        disk_info = dict()
+        for disk in self.disks:
+            pdisk = disk.format.partedDisk
+            max_primary = pdisk.maxPrimaryPartitionCount
+            max_partitions = pdisk.maxSupportedPartitionCount
+            supports_extended = pdisk.supportsFeature(parted.DISK_TYPE_EXTENDED)
+            primary_count = pdisk.primaryPartitionCount
+            has_extended = pdisk.getExtendedPartition() is not None
+            partition_count = len(pdisk.partitions)
+            disk_info[disk.path] = { "max_partitions": max_partitions,
+                                     "max_primary": max_primary,
+                                     "supports_extended": supports_extended,
+                                     "primary_count": primary_count,
+                                     "has_extended": has_extended,
+                                     "partition_count": partition_count }
+        """
+        request_idx = 0
+        for chunk_idx in layout:
+            log.debug("check_layout_requirements: %s", layout)
+            request = self.requests[request_idx]
+            chunk = self.chunks[chunk_idx]
+            disk = self.disk_dict[chunk.path]
+
+            req_disks = request.device.req_disks
+            if req_disks and disk not in req_disks:
+                return False
+
+            start_sector = request.device.req_start_sector
+            if start_sector is not None and \
+               not chunk.geometry.containsSector(start_sector):
+                return False
+
+            # FIXME: we also need to track the number of primaries including
+            #        those we will create
+            extended = disk.format.extendedPartition
+            if request.device.req_primary and extended and \
+               extended.geometry.contains(chunk.geometry):
+                return False
+
+            # XXX Is this really necessary on all(any) platforms?
+            if self.boot_disk and request.device.req_base_weight > 1000 and \
+               disk != self.boot_disk:
+                return False
+
+            request_idx += 1
+
+        return True
+
+    def get_layout_weight_score(self, layout):
+        """ Score the layout based on request weights.
+
+            Scores are based on each request's weight and its allocation order.
+        """
+        score = 0
+        return score
+
+        weights = []
+        request_idx = 0
+        for chunk_idx in layout:
+            request = self.requests[request_idx]
+            weight = request.device.req_base_weight
+            if not weight:
+                request_idx += 1
+                continue
+
+            weights.append(weight) 
+            score += weight * (request_idx + 1)
+
+    def estimate_layout_growth(self, layout):
+        """ Return a rough estimate of total growth for the given layout.
+
+            The purpose of this method is to try to eliminate significantly
+            under-performing layouts (in terms of growth) to minimize the number
+            of layouts for which we actually run the full request growing
+            routine. If this is not significantly faster than actually growing
+            the requests it serves no purpose.
+
+            As of now this does not account for size sets at all.
+        """
+        # everything is tracked in sectors, which is bad since sector sizes
+        # could vary between disks
+        total_growth = 0
+        chunk_bases = [0] * len(self.chunks)
+        max_by_request = [0] * len(self.requests)
+        chunk_max_part_lengths = [self.disk_dict[c.path].format.partedDisk.maxPartitionLength for c in self.chunks]
+        requests_by_chunk = dict(imap(lambda i: (i, list()),
+                                      range(len(self.chunks))))
+
+        request_idx = 0
+        for chunk_idx in layout:
+            chunk = self.chunks[chunk_idx]
+            requests_by_chunk[chunk_idx].append(request_idx)
+            device = self.partitions[request_idx]
+            chunk_bases[chunk_idx] += chunk.sizeToLength(device.req_base_size)
+            max_size_candidates = [m for m in
+                                    [chunk.sizeToLength(device.req_max_size),
+                                     chunk.sizeToLength(device.format.maxSize),
+                                     chunk_max_part_lengths[chunk_idx]]
+                                    if m > 0]
+            if max_size_candidates:
+               max_by_request[request_idx] = min(max_size_candidates)
+
+            request_idx += 1
+
+        for idx, chunk in enumerate(self.chunks):
+            extra = chunk.length - chunk_bases[idx]
+            log.debug("total growth increased by %s for chunk %d", chunk.lengthToSize(extra), idx)
+            growable = [r for r in requests_by_chunk[idx]
+                                   if self.partitions[r].req_grow]
+            num_growable = len(growable)
+            if extra < num_growable*250:
+                # insignificant growth. move on.
+                continue
+
+            growth = [0] * len(self.requests)
+            for i in range(3):
+                if num_growable == 0:
+                    break
+
+                share = extra/num_growable
+                done = []
+
+                # grow all size-limited requests up to the base share
+                for request_idx in growable:
+                    old_growth = growth[request_idx]
+                    new_growth = min(max_by_request[request_idx],
+                                     old_growth + share)
+                    if new_growth == old_growth:
+                        continue
+
+                    delta = old_growth - new_growth
+                    growth[request_idx] = new_growth
+                    extra += delta
+                    total_growth -= delta
+                    if new_growth == max_by_request[request_idx]:
+                        done.append(request_idx)
+                        num_growable -= 1
+                        continue
+
+        return total_growth
+
+    def grow_layout(self, layout):
+        """ Grow requests and return total request growth for the layout.
+
+            Growth amounts are saved for each request in each layout so that we
+            only run the calculations once.
+        """
+        log.debug("BEGIN GROW %s", layout)
+        # set up the chunks to reflect the layout
+        request_idx = 0
+        for chunk_idx in layout:
+            request = self.requests[request_idx]
+            chunk = self.chunks[chunk_idx]
+            request.reset() # reset it since the partition has been allocated
+            chunk.addRequest(request)
+            request_idx += 1
+
+        # grow the chunks' requests
+        total_growth = 0
+        chunk_set = ChunkSet(chunks=self.chunks, size_sets=self.size_sets)
+        chunk_set.growRequests()#uniform=True)
+
+        log.debug("END GROW %s", layout)
+
+        # calculate total layout growth
+        total_growth = sum(c.growth for c in self.chunks)
+
+        # save the growth amount for each request for this layout
+        layout_idx = self.layouts.index(layout)
+        self.layout_growth[layout_idx] = [r.growth for r in self.requests]
+        log.debug("%s", [self.chunks[0].lengthToSize(l) for l in self.layout_growth[layout_idx]])
+
+        return total_growth
+
+    def allocate_base_partitions(self, layout):
+        """ Allocate partitions at their base sizes according to the layout. """
+        layout_idx = self.layouts.index(layout)
+
+        ## 1. get a list of requests for each chunk
+        requests_by_chunk = dict(imap(lambda i: (i, list()),
+                                      range(len(self.chunks))))
+        request_idx = 0
+        for chunk_idx in layout:
+            requests_by_chunk[chunk_idx].append(request_idx)
+            request_idx += 1
+
+        # Create new identical geometries representing each chunk. We will
+        # adjust these as we allocate each base partition.
+        free_by_chunk = [parted.Geometry(device=c.geometry.device,
+                                         start=c.geometry.start,
+                                         end=c.geometry.end)
+                         for c in self.chunks]
+
+        for (chunk_idx, request_indices) in requests_by_chunk.iteritems():
+            ## 2. sort the request index list by partition weight
+            ## FIXME: handle asPrimary here, too, or express it as a weight.
+            chunk = self.chunks[chunk_idx]
+            request_indices.sort(key=lambda r: self.partitions[r].req_base_weight, reverse=True)
+            disk = self.disk_dict[chunk.path]
+            alignment = disk.format.alignment
+            free = free_by_chunk[chunk_idx]
+
+            ## 3. allocate each one at its base size at the next aligned sector
+            part_type = parted.PARTITION_NORMAL
+            extended = disk.format.extendedPartition
+            if extended and extended.geometry.contains(free):
+                part_type = parted.PARTITION_LOGICAL
+
+            # FIXME: we need to be able to create an extended here
+
+            # FIXME: check partition count limits somewhere
+
+            for request_idx in request_indices:
+                request = self.requests[request_idx]
+                try:
+                    partition = addPartition(disk.format, free, part_type,
+                                             request.device.req_base_size)
+                except Exception as e:
+                    raise StorageError(e.message)
+
+                #part = disk.format.partedDisk.getPartitionByPath(part.path)
+                request.device.partedPartition = partition
+                request.device.disk = disk
+
+                # delay resetting the request to account for the allocation
+                # until grow_layout; if this ends up a duplicate layout we can
+                # save that time instead of wasting it.
+
+                free.start = partition.geometry.end + 1
+
+                # now save the base partition info so we don't have to do that
+                # again
+                part_info = (disk.path, partition.type,
+                             partition.geometry.start, partition.geometry.end)
+                self.layout_partitions[layout_idx][request_idx] = part_info
+
+    def get_possible_layouts(self):
+        """ Generate a list of all possible layouts.
+
+            We end up with a list of lists. Each list's indices correspond to
+            those in the list of requests and each value references an index in
+            the chunk list. This tells us which chunk each request would go in
+            for that layout. It does not tell us an order for the partitions
+            within a chunk.
+        """
+        all_layouts = product(range(len(self.chunks)),
+                              repeat=len(self.requests))
+        fitting_layouts = ifilter(self.check_layout_fit, all_layouts)
+        working_layouts = ifilter(self.check_layout_requirements,
+                                  fitting_layouts)
+        return working_layouts
+
+    def get_best_layout(self):
+        """ Choose the best layout based on placement and growth.
+
+            Allocate and grow partitions for each layout. Score each layout
+            based primarily on placement of weighted requests and secondarily on
+            total growth.
+        """
+        self.set_up_chunks()
+        log.debug("partition order: %s", [d.name for d in self.partitions])
+        self.layouts = list(self.get_possible_layouts())
+
+        for layout in self.layouts:
+            log.debug("layout: %s", layout)
+
+        weight_scores = [0] * len(self.layouts)
+        disks_scores = [0] * len(self.layouts)
+        growth_scores = [len(self.disks)] * len(self.layouts)
+        for idx, layout in enumerate(self.layouts):
+            self.layout_partitions.append([None] * len(layout))
+            self.layout_growth.append([])
+
+            try:
+                self.allocate_base_partitions(layout)
+            except StorageError as e:
+                log.info("discarding layout: %s", e)
+                self.reset_chunks()
+                continue
+
+            # XXX Trim the set of layouts to eliminate duplicates
+            #     since grow_layout is so expensive.
+            part_info = self.layout_partitions[idx]
+            duplicate = False
+            for layout_partitions in self.layout_partitions[:idx]:
+                # boot disk is handled by check_layout_requirements
+                if all(z[0][1:] == z[1][1:]
+                        for z in izip(part_info, layout_partitions)
+                         if z[1] is not None):
+                    duplicate = True
+                    break
+
+            if duplicate:
+                log.info("discarding duplicate layout (%d)", idx)
+                self.reset_chunks()
+                continue
+
+            try:
+                growth_scores[idx] = self.estimate_layout_growth(layout)
+                #growth_scores[idx] = self.grow_layout(layout)
+            except StorageError as e:
+                log.error("error growing layout: %s", e)
+                raise
+
+            weight_scores[idx] = self.get_layout_weight_score(layout)
+            disks_scores[idx] = len(set(layout))
+
+            self.reset_chunks()
+
+        layouts = range(len(self.layouts))
+
+        ## weight
+        best_weight_score = max(weight_scores)
+        layouts = [i for i in layouts if weight_scores[i] == best_weight_score]
+        for idx in layouts:
+            log.debug("weight winner: %s", self.layouts[idx])
+
+        if len(layouts) < 1:
+            layouts = range(len(self.layouts))
+        elif len(layouts) > 1:
+            ## growth
+            log.debug("pre-growth: %s", [str(self.chunks[0].lengthToSize(s)) for s in growth_scores])
+            best_growth_score = 0
+            for idx, score in enumerate(growth_scores):
+                if idx in layouts and score > best_growth_score:
+                    best_growth_score = score
+
+            GROWTH_MARGIN = 0.7
+            layouts = [i for i in layouts if growth_scores[i] > best_growth_score*GROWTH_MARGIN]
+            for idx in layouts:
+                log.debug("growth pre-winner: %s", self.layouts[idx])
+                growth_scores[idx] = self.grow_layout(self.layouts[idx])
+                self.reset_chunks()
+
+            log.debug("growth: %s", [str(self.chunks[0].lengthToSize(s)) for s in growth_scores])
+            best_growth_score = 0
+            for idx, score in enumerate(growth_scores):
+                if idx in layouts and score > best_growth_score:
+                    best_growth_score = score
+
+            layouts = [i for i in layouts if growth_scores[i] == best_growth_score]
+            for idx in layouts:
+                log.debug("growth winner: %s", self.layouts[idx])
+
+            if len(layouts) < 1:
+                layouts = range(len(self.layouts))
+            elif len(layouts) > 1:
+                ## disks
+                best_disks_score = len(self.disks)
+                for idx, score in enumerate(disks_scores):
+                    if idx in layouts and score < best_disks_score:
+                        best_disks_score = score
+
+                 # FIXME: disks score should also favor using first/lowest disk
+                layouts = [i for i in layouts if disks_scores[i] == best_disks_score]
+                for idx in layouts:
+                    log.debug("disks winner: %s", self.layouts[idx])
+
+                if not layouts:
+                    layouts = range(len(self.layouts))
+
+        for idx in layouts:
+            log.debug("winner: %s", self.layouts[idx])
+
+        best_layout = self.layouts[layouts[0]]
+
+        if best_layout is None:
+            log.error("failed to find any suitable layout")
+            raise PartitioningError("failed to find a suitable layout")
+
+        log.info("best layout: %s", best_layout)
+        return best_layout
-- 
1.9.3

