phase1,phase2: implement round robin builds

Gather newest complete and not skipped build timestamps, reverse sort
them and use that as builder priority, so the newest built targets are
at the bottom.

Signed-off-by: Petr Štetiar <ynezz@true.cz>

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 2ccac8e9705972592b345865a7fa3e99545c1ade..53d1812fd081362fc1bb142a1446d4823c6792ed 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -4,13 +4,17 @@
 import os
 import re
 import base64
-import random
 import subprocess
 import configparser
 
-from datetime import timedelta
+from dateutil.tz import tzutc
+from datetime import datetime, timedelta
+
+from twisted.internet import defer
+from twisted.python import log
 
 from buildbot import locks
+from buildbot.data import resultspec
 from buildbot.changes import filter
 from buildbot.changes.gitpoller import GitPoller
 from buildbot.config import BuilderConfig
@@ -19,6 +23,7 @@ from buildbot.plugins import schedulers
 from buildbot.plugins import steps
 from buildbot.plugins import util
 from buildbot.process import properties
+from buildbot.process import results
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate
 from buildbot.process.properties import Property
@@ -123,6 +128,69 @@ c['configurators'] = [util.JanitorConfigurator(
     hour=6,
 )]
 
+@defer.inlineCallbacks
+def getNewestCompleteTime(bldr):
+       """Returns the complete_at of the latest completed and not SKIPPED
+       build request for this builder, or None if there are no such build
+       requests. We need to filter out SKIPPED requests because we're
+       using collapseRequests=True which is unfortunately marking all
+       previous requests as complete when new buildset is created.
+
+       @returns: datetime instance or None, via Deferred
+       """
+
+       bldrid = yield bldr.getBuilderId()
+       completed = yield bldr.master.data.get(
+                       ('builders', bldrid, 'buildrequests'),
+                       [
+                               resultspec.Filter('complete', 'eq', [True]),
+                               resultspec.Filter('results', 'ne', [results.SKIPPED]),
+                       ],
+                       order=['-complete_at'], limit=1)
+       if not completed:
+               return
+
+       return completed[0]['complete_at']
+
+@defer.inlineCallbacks
+def prioritizeBuilders(master, builders):
+       """Returns sorted list of builders by their last timestamp of completed and
+       not skipped build.
+
+       @returns: list of sorted builders
+       """
+
+       def is_building(bldr):
+               return bool(bldr.building) or bool(bldr.old_building)
+
+       def bldr_info(bldr):
+               d = defer.maybeDeferred(getNewestCompleteTime, bldr)
+               d.addCallback(lambda complete_at: (complete_at, bldr))
+               return d
+
+       def bldr_sort(item):
+               (complete_at, bldr) = item
+
+               if not complete_at:
+                       date = datetime.min
+                       complete_at = date.replace(tzinfo=tzutc())
+
+               if is_building(bldr):
+                       date = datetime.max
+                       complete_at = date.replace(tzinfo=tzutc())
+
+               return (complete_at, bldr.name)
+
+       results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
+       results.sort(key=bldr_sort)
+
+       for r in results:
+               log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
+
+       return [r[1] for r in results]
+
+c['prioritizeBuilders'] = prioritizeBuilders
+
 ####### CHANGESOURCES
 
 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
@@ -456,13 +524,10 @@ def GetNextBuild(builder, requests):
        for r in requests:
                if r.properties and r.properties.hasProperty("tag"):
                        return r
-       return requests[0]
 
-def prioritizeBuilders(buildmaster, builders):
-       random.shuffle(builders)
-       return builders
-
-c['prioritizeBuilders'] = prioritizeBuilders
+       r = requests[0]
+       log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
+       return r
 
 def MakeEnv(overrides=None, tryccache=False):
        env = {

diff --git a/phase2/master.cfg b/phase2/master.cfg
index ac44fcd6b929244ba043d5677ab5639a2525fe49..a54c191c982c8d4ee485b2ba5e2d53b9dc7a8376 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -8,15 +8,21 @@ import random
 import subprocess
 import configparser
 
-from datetime import timedelta
+from dateutil.tz import tzutc
+from datetime import datetime, timedelta
+
+from twisted.internet import defer
+from twisted.python import log
 
 from buildbot import locks
+from buildbot.data import resultspec
 from buildbot.changes import filter
 from buildbot.changes.gitpoller import GitPoller
 from buildbot.config import BuilderConfig
 from buildbot.plugins import schedulers
 from buildbot.plugins import steps
 from buildbot.plugins import util
+from buildbot.process import results
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Property
 from buildbot.process.properties import WithProperties
@@ -319,9 +325,66 @@ def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
 def IsSharedWorkdir(step):
        return bool(step.getProperty("shared_wd"))
 
-def prioritizeBuilders(buildmaster, builders):
-       random.shuffle(builders)
-       return builders
+@defer.inlineCallbacks
+def getNewestCompleteTime(bldr):
+       """Returns the complete_at of the latest completed and not SKIPPED
+       build request for this builder, or None if there are no such build
+       requests. We need to filter out SKIPPED requests because we're
+       using collapseRequests=True which is unfortunately marking all
+       previous requests as complete when new buildset is created.
+
+       @returns: datetime instance or None, via Deferred
+       """
+
+       bldrid = yield bldr.getBuilderId()
+       completed = yield bldr.master.data.get(
+                       ('builders', bldrid, 'buildrequests'),
+                       [
+                               resultspec.Filter('complete', 'eq', [True]),
+                               resultspec.Filter('results', 'ne', [results.SKIPPED]),
+                       ],
+                       order=['-complete_at'], limit=1)
+       if not completed:
+               return
+
+       return completed[0]['complete_at']
+
+@defer.inlineCallbacks
+def prioritizeBuilders(master, builders):
+       """Returns sorted list of builders by their last timestamp of completed and
+       not skipped build.
+
+       @returns: list of sorted builders
+       """
+
+       def is_building(bldr):
+               return bool(bldr.building) or bool(bldr.old_building)
+
+       def bldr_info(bldr):
+               d = defer.maybeDeferred(getNewestCompleteTime, bldr)
+               d.addCallback(lambda complete_at: (complete_at, bldr))
+               return d
+
+       def bldr_sort(item):
+               (complete_at, bldr) = item
+
+               if not complete_at:
+                       date = datetime.min
+                       complete_at = date.replace(tzinfo=tzutc())
+
+               if is_building(bldr):
+                       date = datetime.max
+                       complete_at = date.replace(tzinfo=tzutc())
+
+               return (complete_at, bldr.name)
+
+       results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
+       results.sort(key=bldr_sort)
+
+       for r in results:
+               log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
+
+       return [r[1] for r in results]
 
 c['prioritizeBuilders'] = prioritizeBuilders
 c['builders'] = []