From f5ba40b510598341d2bb1e57ceabd6191469cf9f Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 16 May 2016 12:40:22 +0200 Subject: [PATCH] Determine number of cores dynamically Signed-off-by: Jo-Philipp Wich --- phase1/master.cfg | 25 ++++++++++++++++--------- phase2/master.cfg | 10 ++++++++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index fa3161d..9018c5e 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -106,7 +106,9 @@ c['schedulers'].append(ForceScheduler( from buildbot.process.factory import BuildFactory from buildbot.steps.source import Git from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import SetProperty from buildbot.steps.transfer import FileDownload +from buildbot.process.properties import WithProperties MakeTargetMap = { @@ -239,6 +241,12 @@ for target in targets: factory = BuildFactory() + # find number of cores + factory.addStep(SetProperty( + name = "nproc", + description = "Finding number of CPUs", + command = ["nproc"])) + # check out the source factory.addStep(Git(repourl=repo_url, mode='update')) @@ -280,7 +288,7 @@ CONFIG_DEVEL=y CONFIG_CCACHE=y CONFIG_SIGNED_PACKAGES=y # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set -CONFIG_KERNEL_KALLSYMS=y +CONFIG_KERNEL_KALLSYMS=y CONFIG_COLLECT_KERNEL_DEBUG=y EOT''' %(ts[0], ts[0], ts[1]) )) @@ -334,7 +342,7 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "dlrun", description = "Populating dl/", - command = ["make", "-j4", "download", "V=s"], + command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"], logEnviron = False, locks = [dlLock.access('exclusive')] )) @@ -357,21 +365,21 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "tools", description = "Building tools", - command = ["make", "-j4", "tools/install", "V=s"], + command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "toolchain", description = "Building toolchain", - command=["make", "-j4", "toolchain/install", "V=s"], + command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "kmods", description = "Building kmods", - command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -379,7 +387,7 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "pkgbuild", description = "Building packages", - command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -388,7 +396,7 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "pkginstall", description = "Installing packages", - command=["make", "-j4", "package/install", "V=s"], + command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"], doStepIf = isChangeBuiltin, haltOnFailure = True )) @@ -396,7 +404,7 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "pkgindex", description = "Indexing packages", - command=["make", "-j4", "package/index", "V=s"], + command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"], haltOnFailure = True )) @@ -508,4 +516,3 @@ c['db'] = { # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } - diff --git a/phase2/master.cfg b/phase2/master.cfg index 21862d4..39e36e5 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -112,6 +112,7 @@ c['schedulers'].append(ForceScheduler( from buildbot.process.factory import BuildFactory from buildbot.steps.source import Git from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import SetProperty from buildbot.steps.transfer import FileDownload from buildbot.process.properties import WithProperties @@ -129,6 +130,12 @@ for arch in arches: factory = BuildFactory() + # find number of cores + factory.addStep(SetProperty( + name = "nproc", + description = "Finding number of CPUs", + command = ["nproc"])) + # prepare workspace factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755)) @@ -188,7 +195,7 @@ for arch in arches: name = "compile", description = "Building packages", workdir = "build/sdk", - command = ["make", "-j4", "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"])) + command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"])) factory.addStep(ShellCommand( name = "uploadprepare", @@ -304,4 +311,3 @@ c['db'] = { # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } - -- 2.30.2