From: Thibaut VARÈNE <hacks@slashdirt.org>
Date: Thu, 20 Oct 2022 13:30:45 +0000 (+0200)
Subject: phase1: rework GetNumJobs()
X-Git-Tag: v2~52
X-Git-Url: http://git.cdn.openwrt.org/?a=commitdiff_plain;h=4402510d845ccf2018bf58a7fe1c8315fef9439c;p=buildbot.git

phase1: rework GetNumJobs()

Ensure we never end up with "0".
Set a build property step with the computed concurrency level, so that
this information is available in build status.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>
---

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 945533f..f60b778 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -31,7 +31,7 @@ from buildbot.schedulers.basic import SingleBranchScheduler
 from buildbot.schedulers.forcesched import BaseParameter
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.schedulers.forcesched import ValidationError
-from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.master import MasterShellCommand, SetProperty
 from buildbot.steps.shell import SetPropertyFromCommand
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.source.git import Git
@@ -425,10 +425,9 @@ def GetVersionPrefix(props):
 
 @properties.renderer
 def GetNumJobs(props):
-	if props.hasProperty("max_builds") and props.hasProperty("nproc"):
-		return str(int(int(props["nproc"]) / props["max_builds"]))
-	else:
-		return "1"
+	nproc = int(props.getProperty("nproc", "1"))
+	njobs = int(nproc / props.getProperty("max_builds", 1)) or 1
+	return str(njobs)
 
 def GetNextBuild(builder, requests):
 	for r in requests:
@@ -546,6 +545,13 @@ for target in targets:
 		description = "Finding number of CPUs",
 		command = ["nproc"]))
 
+	# set number of jobs
+	factory.addStep(SetProperty(
+		name = "njobs",
+		property = "njobs",
+		description = "Set max concurrency",
+		value = Interpolate("%(kw:jobs)s", jobs=GetNumJobs)))
+
 	# find gcc and g++ compilers
 	factory.addStep(FileDownload(
 		name = "dlfindbinpl",
@@ -800,7 +806,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "dltar",
 		description = "Building and installing GNU tar",
-		command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+		command = ["make", Interpolate("-j%(prop:njobs)s"), "tools/tar/compile", "V=s"],
 		env = MakeEnv(tryccache=True),
 		haltOnFailure = True
 	))
@@ -809,7 +815,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "dlrun",
 		description = "Populating dl/",
-		command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+		command = ["make", Interpolate("-j%(prop:njobs)s"), "download", "V=s"],
 		env = MakeEnv(),
 		logEnviron = False,
 		locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
@@ -825,7 +831,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "tools",
 		description = "Building and installing tools",
-		command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+		command = ["make", Interpolate("-j%(prop:njobs)s"), "tools/install", "V=s"],
 		env = MakeEnv(tryccache=True),
 		haltOnFailure = True
 	))
@@ -833,7 +839,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "toolchain",
 		description = "Building and installing toolchain",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "toolchain/install", "V=s"],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -841,7 +847,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "kmods",
 		description = "Building kmods",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -864,7 +870,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "pkgbuild",
 		description = "Building packages",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -872,7 +878,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "pkginstall",
 		description = "Installing packages",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "package/install", "V=s"],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -880,7 +886,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "pkgindex",
 		description = "Indexing packages",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -888,7 +894,7 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "images",
 		description = "Building and installing images",
-		command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+		command=["make", Interpolate("-j%(prop:njobs)s"), "target/install", "V=s"],
 		env = MakeEnv(),
 		haltOnFailure = True
 	))
@@ -937,7 +943,7 @@ for target in targets:
 		factory.addStep(ShellCommand(
 			name = "kmodindex",
 			description = "Indexing kmod archive",
-			command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+			command=["make", Interpolate("-j%(prop:njobs)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 				Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 			env = MakeEnv(),
 			haltOnFailure = True