From cb7c9c21db0548eff6d6d92f899f23a7e9ade0bd Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Mon, 24 Oct 2022 14:41:28 +0200 Subject: [PATCH] phase1: use branch config in factory MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 303 ++++++++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 132 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 0b7c609..eed887b 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -51,7 +51,7 @@ if not os.path.exists("twistd.pid"): ini = configparser.ConfigParser() ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini")) -if "general" not in ini or "phase1" not in ini or "rsync" not in ini: +if "general" not in ini or "phase1" not in ini: raise ValueError("Fix your configuration") inip1 = ini['phase1'] @@ -392,15 +392,24 @@ c['schedulers'].append(ForceScheduler( # only take place on one worker. def IsTaggingRequested(step): - val = step.getProperty("tag") - if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val): - return True - else: - return False + tag = step.getProperty("tag") + return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag) def IsNoMasterBuild(step): return step.getProperty("branch") != "master" +def IsUsignEnabled(step): + branch = step.getProperty("branch") + return branch and branches[branch].get("usign_key") + +def IsSignEnabled(step): + branch = step.getProperty("branch") + return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key") + +def IsKmodArchiveEnabled(step): + branch = step.getProperty("branch") + return branch and branches[branch].get("kmod_archive") + def GetBaseVersion(branch): if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch): return branch.split('-')[1] @@ -418,6 +427,23 @@ def GetVersionPrefix(props): else: return "" +@util.renderer +def GetConfigSeed(props): + branch = props.getProperty("branch") + return branch and branches[branch].get("config_seed") or "" + +@util.renderer +def GetRsyncParams(props, srcorbin, urlorkey): + # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key' + branch = props.getProperty("branch") + opt = srcorbin + "_" + urlorkey + return branch and branches[branch].get(opt) + +@util.renderer +def GetUsignKey(props): + branch = props.getProperty("branch") + return branch and branches[branch].get("usign_key") + def GetNextBuild(builder, requests): for r in requests: if r.properties and r.properties.hasProperty("tag"): @@ -487,8 +513,12 @@ def IsTargetSelected(target): return CheckTargetProperty -def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): +@util.renderer +def UsignSec2Pub(props): + branch = props.getProperty("branch") try: + comment = branches[branch].get("usign_comment") or "untrusted comment: secret key" + seckey = branches[branch].get("usign_key") seckey = base64.b64decode(seckey) except: return None @@ -660,19 +690,18 @@ for target in targets: )) # seed config - if config_seed is not None: - factory.addStep(StringDownload( - name = "dlconfigseed", - s = config_seed + '\n', - workerdest = ".config", - mode = 0o644 - )) + factory.addStep(StringDownload( + name = "dlconfigseed", + s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed), + workerdest = ".config", + mode = 0o644 + )) # configure factory.addStep(ShellCommand( name = "newconfig", description = "Seeding .config", - command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n') + command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey) )) factory.addStep(ShellCommand( @@ -707,27 +736,29 @@ for target in targets: command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"])) # install build key - if usign_key is not None: - factory.addStep(StringDownload( - name = "dlkeybuildpub", - s = UsignSec2Pub(usign_key, usign_comment), - workerdest = "key-build.pub", - mode = 0o600, - )) - - factory.addStep(StringDownload( - name = "dlkeybuild", - s = "# fake private key", - workerdest = "key-build", - mode = 0o600, - )) - - factory.addStep(StringDownload( - name = "dlkeybuilducert", - s = "# fake certificate", - workerdest = "key-build.ucert", - mode = 0o600, - )) + factory.addStep(StringDownload( + name = "dlkeybuildpub", + s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub), + workerdest = "key-build.pub", + mode = 0o600, + doStepIf = IsUsignEnabled, + )) + + factory.addStep(StringDownload( + name = "dlkeybuild", + s = "# fake private key", + workerdest = "key-build", + mode = 0o600, + doStepIf = IsUsignEnabled, + )) + + factory.addStep(StringDownload( + name = "dlkeybuilducert", + s = "# fake certificate", + workerdest = "key-build.ucert", + mode = 0o600, + doStepIf = IsUsignEnabled, + )) # prepare dl factory.addStep(ShellCommand( @@ -859,75 +890,83 @@ for target in targets: haltOnFailure = True )) - if enable_kmod_archive: - factory.addStep(ShellCommand( - name = "kmoddir", - description = "Creating kmod directory", - command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])], - haltOnFailure = True - )) - - factory.addStep(ShellCommand( - name = "kmodprepare", - description = "Preparing kmod archive", - command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va", - Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]), - Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], - haltOnFailure = True - )) - - factory.addStep(ShellCommand( - name = "kmodindex", - description = "Indexing kmod archive", - command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=", - Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], - env = MakeEnv(), - haltOnFailure = True - )) + factory.addStep(ShellCommand( + name = "kmoddir", + description = "Creating kmod directory", + command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])], + haltOnFailure = True, + doStepIf = IsKmodArchiveEnabled, + )) + + factory.addStep(ShellCommand( + name = "kmodprepare", + description = "Preparing kmod archive", + command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va", + Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]), + Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], + haltOnFailure = True, + doStepIf = IsKmodArchiveEnabled, + )) + + factory.addStep(ShellCommand( + name = "kmodindex", + description = "Indexing kmod archive", + command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=", + Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], + env = MakeEnv(), + haltOnFailure = True, + doStepIf = IsKmodArchiveEnabled, + )) # sign - if ini.has_option("gpg", "key") or usign_key is not None: - factory.addStep(MasterShellCommand( - name = "signprepare", - description = "Preparing temporary signing directory", - command = ["mkdir", "-p", "%s/signing" %(work_dir)], - haltOnFailure = True - )) - - factory.addStep(ShellCommand( - name = "signpack", - description = "Packing files to sign", - command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]), - haltOnFailure = True - )) - - factory.addStep(FileUpload( - workersrc = "sign.tar.gz", - masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), - haltOnFailure = True - )) - - factory.addStep(MasterShellCommand( - name = "signfiles", - description = "Signing files", - command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])], - env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") }, - haltOnFailure = True - )) - - factory.addStep(FileDownload( - name = "dlsigntargz", - mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), - workerdest = "sign.tar.gz", - haltOnFailure = True - )) - - factory.addStep(ShellCommand( - name = "signunpack", - description = "Unpacking signed files", - command = ["tar", "-xzf", "sign.tar.gz"], - haltOnFailure = True - )) + factory.addStep(MasterShellCommand( + name = "signprepare", + description = "Preparing temporary signing directory", + command = ["mkdir", "-p", "%s/signing" %(work_dir)], + haltOnFailure = True, + doStepIf = IsSignEnabled, + + )) + + factory.addStep(ShellCommand( + name = "signpack", + description = "Packing files to sign", + command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]), + haltOnFailure = True, + doStepIf = IsSignEnabled, + )) + + factory.addStep(FileUpload( + workersrc = "sign.tar.gz", + masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), + haltOnFailure = True, + doStepIf = IsSignEnabled, + )) + + factory.addStep(MasterShellCommand( + name = "signfiles", + description = "Signing files", + command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])], + env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") }, + haltOnFailure = True, + doStepIf = IsSignEnabled, + )) + + factory.addStep(FileDownload( + name = "dlsigntargz", + mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), + workerdest = "sign.tar.gz", + haltOnFailure = True, + doStepIf = IsSignEnabled, + )) + + factory.addStep(ShellCommand( + name = "signunpack", + description = "Unpacking signed files", + command = ["tar", "-xzf", "sign.tar.gz"], + haltOnFailure = True, + doStepIf = IsSignEnabled, + )) # upload factory.addStep(ShellCommand( @@ -945,19 +984,19 @@ for target in targets: haltOnFailure = True )) - if enable_kmod_archive: - factory.addStep(ShellCommand( - name = "kmoddirprepare", - description = "Preparing kmod archive upload directory", - command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], - haltOnFailure = True - )) + factory.addStep(ShellCommand( + name = "kmoddirprepare", + description = "Preparing kmod archive upload directory", + command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], + haltOnFailure = True, + doStepIf = IsKmodArchiveEnabled, + )) factory.addStep(ShellCommand( name = "dirupload", description = "Uploading directory structure", - command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)], - env={'RSYNC_PASSWORD': rsync_bin_key}, + command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, logEnviron = False, locks = NetLockUl, @@ -967,8 +1006,8 @@ for target in targets: factory.addStep(ShellCommand( name = "target-sha256sums", description = "Fetching remote sha256sums for target", - command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"], - env={'RSYNC_PASSWORD': rsync_bin_key}, + command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, logEnviron = False, haltOnFailure = False, flunkOnFailure = False, @@ -1003,8 +1042,8 @@ for target in targets: description = "Uploading target files", command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts + ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), - Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], - env={'RSYNC_PASSWORD': rsync_bin_key}, + Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, logEnviron = False, )) @@ -1015,25 +1054,25 @@ for target in targets: description = "Pruning target files", command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts + ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), - Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], - env={'RSYNC_PASSWORD': rsync_bin_key}, + Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, logEnviron = False, locks = NetLockUl, )) - if enable_kmod_archive: - factory.addStep(ShellCommand( - name = "kmodupload", - description = "Uploading kmod archive", - command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts + - ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]), - Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], - env={'RSYNC_PASSWORD': rsync_bin_key}, - haltOnFailure = True, - logEnviron = False, - locks = NetLockUl, - )) + factory.addStep(ShellCommand( + name = "kmodupload", + description = "Uploading kmod archive", + command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts + + ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]), + Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, + haltOnFailure = True, + logEnviron = False, + locks = NetLockUl, + doStepIf = IsKmodArchiveEnabled, + )) factory.addStep(ShellCommand( name = "sourcelist", @@ -1046,8 +1085,8 @@ for target in targets: name = "sourceupload", description = "Uploading source archives", command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts + - [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)], - env={'RSYNC_PASSWORD': rsync_src_key}, + [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))], + env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) }, haltOnFailure = True, logEnviron = False, locks = NetLockUl, -- 2.30.2