git » autoupdaters.git » commit d52b3ea

diff --git a/.gitignore b/.gitignore
index bee8a64..ab78e92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 __pycache__
+/linux-kbb/linux*/
diff --git a/linux-kbb/bisect-tools/bad.sh b/linux-kbb/bisect-tools/bad.sh
new file mode 100755
index 0000000..3c10295
--- /dev/null
+++ b/linux-kbb/bisect-tools/bad.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+git reset --hard
+git bisect bad
diff --git a/linux-kbb/bisect-tools/build.sh b/linux-kbb/bisect-tools/build.sh
new file mode 100755
index 0000000..fa19537
--- /dev/null
+++ b/linux-kbb/bisect-tools/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+set -x
+git cherry-pick -n 480b54e36c6
+git cherry-pick -n ab969bff88e
+make olddefconfig
+make -j3 zImage modules dtbs
+rm -rf inst
+mkdir -p inst/{boot,usr/lib/modules}
+make INSTALL_MOD_PATH="inst/usr" modules_install
+make INSTALL_DTBS_PATH="inst/boot/dtbs-test" dtbs_install
+cp arch/arm/boot/zImage "inst/boot/zImage-test"
+cd inst
+tar czf test.tar.gz boot usr
+cd ..
+
diff --git a/linux-kbb/bisect-tools/good.sh b/linux-kbb/bisect-tools/good.sh
new file mode 100755
index 0000000..db71645
--- /dev/null
+++ b/linux-kbb/bisect-tools/good.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+git reset --hard
+git bisect good
diff --git a/linux-kbb/build-armlfs-6.6.sh b/linux-kbb/build-armlfs-6.6.sh
new file mode 100755
index 0000000..6e9718f
--- /dev/null
+++ b/linux-kbb/build-armlfs-6.6.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+set -x
+# Usage: $0 [git-tag]
+if [ -n "$1" ]; then
+	./patchset-pkgbuild.py "$1" /sources/base-pkgbuilds/linux-armlfs
+fi
+(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs)
diff --git a/linux-kbb/build-armlfs-mainline.sh b/linux-kbb/build-armlfs-mainline.sh
new file mode 100755
index 0000000..076331c
--- /dev/null
+++ b/linux-kbb/build-armlfs-mainline.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+# We just trust that the worktree is at the appropriate tag :P
+set -e
+set -x
+(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs-test)
diff --git a/linux-kbb/get-linux.sh b/linux-kbb/get-linux.sh
new file mode 100755
index 0000000..fd33425
--- /dev/null
+++ b/linux-kbb/get-linux.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+set -e
+SINCE=2022-12-01
+git clone --bare --shallow-since=2021-01-01 github:urjaman/linux-for-my-autobuilder.git linux
+cd linux
+git remote rename origin publish
+echo "	fetch = +refs/heads/*:refs/remotes/publish/*" >> config
+git config pack.packSizeLimit 512m
+git fetch -v --shallow-since=$SINCE
+git remote add mainline https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
+# Note: add stable trees as needed (or runtime with set-branches --add)
+git remote add -t linux-6.1.y -t linux-6.6.y stable https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
+git fetch -v --shallow-since=$SINCE --shallow-exclude=v6.0 mainline
+git fetch -v --shallow-since=$SINCE --shallow-exclude=v6.0.1 stable
+git gc
diff --git a/linux-kbb/kbb.py b/linux-kbb/kbb.py
new file mode 100755
index 0000000..52aa864
--- /dev/null
+++ b/linux-kbb/kbb.py
@@ -0,0 +1,458 @@
+#!/usr/bin/env python3
+
+# kbb aka kernel build bot
+# but not referred to with the full name,
+# since there's many a "kernel build bot" out there...
+# but I dont have a name really, so ... kbb.py it is.
+
+import os
+import sys
+import json
+import urllib.request
+import urllib.error
+import shutil
+import subprocess
+import time
+import datetime
+import traceback
+from subprocess import DEVNULL, PIPE, STDOUT
+from email.message import EmailMessage
+
+def sub(*args, **kwargs):
+    c = subprocess.run(*args, **kwargs)
+    if c.returncode != 0:
+        return False
+    if c.stdout:
+        return c.stdout
+    return True
+
+
+def subc(*args, **kwargs):
+    c = subprocess.run(*args, **kwargs)
+    if c.returncode != 0:
+        print("subprocess failed: ", args)
+        print("code:", c.returncode)
+        sys.exit(1)
+    return c.stdout
+
+# Kernel Build
+class KB:
+    def __init__(self, patchset, verpolicy="mainline", build=None, dir=None):
+        self.patchset = patchset
+        self.verpolicy = verpolicy
+        if dir is None:
+            dir = f"linux-kbb-{patchset}-{verpolicy}"
+        self.dir = dir
+        if build is None:
+            build = f"./build-{patchset}-{verpolicy}.sh"
+        self.indir = False
+        self.mounted = False
+
+    def __str__(self):
+        return self.patchset + "-" + self.verpolicy
+
+    def mkworktree(self):
+        try:
+            os.mkdir(self.dir)
+        except FileExistsError:
+            pass
+        subc(["mount","-t","tmpfs","-o","rw,size=8G","tmpfs",self.dir])
+        self.mounted = True
+        os.chdir("linux")
+        subc(["git", "worktree", "prune"])
+        subc(["git", "worktree", "add", "../" + self.dir])
+        os.chdir("..")
+
+    def cleanup(self):
+        assert self.indir is False, "cleanup while in dir"
+        if self.mounted:
+            subc(["umount", "-l", self.dir])
+        self.mounted = False
+
+    def cd_into(self):
+        assert self.indir is False, "double cd_into"
+        try:
+            _ = os.stat(self.dir + "/Makefile")
+        except FileNotFoundError:
+            mkworktree()
+        os.chdir(self.dir)
+        self.indir = True
+
+    def cd_out(self):
+        os.chdir("..")
+        self.indir = False
+
+
+
+kernels = [
+    KB("armlfs")
+    KB("armlfs", "6.6")
+]
+
+url = "https://www.kernel.org/releases.json"
+
+prev2_releases = "old_releases.json"
+prev_releases = "prev_releases.json"
+curr_fn = "releases.json"
+
+emhost = '\x40tea.urja.dev'
+whoami = f'KBB <kbb{emhost}>'
+toaddr = f'urja{emhost}, urja\x40urja.dev'
+emailproc = ['ssh', 'kbb\x40urja.dev', 'sendmail', '-t']
+
+def vtag4xfer(x):
+    return f"refs/tags/v{x}:refs/tags/v{x}"
+
+def htmlize(s):
+    escapes = {
+        '&': '&amp;',
+        '>': '&gt;',
+        '<': '&lt;'
+    }
+    prefix = '<html><head></head><body><pre>\n'
+    suffix = '</pre></body></html>\n'
+    for k in escapes:
+        s = s.replace(k, escapes[k])
+    return prefix + s + suffix
+
+def mail(subj, logfn = None, log = None):
+    if logfn:
+        with open(logfn) as f:
+            log = f.read()
+
+    attach_threshold = 25
+    msg = EmailMessage()
+    msg['Subject'] = '[KBB] ' + subj
+    msg['From'] = whoami
+    msg['To'] = toaddr
+
+    sniplocator = '*** Waiting for unfinished jobs'
+
+    if log.count('\n') > attach_threshold:
+        attach = log;
+        log = log.splitlines()
+        sniploc = None
+        for i in range(len(log)-1,0,-1):
+            if sniplocator in log[i]:
+                sniploc = i
+                break
+        if sniploc:
+            presnip = "<snip>\n"
+            postsnip = "<snip>\n"
+            if sniploc < attach_threshold:
+                sniploc = attach_threshold
+                presnip = ""
+            log = log[sniploc-attach_threshold:sniploc]
+            if sniploc >= len(log):
+                postsnip = ""
+            log = presnip + '\n'.join(log) + '\n' + postsnip
+        else:
+            log = log[-attach_threshold:]
+            log = '<snip>\n' + '\n'.join(log) + '\n'
+    else:
+        attach = None
+
+    msg.set_content(log)
+    msg.add_alternative(htmlize(log), subtype='html')
+    if attach:
+        msg.add_attachment(attach, filename='log.txt', cte='quoted-printable')
+
+    #print(msg)
+    #subprocess.run(emailproc, input=msg.as_bytes())
+    print(f"Would mail: {subj}")
+
+def json_readf(fn):
+    with open(fn, "r") as f:
+        d = f.read()
+    return json.loads(d)
+
+
+
+
+def versplit(ver):
+    return [int(x) if x.isdigit() else x for x in ver.split(sep=".")]
+
+
+def tag_exists(tag):
+    return sub(["git", "rev-parse", "refs/tags/" + tag], stdout=DEVNULL, stderr=DEVNULL)
+
+def get_current_branch():
+    return subc(["git", "branch", "--show-current"], stdout=PIPE).decode().strip()
+
+
+# Calling rebase and tag "repatch" ... ok.
+def repatch_indir(patchset, newver, verpolicy):
+    tagname = patchset + "-" + newver
+    if os.path.exists(".skipkbb"):
+        return False
+
+    if tag_exists(tagname):
+        return tagname
+
+    oldtag = subc(["git", "describe", "--tags"], stdout=PIPE).decode().strip()
+    (oldset, oldver) = oldtag.split(sep="-", maxsplit=1)
+
+    if '-v' in oldver:
+        oldver, _ = oldver.rsplit(sep="-v", maxsplit=1)
+
+    if verpolicy != "mainline":
+        newno = versplit(newver)
+        oldno = versplit(oldver)
+        if newno[0:2] != oldno[0:2]:
+            mlvertag = patchset + "-" + str(newno[0]) + "." + str(newno[1])
+            if tag_exists(mlvertag):
+                branchname = get_current_branch()
+                subc(["git", "switch", "-C", branchname, "refs/tags/" + mlvertag])
+                (oldset, oldver) = mlvertag.split(sep="-", maxsplit=1)
+
+    if oldset != patchset:
+        print(f"Error: this git tree is on a tag for patchset {oldset}, not {patchset}?")
+        return False
+
+    oldvertag = "tags/v" + oldver
+
+    patches = subc(["git", "rev-list", "--count", oldvertag + "..HEAD"], stdout=PIPE)
+    patches = int(patches.decode())
+
+    vertag = "tags/v" + newver
+
+    rebase_cmd = ["git", "rebase", "HEAD~" + str(patches), "--onto", vertag]
+    rebase_log = ".kbb-rebase-log"
+    rbproc = subprocess.Popen(rebase_cmd, stdin=DEVNULL, stderr=STDOUT, stdout=PIPE)
+    tee = subprocess.Popen(["tee", rebase_log], stdin=rbproc.stdout)
+    r1 = rbproc.wait()
+    tee.wait()
+    if r1:
+        mail("Uhh, rebase trouble with " + tagname, rebase_log)
+        print(
+            "Finish rebase manually. Use 'touch .skipkbb' to skip this kernel for now instead."
+        )
+        sub(["/bin/bash"])
+
+    if os.path.exists(".skipkbb"):
+        return False
+
+    count = subc(["git", "rev-list", "--count", vertag + "..HEAD"], stdout=PIPE)
+    count = int(count.decode())
+    if count != patches:
+        print(f"Info: patchset {patchset} now (version {newver}) has {count} patches.")
+
+    subc(["git", "tag", tagname])
+    return tagname
+
+
+def repatch(k, newver):
+    k.cd_into()
+    r = repatch_indir(k.patchset, newver, k.verpolicy)
+    k.cd_out()
+    return r
+
+
+def find_new_version(k, rels, update_mainline, update_stable):
+    nv = None
+    vp = k.verpolicy
+    if vp == "mainline":
+        if update_mainline:
+            nv = update_mainline
+    elif vp == "stable":
+        if rels["latest_stable"]["version"] in update_stable:
+            nv = rels["latest_stable"]["version"]
+    else:
+        # Operate with integer versions so that we dont consider
+        # like 6.11.x to be a new version of "6.1".
+        vps = versplit(vp)
+        vl = len(vps)
+        for v in update_stable:
+            vs = versplit(v)
+            if vps[0:vl] == vs[0:vl]:
+                nv = v
+    return nv
+
+
+def publish_indir(k, nv, tag):
+    branch = get_current_branch()
+    if tag:
+        tagname = tag
+    else:
+        tagname = k.patchset + "-" + nv
+    pids = str(os.getpid())
+    logfn = f"../log/publish-{k}-v{nv}_{pids}.log"
+    print(f"Publishing git tree, log: {logfn}")
+    b1 = f"{branch}:refs/heads/{branch}"
+    t1 = f"refs/tags/v{nv}:refs/tags/v{nv}"
+    t2 = f"refs/tags/{tagname}:refs/tags/{tagname}"
+    with open(logfn, "w") as f:
+        if not sub(['git','push','-f',"publish", b1, t1, t2], stdin=DEVNULL, stderr=STDOUT, stdout=f):
+            mail(f"Publish failure (build success) {k} {nv}", logfn)
+
+def publish(k, nv, tag=None):
+    k.cd_into()
+    try:
+        publish_indir(k, nv, tag)
+    except Exception:
+        traceback.print_exc()
+    k.cd_out()
+
+
+def build(k, nv, tag):
+    today = datetime.date.today()
+    ymd = today.strftime("%y%m%d")
+    pids = str(os.getpid())
+    logfn = f"log/build-{k}-{ymd}-v{nv}_{pids}.log"
+    print(f"Building - for details see '{logfn}'")
+    with open(logfn, "w") as f:
+        if sub([k.build, tag], stdin=DEVNULL, stderr=STDOUT, stdout=f):
+            print("Done. Return value zero (y).")
+            print("Running publish()..")
+            publish(k, nv, tag)
+            print("Done")
+            return f"{k} {nv}"
+        else:
+            print("Oopsie? Build ended with nonzero return value :(")
+            mail(f"Build failure {k} {nv}", logfn)
+            with open("ATTN.txt", "a") as of:
+                of.write(logfn + "\n")
+            return None
+
+
+def doakernel(k, rels, update_mainline, update_stable):
+    nv = find_new_version(k, rels, update_mainline, update_stable)
+    if not nv:
+        return None
+
+    print(f"Re/patching {k} to version {nv}")
+    tag = repatch(k, nv)
+    if not tag:
+        print("No tag returned - skipping build")
+        return None
+
+    return build(k, nv, tag)
+
+def rebuild_kernel(k):
+    # Figure out the version the kernel is "supposed to" be
+    k.cd_into()
+    kv = subc(["git", "describe", "--tags", "--match", "v*", "--exclude", k.patchset + '-*' ], stdout=PIPE).decode().strip()
+    kv = kv[1:].split(sep='-')
+    if kv[1].startswith("rc"):
+        kv = kv[0] + '-' + kv[1]
+    else:
+        kv = kv[0]
+
+    print(f"Determined the kernel version to be {kv}")
+
+    tagbase = k.patchset + '-' + kv
+    sub(["git", "tag", "-d", tagbase])
+    subc(["git", "tag", tagbase])
+    print(f"(Re-)created tag {tagbase} - now creating a fresh tag for build processes")
+
+    vnum = 2
+    while True:
+        buildtag = f"{tagbase}-v{vnum}"
+        if tag_exists(buildtag):
+            vnum += 1
+            continue
+        subc(["git", "tag", buildtag])
+        break
+    print(f"Tag for build: {buildtag}")
+    k.cd_out()
+    return build(k, kv, buildtag)
+
+
+def successmail(successlist):
+    if successlist:
+        if len(successlist) > 1:
+            mail(f"Success building {len(successlist)} kernels", log="\n".join(successlist) + "\n")
+        else:
+            mail("Success building " + successlist[0], log="\n")
+
+def update_and_build():
+    if not os.path.exists(curr_fn):
+        r = urllib.request.urlopen(url, timeout=30)
+        with open(curr_fn, "w+b") as t:
+            shutil.copyfileobj(r, t)
+
+    if not os.path.exists(prev_releases):
+        os.rename(curr_fn, prev_releases)
+        sys.exit(0)
+
+    rels = json_readf(curr_fn)
+    prevs = json_readf(prev_releases)
+
+    update_stable = []
+    update_mainline = False
+    for r in rels["releases"]:
+        if not r["version"][0].isdigit():
+            continue
+        verno = versplit(r["version"])
+        print(r["moniker"], verno)
+        found = False
+        for o in prevs["releases"]:
+            # mainline is just compared to mainline, rest to matching 2 numbers of version
+            if r["moniker"] == "mainline":
+                if o["moniker"] != "mainline":
+                    continue
+                found = True
+                if r["version"] != o["version"]:
+                    update_mainline = r["version"]
+                break
+            if o["moniker"] == "mainline":
+                continue
+            oldver = versplit(o["version"])
+            if oldver[0:2] == verno[0:2]:
+                found = True
+                if o["version"] != r["version"]:
+                    update_stable.append(r["version"])
+                break
+        if not found:
+            if r["moniker"] == "mainline":
+                update_mainline = r["version"]
+            else:
+                update_stable.append(r["version"])
+
+    print("Update stable:", bool(update_stable), update_stable)
+    print("Update mainline:", bool(update_mainline), update_mainline)
+
+    if update_stable or update_mainline:
+        os.chdir("linux")
+        if update_mainline:
+            print("Fetching mainline")
+            subc(["git", "fetch", "mainline", "master", vtag4xfer(update_mainline)])
+            print("Done")
+
+        if update_stable:
+            print("Fetching stable(s)")
+            targets = [vtag4xfer(x) for x in update_stable]
+            subc(["git", "fetch", "stable"] + targets)
+            print("Done")
+        os.chdir("..")
+
+    successlist = []
+    for k in kernels:
+        r = doakernel(k, rels, update_mainline, update_stable)
+        if r:
+            successlist.append(r)
+        k.cleanup()
+
+    # finally, move prev to old, releases to prev
+    os.replace(prev_releases, prev2_releases)
+    os.replace(curr_fn, prev_releases)
+    successmail(successlist)
+    if successlist:
+        subc(["git", "commit", "-a", "-m", "Automatic update"])
+        subc(["git", "push"])
+
+
+if len(sys.argv) == 1:
+    update_and_build()
+elif len(sys.argv) == 3 and sys.argv[1] == "--rebuild":
+    successlist = []
+    for k in kernels:
+        if str(k) == sys.argv[2]:
+            print(f"Found definition for kernel {k} - trying to rebuild")
+            r = rebuild_kernel(k)
+            if r:
+                successlist.append(r)
+            k.cleanup()
+    successmail(successlist)
+else:
+    print(f"usage: {sys.argv[0]} [--rebuild <patchset-verpolicy>]")
diff --git a/linux-kbb/kbbcron.sh b/linux-kbb/kbbcron.sh
new file mode 100755
index 0000000..ebe908d
--- /dev/null
+++ b/linux-kbb/kbbcron.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+screen -h 1000 -L -Logfile "log/cron-$(date +%y%m%d-%H%M%S).log" -dmS kbbcron -t kbb flock -n kbb.lock ./kbb.py
diff --git a/linux-kbb/patchset-pkgbuild.py b/linux-kbb/patchset-pkgbuild.py
new file mode 100755
index 0000000..42da24e
--- /dev/null
+++ b/linux-kbb/patchset-pkgbuild.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import subprocess
+from subprocess import DEVNULL, PIPE, STDOUT
+
+def subc(*args, **kwargs):
+    c = subprocess.run(*args, **kwargs)
+    if c.returncode != 0:
+        print("subprocess failed: ", args)
+        print("code:", c.returncode)
+        sys.exit(1)
+    return c.stdout
+
+def adjust_pkgbuild(dir, ver, rel):
+    b = dir + '/PKGBUILD'
+
+    with open(b) as f:
+       d = f.readlines()
+
+    vv = ver.split(sep='.')
+
+    with open(b,"w") as f:
+        for L in d:
+            if len(L) and L[-1] == '\n':
+                L = L[:-1]
+            c1 = "_srcname="
+            c2 = "pkgver="
+            c3 = "pkgrel="
+            brk1 = "md5sums=("
+            brk2 = "sha256sums=("
+            if L.startswith(c1):
+                f.write(f'{c1}linux-{vv[0]}.{vv[1]}\n')
+            elif L.startswith(c2):
+                f.write(f'{c2}{ver}\n')
+            elif L.startswith(c3):
+                f.write(f'{c3}{rel}\n')
+            elif L.startswith(brk1) or L.startswith(brk2):
+                break
+            else:
+                f.write(L + '\n')
+
+
+if len(sys.argv) == 1:
+    print(f"usage: {sys.argv[0]} <git-tag> <pkgbuild-dir>")
+    sys.exit(1)
+
+[tag, pkgbdir] = sys.argv[1:]
+
+setname,ver = tag.split(sep='-',maxsplit=1)
+
+if "-v" in ver:
+    ver,rel = ver.rsplit(sep="-v",maxsplit=1)
+else:
+    rel = 1
+
+pkgbdir = os.path.realpath(pkgbdir)
+patchname = pkgbdir + '/' + setname + '.patch'
+
+os.chdir("linux")
+basetag = "tags/v" + ver
+range = basetag + "..tags/" + tag
+count = subc(["git", "rev-list", "--count", range], stdout=PIPE)
+count = int(count.decode())
+
+print(f"{count} patches -> {patchname}")
+
+with open(patchname, 'wb') as f:
+    subc(["git","format-patch","--stdout", range], stdout=f)
+
+adjust_pkgbuild(pkgbdir, ver, rel)
+os.chdir(pkgbdir)
+os.system("makepkg -g >> PKGBUILD")