git » autoupdaters.git » commit af7fdc5

Add an unified infra to call mpkg.sh, and log stuff

author Urja (ARMLFS builder)
2024-07-30 19:08:30 UTC
committer Urja (ARMLFS builder)
2024-07-30 19:08:30 UTC
parent 49b86c7cfcd49844bf223e9f62d79daa4157fa28

Add an unified infra to call mpkg.sh, and log stuff

So that there's some method for me to get reports on things
the build server has done by itself.

autoupdater_helpers/__init__.py +31 -0
firefox.py +4 -6
kicad.py +3 -6
linux-kbb/build-armlfs-6.1.sh +1 -1
linux-kbb/build-armlfs-6.6.sh +1 -1
linux-kbb/build-armlfs-mainline.sh +1 -1
linux-kbb/build-armlfs-stable.sh +1 -1
linux-kbb/kbb.py +6 -0
mpkg.py +9 -0
report-events.sh +33 -0
yt-dlp.py +2 -6

diff --git a/autoupdater_helpers/__init__.py b/autoupdater_helpers/__init__.py
index 7a78d92..7c943a3 100644
--- a/autoupdater_helpers/__init__.py
+++ b/autoupdater_helpers/__init__.py
@@ -26,4 +26,35 @@ def pkgbuild_new_ver(ver, sha256sum=None, rel="1"):
     os.rename("PKGBUILD.new", "PKGBUILD")
     return
 
+def ssl_ctx():
+    import ssl
+    cabundle = "/etc/pki/tls/certs/ca-bundle.crt"
+    ctx = ssl.create_default_context(cafile=cabundle)
+    return ctx
 
+def mpkg(pkg, ver=None):
+    statepath = "/tmp/armlfs-evt"
+    activepath = statepath + "/active"
+    donepath = statepath + "/done"
+    failpath = statepath + "/failed"
+    from os import makedirs, chdir, getcwd, rename
+    from os.path import join
+    from subprocess import run, CalledProcessError
+    desc = f"{pkg} {ver}" if ver else pkg
+    fn = desc.replace(" ","-")
+    makedirs(activepath, exist_ok=True)
+    fpath = join(activepath,fn)
+    with open(fpath,"w") as f:
+        f.write(desc + "\n")
+    prev_path = getcwd()
+    chdir("/sources/base-pkgbuilds")
+    try:
+        run(["./mpkg.sh", pkg], check=True)
+        chdir(prev_path)
+        makedirs(donepath, exist_ok=True)
+        rename(fpath, join(donepath,fn))
+    except CalledProcessError:
+        chdir(prev_path)
+        makedirs(failpath, exist_ok=True)
+        rename(fpath, join(failpath,fn))
+        raise
diff --git a/firefox.py b/firefox.py
index d76c033..486cd46 100755
--- a/firefox.py
+++ b/firefox.py
@@ -2,7 +2,6 @@
 
 from urllib.request import urlopen
 import json
-import ssl
 import os
 import sys
 from subprocess import run,DEVNULL,STDOUT
@@ -10,10 +9,8 @@ from autoupdater_helpers import *
 
 os.chdir("/sources/base-pkgbuilds/firefox")
 
-cabundle = "/etc/pki/tls/certs/ca-bundle.crt"
-ctx = ssl.create_default_context(cafile=cabundle)
 url = "https://product-details.mozilla.org/1.0/firefox_versions.json"
-
+ctx = ssl_ctx()
 with urlopen(url, context=ctx) as f:
     versions = json.load(f)
 
@@ -43,5 +40,6 @@ os.chdir("..")
 
 
 print(f"Updating {prev_ver} to {ver} (sum:{sha256sum})")
-run(["./mpkg.sh", "firefox"], check=True)
-run(["./mpkg.sh", "firefox-i18n"], check=True)
+mpkg("firefox", ver)
+mpkg("firefox-i18n", ver)
+
diff --git a/kicad.py b/kicad.py
index 8a10686..1c5b7f3 100755
--- a/kicad.py
+++ b/kicad.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 from urllib.request import urlopen, Request
-import ssl
 import os
 import sys
 import xml.etree.ElementTree as ET
@@ -10,13 +9,11 @@ from autoupdater_helpers import *
 
 os.chdir("/sources/base-pkgbuilds/kicad")
 
-cabundle = "/etc/pki/tls/certs/ca-bundle.crt"
-ctx = ssl.create_default_context(cafile=cabundle)
 url = "https://www.kicad.org/blog/index.xml"
 user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0"
 req = Request(url, headers={'User-Agent': user_agent})
 
-with urlopen(req, context=ctx) as f:
+with urlopen(req, context=ssl_ctx()) as f:
     blog_rss = ET.parse(f)
 
 for title in blog_rss.getroot().findall('./channel/item/title'): 
@@ -41,5 +38,5 @@ pkgbuild_new_ver(ver)
 os.chdir("..")
 
 print(f"Updating {prev_ver} to {ver}")
-run(["./mpkg.sh", "kicad"], check=True)
-run(["./mpkg.sh", "kicad-library"], check=True)
+mpkg("kicad", ver)
+mpkg("kicad-library", ver)
diff --git a/linux-kbb/build-armlfs-6.1.sh b/linux-kbb/build-armlfs-6.1.sh
index 6d8ccb6..a1184dc 100755
--- a/linux-kbb/build-armlfs-6.1.sh
+++ b/linux-kbb/build-armlfs-6.1.sh
@@ -5,4 +5,4 @@ set -x
 if [ -n "$1" ]; then
 	./patchset-pkgbuild.py "$1" /sources/base-pkgbuilds/linux-armlfs-lts61
 fi
-(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs-lts61)
+../mpkg.py linux-armlfs-lts61 $1
diff --git a/linux-kbb/build-armlfs-6.6.sh b/linux-kbb/build-armlfs-6.6.sh
index e05a184..2938f36 100755
--- a/linux-kbb/build-armlfs-6.6.sh
+++ b/linux-kbb/build-armlfs-6.6.sh
@@ -5,4 +5,4 @@ set -x
 if [ -n "$1" ]; then
 	./patchset-pkgbuild.py "$1" /sources/base-pkgbuilds/linux-armlfs-lts
 fi
-(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs-lts)
+../mpkg.py linux-armlfs-lts $1
diff --git a/linux-kbb/build-armlfs-mainline.sh b/linux-kbb/build-armlfs-mainline.sh
index 076331c..52731a6 100755
--- a/linux-kbb/build-armlfs-mainline.sh
+++ b/linux-kbb/build-armlfs-mainline.sh
@@ -2,4 +2,4 @@
 # We just trust that the worktree is at the appropriate tag :P
 set -e
 set -x
-(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs-test)
+../mpkg.py linux-armlfs-test $1
diff --git a/linux-kbb/build-armlfs-stable.sh b/linux-kbb/build-armlfs-stable.sh
index 6e9718f..cf29422 100755
--- a/linux-kbb/build-armlfs-stable.sh
+++ b/linux-kbb/build-armlfs-stable.sh
@@ -5,4 +5,4 @@ set -x
 if [ -n "$1" ]; then
 	./patchset-pkgbuild.py "$1" /sources/base-pkgbuilds/linux-armlfs
 fi
-(cd /sources/base-pkgbuilds && ./mpkg.sh linux-armlfs)
+../mpkg.py linux-armlfs $1
diff --git a/linux-kbb/kbb.py b/linux-kbb/kbb.py
index 18a1692..4a61814 100755
--- a/linux-kbb/kbb.py
+++ b/linux-kbb/kbb.py
@@ -449,6 +449,12 @@ def update_and_build():
 
 if len(sys.argv) == 1:
     update_and_build()
+elif len(sys.argv) == 3 and sys.argv[1] == "--mkworktree":
+    for k in kernels:
+        if str(k) == sys.argv[2]:
+            print(f"Making worktree for {k}: {k.dir}")
+            k.cd_into()
+            print("Remember to cleanup the worktree when you're done with it (umount)")
 elif len(sys.argv) == 3 and sys.argv[1] == "--rebuild":
     successlist = []
     for k in kernels:
diff --git a/mpkg.py b/mpkg.py
new file mode 100755
index 0000000..73b664f
--- /dev/null
+++ b/mpkg.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+from autoupdater_helpers import mpkg
+import sys
+if len(sys.argv) == 3:
+    mpkg(sys.argv[1], sys.argv[2])
+elif len(sys.argv) == 2:
+    mpkg(sys.argv[1])
+else:
+    print(f"usage: {sys.argv[0]} <package> [version]")
diff --git a/report-events.sh b/report-events.sh
new file mode 100755
index 0000000..1ca9302
--- /dev/null
+++ b/report-events.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+if [ -d /tmp/armlfs-evt ]; then
+	(
+	cd /tmp/armlfs-evt
+	if [ -d ./done ] && [ $(ls -1 ./done | wc -l) -gt 0 ]; then
+		echo "Autobuild report -- DONE:"
+		cd "done"
+		for p in $(ls -1trc); do
+			cat $p
+			rm $p
+		done
+		cd ..
+	fi
+	if [ -d ./failed ] && [ $(ls -1 failed | wc -l) -gt 0 ]; then
+		echo "Autobuild report -- FAILED:"
+		cd "failed"
+		for p in $(ls -1trc); do
+			cat $p
+			rm $p
+		done
+		cd ..
+	fi
+
+	if [ -d ./active ] && [ $(ls -1 active | wc -l) -gt 0 ]; then
+		echo "The system is building - ACTIVE builds:"
+		cd "active"
+		for p in $(ls -1trc); do
+			cat $p
+		done
+		cd ..
+	fi
+	)
+fi
diff --git a/yt-dlp.py b/yt-dlp.py
index c331ba4..db8bac5 100755
--- a/yt-dlp.py
+++ b/yt-dlp.py
@@ -2,7 +2,6 @@
 
 from urllib.request import urlopen, Request
 import json
-import ssl
 import os
 import sys
 from subprocess import run,DEVNULL,STDOUT
@@ -10,9 +9,6 @@ from autoupdater_helpers import *
 
 os.chdir("/sources/base-pkgbuilds/yt-dlp")
 
-cabundle = "/etc/pki/tls/certs/ca-bundle.crt"
-ctx = ssl.create_default_context(cafile=cabundle)
-
 url = "https://api.github.com/repos/yt-dlp/yt-dlp/releases"
 
 headers = {
@@ -22,7 +18,7 @@ headers = {
 
 req = Request(url, headers=headers)
 
-with urlopen(req, context=ctx) as f:
+with urlopen(req, context=ssl_ctx()) as f:
     versions = json.load(f)
 
 for vdata in versions:
@@ -43,4 +39,4 @@ pkgbuild_new_ver(ver)
 os.chdir("..")
 
 print(f"Updating {prev_ver} to {ver}")
-run(["./mpkg.sh", "yt-dlp"], check=True)
+mpkg("yt-dlp", ver)