add multithreading support

.github/workflows/gen-i386-whitelist.yml

@@ -15,6 +15,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: Get script depends
+      run: apt-get install -y python3-numpy
+
     - name: Generate whitelist json files
       run: ./gen-i386-whitelist.py
 

gen-i386-whitelist.py

@@ -4,6 +4,8 @@ import os, errno
 import json
 import subprocess
 import apt
+import numpy as np
+import threading
 
 def silentremove(filename):
     try:
@@ -12,24 +14,43 @@ def silentremove(filename):
         if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
             raise # re-raise exception if a different error occurred    
 
-current_path = os.path.dirname(os.path.realpath(__file__))
+def pharse_build_tree(pkg_arr, current_path, pkgname_lines):
-
+    for pkgname in pkg_arr:
-pkgname_lines = []
-srcname_lines = []
-srcnames_clean = []
-file = open(current_path + "/i386_whitelist_bins", "r")
-for line in file.readlines():
-    pkgname = line.strip()
-    if pkgname != "" and not pkgname.endswith("-udeb"):
-        pkgname_lines.append(pkgname)
         print("Parsing dep tree for: " + pkgname)
         result = subprocess.run([current_path + '/get_depend_tree.sh', pkgname], stdout=subprocess.PIPE)
         stdout = result.stdout.decode('utf-8')
         for line in stdout.splitlines():
                 if line != "":
                     pkgname_lines.append(line)
+
+current_path = os.path.dirname(os.path.realpath(__file__))
+
+whitelist_arr = np.array([])
+thread_arr = []
+pkgname_lines = []
+srcname_lines = []
+srcnames_clean = []
+file = open(current_path + "/i386_whitelist_bins", "r")
+
+for line in file.readlines():
+    pkgname = line.strip()
+    if pkgname != "" and not pkgname.endswith("-udeb"):
+        pkgname_lines.append(pkgname)
+        np.append(whitelist_arr, pkgname)
 file.close()
 
+newarr = np.array_split(whitelist_arr, 8)
+
+for array in newarr:
+    t0 = threading.Thread(target=pharse_build_tree, args=(array, current_path, pkgname_lines,))
+    thread_arr.append(t0)
+    
+for thread_proc in thread_arr:
+    thread_proc.start()
+
+for thread_proc in thread_arr:
+    thread_proc.join()
+
 c = apt.Cache()
 
 for pkgname in pkgname_lines:
@@ -41,7 +62,7 @@ for pkgname in pkgname_lines:
         pass
 
 for i in srcname_lines:
-  if i not in srcnames_clean:
+  if i not in srcnames_clean and i + "-dmo" not in srcnames_clean:
     srcnames_clean.append(i)
 
 src_data = {
@@ -54,4 +75,3 @@ with open("i386_src_whitelist.json", "w") as twitterDataFile:
     twitterDataFile.write(
         json.dumps(src_data, indent=4)
     )
-    

get_source_name.py

@@ -0,0 +1,9 @@
+#!/usr/bin/python3
+import sys
+
+import apt
+
+pkgname = sys.argv[1]
+c = apt.Cache()
+print(c[pkgname].candidate.source_name)
+

i386_src_whitelist.json

@@ -6,12 +6,12 @@
         "adduser",
         "adwaita-icon-theme",
         "alsa-lib",
-		"alsa-plugins",
+        "alsa-plugins-dmo",
-		"aom",
+        "aom-dmo",
         "apparmor",
         "apt",
         "argon2",
-		"aribb24",
+        "aribb24-dmo",
         "aspell",
         "at-spi2-core",
         "attica-kf5",
@@ -29,7 +29,7 @@
         "cdebconf",
         "cdparanoia",
         "chardet",
-		"chromaprint",
+        "chromaprint-dmo",
         "cjson",
         "codec2",
         "colord",
@@ -38,7 +38,7 @@
         "cups",
         "curl",
         "cyrus-sasl2",
-		"dav1d",
+        "dav1d-dmo",
         "db5.3",
         "dbus",
         "dbus-broker",
@@ -56,8 +56,8 @@
         "enchant-2",
         "expat",
         "faad2",
-		"fdk-aac-free",
+        "ffmpeg-dmo",
-		"ffmpeg",
+        "ffmpeg-6-dmo",
         "fftw3",
         "file",
         "flac",
@@ -74,9 +74,12 @@
         "fribidi",
         "fuse3",
         "game-music-emu",
-		"gamin",
         "gcc-14",
+        "gcc-14-cross-ports",
+        "gcc-14-cross",
+        "gcc-14-cross-mipsen",
         "gcc-defaults",
+        "gcc-defaults-ports",
         "gdbm",
         "gdk-pixbuf",
         "giflib",
@@ -94,18 +97,17 @@
         "graphite2",
         "gsettings-desktop-schemas",
         "gssdp",
-		"gst-libav1.0",
+        "gst-libav1.0-dmo",
-		"gst-plugins-bad1.0",
+        "gst-plugins-bad1.0-dmo",
-		"gst-plugins-base1.0",
+        "gst-plugins-base1.0-dmo",
-		"gst-plugins-good1.0",
+        "gst-plugins-good1.0-dmo",
-		"gst-plugins-ugly1.0",
+        "gst-plugins-ugly1.0-dmo",
         "gstreamer1.0",
         "gtk+3.0",
         "gupnp",
         "gupnp-igd",
         "harfbuzz",
         "hicolor-icon-theme",
-		"humanity-icon-theme",
         "hunspell",
         "hyphen",
         "icu",
@@ -159,29 +161,29 @@
         "kwidgetsaddons",
         "kwindowsystem",
         "kxmlgui",
-		"lame",
+        "lame-dmo",
         "lcms2",
         "leptonlib",
         "lerc",
         "libarchive",
-		"libass",
+        "libass-dmo",
         "libassuan",
         "libasyncns",
         "libavc1394",
         "libavtp",
-		"libbluray",
+        "libbluray-dmo",
         "libbs2b",
         "libbsd",
         "libcaca",
         "libcanberra",
         "libcap2",
         "libcap-ng",
-		"libcdio",
+        "libcdio-dmo",
-		"libcdio-paranoia",
+        "libcdio-paranoia-dmo",
         "libdatrie",
         "libdbusmenu-qt",
         "libdc1394",
-		"libdca",
+        "libdca-dmo",
         "libde265",
         "libdebian-installer",
         "libdecor-0",
@@ -212,7 +214,6 @@
         "libimobiledevice",
         "libinput",
         "libinstpatch",
-		"libjpeg8-empty",
         "libjpeg-turbo",
         "libkate",
         "libksba",
@@ -231,7 +232,7 @@
         "libpcap",
         "libpciaccess",
         "libpgm",
-		"libplacebo",
+        "libplacebo-dmo",
         "libplist",
         "libpng1.6",
         "libproxy",
@@ -239,7 +240,7 @@
         "libpthread-stubs",
         "librabbitmq",
         "libraw1394",
-		"librist",
+        "librist-dmo",
         "librsvg",
         "libsamplerate",
         "libsdl2",
@@ -265,7 +266,7 @@
         "libtheora",
         "libtirpc",
         "libtool",
-		"libudfread",
+        "libudfread-dmo",
         "libunistring",
         "libunwind",
         "libusb-1.0",
@@ -273,6 +274,7 @@
         "libutempter",
         "libva",
         "libvdpau",
+        "vid.stab-dmo",
         "libvidstab",
         "libvisual",
         "libvorbis",
@@ -325,8 +327,7 @@
         "media-types",
         "mesa",
         "mesa-demos",
-		"mime-support",
+        "mjpegtools-dmo",
-		"mjpegtools",
         "mpclib3",
         "mpeg2dec",
         "mpfr4",
@@ -347,9 +348,10 @@
         "openal-soft",
         "opencore-amr",
         "openexr",
-		"openh264",
+        "openh264-dmo",
         "openjpeg2",
         "openldap",
+        "libldap2",
         "openssl",
         "opus",
         "orc",
@@ -399,15 +401,14 @@
         "raqm",
         "readline",
         "requests",
-		"rtmpdump",
+        "rtmpdump-dmo",
-		"rubberband",
+        "rubberband-dmo",
         "samba",
         "sane-backends",
         "sbc",
         "scowl",
         "sed",
         "serd",
-		"session-migration",
         "setuptools",
         "sgml-base",
         "shadow",
@@ -428,7 +429,7 @@
         "sqlite3",
         "sratom",
         "srt",
-		"svt-av1",
+        "svt-av1-dmo",
         "syndication",
         "systemd",
         "sysvinit",
@@ -445,10 +446,9 @@
         "tiff",
         "timgm6mb-soundfont",
         "tk8.6",
-		"twolame",
+        "twolame-dmo",
         "tzdata",
         "ubuntu-keyring",
-		"ubuntu-themes",
         "unixodbc",
         "unzip",
         "util-linux",
@@ -466,8 +466,8 @@
         "wildmidi",
         "woff2",
         "x11-xserver-utils",
-		"x264",
+        "x264-dmo",
-		"x265",
+        "x265-dmo",
         "xbitmaps",
         "xcb-util",
         "xcb-util-image",
@@ -488,14 +488,26 @@
         "xorg-sgml-doctools",
         "xterm",
         "xtrans",
-		"xvidcore",
+        "xvidcore-dmo",
         "xxhash",
         "xz-utils",
         "zbar",
         "zenity",
         "zeromq3",
-		"zimg",
+        "zimg-dmo",
         "zlib",
-		"zvbi"
+        "zvbi",
+        "make-dfsg",
+        "binutils",
+        "build-essential",
+        "rust-sequoia-sq",
+        "rust-sequoia-sop",
+        "libio-pty-perl",
+        "libhtml-parser-perl",
+        "liblwp-protocol-https-perl",
+        "liburi-perl",
+        "nodejs",
+        "libglu",
+        "rustc"
     ]
 }