add multithreading support

This commit is contained in:
Ward from fusion-voyager-3 2024-08-04 20:15:06 +03:00
parent dbbdc56e17
commit 3d8600e863
4 changed files with 556 additions and 512 deletions

View File

@ -15,6 +15,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Get script depends
run: apt-get install -y python3-numpy
- name: Generate whitelist json files - name: Generate whitelist json files
run: ./gen-i386-whitelist.py run: ./gen-i386-whitelist.py

View File

@ -4,6 +4,8 @@ import os, errno
import json import json
import subprocess import subprocess
import apt import apt
import numpy as np
import threading
def silentremove(filename): def silentremove(filename):
try: try:
@ -12,24 +14,43 @@ def silentremove(filename):
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
raise # re-raise exception if a different error occurred raise # re-raise exception if a different error occurred
current_path = os.path.dirname(os.path.realpath(__file__)) def pharse_build_tree(pkg_arr, current_path, pkgname_lines):
for pkgname in pkg_arr:
pkgname_lines = []
srcname_lines = []
srcnames_clean = []
file = open(current_path + "/i386_whitelist_bins", "r")
for line in file.readlines():
pkgname = line.strip()
if pkgname != "" and not pkgname.endswith("-udeb"):
pkgname_lines.append(pkgname)
print("Parsing dep tree for: " + pkgname) print("Parsing dep tree for: " + pkgname)
result = subprocess.run([current_path + '/get_depend_tree.sh', pkgname], stdout=subprocess.PIPE) result = subprocess.run([current_path + '/get_depend_tree.sh', pkgname], stdout=subprocess.PIPE)
stdout = result.stdout.decode('utf-8') stdout = result.stdout.decode('utf-8')
for line in stdout.splitlines(): for line in stdout.splitlines():
if line != "": if line != "":
pkgname_lines.append(line) pkgname_lines.append(line)
current_path = os.path.dirname(os.path.realpath(__file__))
whitelist_arr = np.array([])
thread_arr = []
pkgname_lines = []
srcname_lines = []
srcnames_clean = []
file = open(current_path + "/i386_whitelist_bins", "r")
for line in file.readlines():
pkgname = line.strip()
if pkgname != "" and not pkgname.endswith("-udeb"):
pkgname_lines.append(pkgname)
np.append(whitelist_arr, pkgname)
file.close() file.close()
newarr = np.array_split(whitelist_arr, 8)
for array in newarr:
t0 = threading.Thread(target=pharse_build_tree, args=(array, current_path, pkgname_lines,))
thread_arr.append(t0)
for thread_proc in thread_arr:
thread_proc.start()
for thread_proc in thread_arr:
thread_proc.join()
c = apt.Cache() c = apt.Cache()
for pkgname in pkgname_lines: for pkgname in pkgname_lines:
@ -41,7 +62,7 @@ for pkgname in pkgname_lines:
pass pass
for i in srcname_lines: for i in srcname_lines:
if i not in srcnames_clean: if i not in srcnames_clean and i + "-dmo" not in srcnames_clean:
srcnames_clean.append(i) srcnames_clean.append(i)
src_data = { src_data = {
@ -54,4 +75,3 @@ with open("i386_src_whitelist.json", "w") as twitterDataFile:
twitterDataFile.write( twitterDataFile.write(
json.dumps(src_data, indent=4) json.dumps(src_data, indent=4)
) )

9
get_source_name.py Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/python3
import sys
import apt
pkgname = sys.argv[1]
c = apt.Cache()
print(c[pkgname].candidate.source_name)

88
i386_src_whitelist.json Executable file → Normal file
View File

@ -6,12 +6,12 @@
"adduser", "adduser",
"adwaita-icon-theme", "adwaita-icon-theme",
"alsa-lib", "alsa-lib",
"alsa-plugins", "alsa-plugins-dmo",
"aom", "aom-dmo",
"apparmor", "apparmor",
"apt", "apt",
"argon2", "argon2",
"aribb24", "aribb24-dmo",
"aspell", "aspell",
"at-spi2-core", "at-spi2-core",
"attica-kf5", "attica-kf5",
@ -29,7 +29,7 @@
"cdebconf", "cdebconf",
"cdparanoia", "cdparanoia",
"chardet", "chardet",
"chromaprint", "chromaprint-dmo",
"cjson", "cjson",
"codec2", "codec2",
"colord", "colord",
@ -38,7 +38,7 @@
"cups", "cups",
"curl", "curl",
"cyrus-sasl2", "cyrus-sasl2",
"dav1d", "dav1d-dmo",
"db5.3", "db5.3",
"dbus", "dbus",
"dbus-broker", "dbus-broker",
@ -56,8 +56,8 @@
"enchant-2", "enchant-2",
"expat", "expat",
"faad2", "faad2",
"fdk-aac-free", "ffmpeg-dmo",
"ffmpeg", "ffmpeg-6-dmo",
"fftw3", "fftw3",
"file", "file",
"flac", "flac",
@ -74,9 +74,12 @@
"fribidi", "fribidi",
"fuse3", "fuse3",
"game-music-emu", "game-music-emu",
"gamin",
"gcc-14", "gcc-14",
"gcc-14-cross-ports",
"gcc-14-cross",
"gcc-14-cross-mipsen",
"gcc-defaults", "gcc-defaults",
"gcc-defaults-ports",
"gdbm", "gdbm",
"gdk-pixbuf", "gdk-pixbuf",
"giflib", "giflib",
@ -94,18 +97,17 @@
"graphite2", "graphite2",
"gsettings-desktop-schemas", "gsettings-desktop-schemas",
"gssdp", "gssdp",
"gst-libav1.0", "gst-libav1.0-dmo",
"gst-plugins-bad1.0", "gst-plugins-bad1.0-dmo",
"gst-plugins-base1.0", "gst-plugins-base1.0-dmo",
"gst-plugins-good1.0", "gst-plugins-good1.0-dmo",
"gst-plugins-ugly1.0", "gst-plugins-ugly1.0-dmo",
"gstreamer1.0", "gstreamer1.0",
"gtk+3.0", "gtk+3.0",
"gupnp", "gupnp",
"gupnp-igd", "gupnp-igd",
"harfbuzz", "harfbuzz",
"hicolor-icon-theme", "hicolor-icon-theme",
"humanity-icon-theme",
"hunspell", "hunspell",
"hyphen", "hyphen",
"icu", "icu",
@ -159,29 +161,29 @@
"kwidgetsaddons", "kwidgetsaddons",
"kwindowsystem", "kwindowsystem",
"kxmlgui", "kxmlgui",
"lame", "lame-dmo",
"lcms2", "lcms2",
"leptonlib", "leptonlib",
"lerc", "lerc",
"libarchive", "libarchive",
"libass", "libass-dmo",
"libassuan", "libassuan",
"libasyncns", "libasyncns",
"libavc1394", "libavc1394",
"libavtp", "libavtp",
"libbluray", "libbluray-dmo",
"libbs2b", "libbs2b",
"libbsd", "libbsd",
"libcaca", "libcaca",
"libcanberra", "libcanberra",
"libcap2", "libcap2",
"libcap-ng", "libcap-ng",
"libcdio", "libcdio-dmo",
"libcdio-paranoia", "libcdio-paranoia-dmo",
"libdatrie", "libdatrie",
"libdbusmenu-qt", "libdbusmenu-qt",
"libdc1394", "libdc1394",
"libdca", "libdca-dmo",
"libde265", "libde265",
"libdebian-installer", "libdebian-installer",
"libdecor-0", "libdecor-0",
@ -212,7 +214,6 @@
"libimobiledevice", "libimobiledevice",
"libinput", "libinput",
"libinstpatch", "libinstpatch",
"libjpeg8-empty",
"libjpeg-turbo", "libjpeg-turbo",
"libkate", "libkate",
"libksba", "libksba",
@ -231,7 +232,7 @@
"libpcap", "libpcap",
"libpciaccess", "libpciaccess",
"libpgm", "libpgm",
"libplacebo", "libplacebo-dmo",
"libplist", "libplist",
"libpng1.6", "libpng1.6",
"libproxy", "libproxy",
@ -239,7 +240,7 @@
"libpthread-stubs", "libpthread-stubs",
"librabbitmq", "librabbitmq",
"libraw1394", "libraw1394",
"librist", "librist-dmo",
"librsvg", "librsvg",
"libsamplerate", "libsamplerate",
"libsdl2", "libsdl2",
@ -265,7 +266,7 @@
"libtheora", "libtheora",
"libtirpc", "libtirpc",
"libtool", "libtool",
"libudfread", "libudfread-dmo",
"libunistring", "libunistring",
"libunwind", "libunwind",
"libusb-1.0", "libusb-1.0",
@ -273,6 +274,7 @@
"libutempter", "libutempter",
"libva", "libva",
"libvdpau", "libvdpau",
"vid.stab-dmo",
"libvidstab", "libvidstab",
"libvisual", "libvisual",
"libvorbis", "libvorbis",
@ -325,8 +327,7 @@
"media-types", "media-types",
"mesa", "mesa",
"mesa-demos", "mesa-demos",
"mime-support", "mjpegtools-dmo",
"mjpegtools",
"mpclib3", "mpclib3",
"mpeg2dec", "mpeg2dec",
"mpfr4", "mpfr4",
@ -347,9 +348,10 @@
"openal-soft", "openal-soft",
"opencore-amr", "opencore-amr",
"openexr", "openexr",
"openh264", "openh264-dmo",
"openjpeg2", "openjpeg2",
"openldap", "openldap",
"libldap2",
"openssl", "openssl",
"opus", "opus",
"orc", "orc",
@ -399,15 +401,14 @@
"raqm", "raqm",
"readline", "readline",
"requests", "requests",
"rtmpdump", "rtmpdump-dmo",
"rubberband", "rubberband-dmo",
"samba", "samba",
"sane-backends", "sane-backends",
"sbc", "sbc",
"scowl", "scowl",
"sed", "sed",
"serd", "serd",
"session-migration",
"setuptools", "setuptools",
"sgml-base", "sgml-base",
"shadow", "shadow",
@ -428,7 +429,7 @@
"sqlite3", "sqlite3",
"sratom", "sratom",
"srt", "srt",
"svt-av1", "svt-av1-dmo",
"syndication", "syndication",
"systemd", "systemd",
"sysvinit", "sysvinit",
@ -445,10 +446,9 @@
"tiff", "tiff",
"timgm6mb-soundfont", "timgm6mb-soundfont",
"tk8.6", "tk8.6",
"twolame", "twolame-dmo",
"tzdata", "tzdata",
"ubuntu-keyring", "ubuntu-keyring",
"ubuntu-themes",
"unixodbc", "unixodbc",
"unzip", "unzip",
"util-linux", "util-linux",
@ -466,8 +466,8 @@
"wildmidi", "wildmidi",
"woff2", "woff2",
"x11-xserver-utils", "x11-xserver-utils",
"x264", "x264-dmo",
"x265", "x265-dmo",
"xbitmaps", "xbitmaps",
"xcb-util", "xcb-util",
"xcb-util-image", "xcb-util-image",
@ -488,14 +488,26 @@
"xorg-sgml-doctools", "xorg-sgml-doctools",
"xterm", "xterm",
"xtrans", "xtrans",
"xvidcore", "xvidcore-dmo",
"xxhash", "xxhash",
"xz-utils", "xz-utils",
"zbar", "zbar",
"zenity", "zenity",
"zeromq3", "zeromq3",
"zimg", "zimg-dmo",
"zlib", "zlib",
"zvbi" "zvbi",
"make-dfsg",
"binutils",
"build-essential",
"rust-sequoia-sq",
"rust-sequoia-sop",
"libio-pty-perl",
"libhtml-parser-perl",
"liblwp-protocol-https-perl",
"liburi-perl",
"nodejs",
"libglu",
"rustc"
] ]
} }