From 3d8600e863402e7ce96706f8089c8d03f206f260 Mon Sep 17 00:00:00 2001 From: Ward from fusion-voyager-3 Date: Sun, 4 Aug 2024 20:15:06 +0300 Subject: [PATCH] add multithreading support --- .github/workflows/gen-i386-whitelist.yml | 3 + gen-i386-whitelist.py | 44 +- get_source_name.py | 9 + i386_src_whitelist.json | 1012 +++++++++++----------- 4 files changed, 556 insertions(+), 512 deletions(-) create mode 100755 get_source_name.py mode change 100755 => 100644 i386_src_whitelist.json diff --git a/.github/workflows/gen-i386-whitelist.yml b/.github/workflows/gen-i386-whitelist.yml index 5e8438c..f260fab 100644 --- a/.github/workflows/gen-i386-whitelist.yml +++ b/.github/workflows/gen-i386-whitelist.yml @@ -15,6 +15,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get script depends + run: apt-get install -y python3-numpy + - name: Generate whitelist json files run: ./gen-i386-whitelist.py diff --git a/gen-i386-whitelist.py b/gen-i386-whitelist.py index bd70894..b005fc0 100755 --- a/gen-i386-whitelist.py +++ b/gen-i386-whitelist.py @@ -4,6 +4,8 @@ import os, errno import json import subprocess import apt +import numpy as np +import threading def silentremove(filename): try: @@ -12,24 +14,43 @@ def silentremove(filename): if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory raise # re-raise exception if a different error occurred -current_path = os.path.dirname(os.path.realpath(__file__)) - -pkgname_lines = [] -srcname_lines = [] -srcnames_clean = [] -file = open(current_path + "/i386_whitelist_bins", "r") -for line in file.readlines(): - pkgname = line.strip() - if pkgname != "" and not pkgname.endswith("-udeb"): - pkgname_lines.append(pkgname) +def pharse_build_tree(pkg_arr, current_path, pkgname_lines): + for pkgname in pkg_arr: print("Parsing dep tree for: " + pkgname) result = subprocess.run([current_path + '/get_depend_tree.sh', pkgname], stdout=subprocess.PIPE) stdout = result.stdout.decode('utf-8') for line in stdout.splitlines(): if line != "": pkgname_lines.append(line) + +current_path = os.path.dirname(os.path.realpath(__file__)) + +whitelist_arr = np.array([]) +thread_arr = [] +pkgname_lines = [] +srcname_lines = [] +srcnames_clean = [] +file = open(current_path + "/i386_whitelist_bins", "r") + +for line in file.readlines(): + pkgname = line.strip() + if pkgname != "" and not pkgname.endswith("-udeb"): + pkgname_lines.append(pkgname) + np.append(whitelist_arr, pkgname) file.close() +newarr = np.array_split(whitelist_arr, 8) + +for array in newarr: + t0 = threading.Thread(target=pharse_build_tree, args=(array, current_path, pkgname_lines,)) + thread_arr.append(t0) + +for thread_proc in thread_arr: + thread_proc.start() + +for thread_proc in thread_arr: + thread_proc.join() + c = apt.Cache() for pkgname in pkgname_lines: @@ -41,7 +62,7 @@ for pkgname in pkgname_lines: pass for i in srcname_lines: - if i not in srcnames_clean: + if i not in srcnames_clean and i + "-dmo" not in srcnames_clean: srcnames_clean.append(i) src_data = { @@ -54,4 +75,3 @@ with open("i386_src_whitelist.json", "w") as twitterDataFile: twitterDataFile.write( json.dumps(src_data, indent=4) ) - diff --git a/get_source_name.py b/get_source_name.py new file mode 100755 index 0000000..d2393e5 --- /dev/null +++ b/get_source_name.py @@ -0,0 +1,9 @@ +#!/usr/bin/python3 +import sys + +import apt + +pkgname = sys.argv[1] +c = apt.Cache() +print(c[pkgname].candidate.source_name) + diff --git a/i386_src_whitelist.json b/i386_src_whitelist.json old mode 100755 new mode 100644 index ebaae8d..1a1ca86 --- a/i386_src_whitelist.json +++ b/i386_src_whitelist.json @@ -1,501 +1,513 @@ { - "i386_whitelist": [ - "a52dec", - "aalib", - "acl", - "adduser", - "adwaita-icon-theme", - "alsa-lib", - "alsa-plugins", - "aom", - "apparmor", - "apt", - "argon2", - "aribb24", - "aspell", - "at-spi2-core", - "attica-kf5", - "attr", - "audit", - "avahi", - "base-passwd", - "blt", - "brotli", - "bubblewrap", - "bzip2", - "cabextract", - "ca-certificates", - "cairo", - "cdebconf", - "cdparanoia", - "chardet", - "chromaprint", - "cjson", - "codec2", - "colord", - "coreutils", - "cryptsetup", - "cups", - "curl", - "cyrus-sasl2", - "dav1d", - "db5.3", - "dbus", - "dbus-broker", - "dbus-python", - "dconf", - "debconf", - "dictionaries-common", - "distro-info-data", - "double-conversion", - "dpkg", - "duktape", - "e2fsprogs", - "elfutils", - "emacsen-common", - "enchant-2", - "expat", - "faad2", - "fdk-aac-free", - "ffmpeg", - "fftw3", - "file", - "flac", - "flite", - "fluid-soundfont", - "fluidsynth", - "fontconfig", - "fonts-dejavu", - "fonts-freefont", - "fonts-liberation", - "fonts-noto", - "fonts-urw-base35", - "freetype", - "fribidi", - "fuse3", - "game-music-emu", - "gamin", - "gcc-14", - "gcc-defaults", - "gdbm", - "gdk-pixbuf", - "giflib", - "glib2.0", - "glibc", - "glib-networking", - "gmp", - "gnome-terminal", - "gnupg1", - "gnupg2", - "gnutls28", - "gobject-introspection", - "gpgme1.0", - "gpm", - "graphite2", - "gsettings-desktop-schemas", - "gssdp", - "gst-libav1.0", - "gst-plugins-bad1.0", - "gst-plugins-base1.0", - "gst-plugins-good1.0", - "gst-plugins-ugly1.0", - "gstreamer1.0", - "gtk+3.0", - "gupnp", - "gupnp-igd", - "harfbuzz", - "hicolor-icon-theme", - "humanity-icon-theme", - "hunspell", - "hyphen", - "icu", - "imath", - "init-system-helpers", - "intel-gmmlib", - "intel-media-driver-non-free", - "intel-mediasdk", - "intel-vaapi-driver", - "iptables", - "isl", - "iso-codes", - "jackd2", - "jansson", - "jbigkit", - "json-c", - "json-glib", - "karchive", - "kauth", - "kbookmarks", - "kcodecs", - "kcompletion", - "kconfig", - "kconfigwidgets", - "kcoreaddons", - "kcrash", - "kdbusaddons", - "kded", - "kdoctools", - "keyutils", - "kglobalaccel", - "kguiaddons", - "ki18n", - "kiconthemes", - "kio", - "kitemviews", - "kjobwidgets", - "kmod", - "knewstuff", - "knotifications", - "knotifyconfig", - "konsole", - "kpackage", - "kparts", - "kpty", - "krb5", - "kservice", - "ktextwidgets", - "kwallet-kf5", - "kwayland", - "kwidgetsaddons", - "kwindowsystem", - "kxmlgui", - "lame", - "lcms2", - "leptonlib", - "lerc", - "libarchive", - "libass", - "libassuan", - "libasyncns", - "libavc1394", - "libavtp", - "libbluray", - "libbs2b", - "libbsd", - "libcaca", - "libcanberra", - "libcap2", - "libcap-ng", - "libcdio", - "libcdio-paranoia", - "libdatrie", - "libdbusmenu-qt", - "libdc1394", - "libdca", - "libde265", - "libdebian-installer", - "libdecor-0", - "libdeflate", - "libdrm", - "libdv", - "libdvdnav", - "libdvdread", - "libedit", - "libepoxy", - "libevdev", - "libexif", - "libffi", - "libfontenc", - "libgcrypt20", - "libgd2", - "libglvnd", - "libgpg-error", - "libgphoto2", - "libgsm", - "libgudev", - "libhandy-1", - "libice", - "libidn2", - "libiec61883", - "libieee1284", - "libimagequant", - "libimobiledevice", - "libinput", - "libinstpatch", - "libjpeg8-empty", - "libjpeg-turbo", - "libkate", - "libksba", - "libldac", - "libmanette", - "libmd", - "libmodplug", - "libmpc", - "libmspack", - "libmysofa", - "libnice", - "libnotify", - "libnsl", - "libogg", - "libopenmpt", - "libpcap", - "libpciaccess", - "libpgm", - "libplacebo", - "libplist", - "libpng1.6", - "libproxy", - "libpsl", - "libpthread-stubs", - "librabbitmq", - "libraw1394", - "librist", - "librsvg", - "libsamplerate", - "libsdl2", - "libseccomp", - "libsecret", - "libselinux", - "libsemanage", - "libsepol", - "libshout", - "libsidplay", - "libsm", - "libsndfile", - "libsodium", - "libsoup2.4", - "libsoup3", - "libsoxr", - "libsrtp2", - "libssh", - "libtasn1-6", - "libtext-iconv-perl", - "libtextwrap", - "libthai", - "libtheora", - "libtirpc", - "libtool", - "libudfread", - "libunistring", - "libunwind", - "libusb-1.0", - "libusbmuxd", - "libutempter", - "libva", - "libvdpau", - "libvidstab", - "libvisual", - "libvorbis", - "libvpx", - "libwacom", - "libwebp", - "libx11", - "libxau", - "libxaw", - "libxcb", - "libxcomposite", - "libxcrypt", - "libxcursor", - "libxdamage", - "libxdmcp", - "libxext", - "libxfixes", - "libxi", - "libxinerama", - "libxkbcommon", - "libxml2", - "libxmu", - "libxpm", - "libxrandr", - "libxrender", - "libxshmfence", - "libxslt", - "libxss", - "libxt", - "libxv", - "libxxf86vm", - "libyaml", - "libzstd", - "lilv", - "llvm-toolchain-15", - "llvm-toolchain-17", - "llvm-toolchain-18", - "lmdb", - "lm-sensors", - "lsb", - "lsb-release-minimal", - "lutris", - "lvm2", - "lxml", - "lz4", - "mailcap", - "mangohud", - "mbedtls", - "md4c", - "media-types", - "mesa", - "mesa-demos", - "mime-support", - "mjpegtools", - "mpclib3", - "mpeg2dec", - "mpfr4", - "mpg123", - "mtdev", - "ncurses", - "net-snmp", - "nettle", - "newt", - "nghttp2", - "norm", - "npth", - "nspr", - "nss", - "numactl", - "nvidia-cuda-toolkit", - "ocl-icd", - "openal-soft", - "opencore-amr", - "openexr", - "openh264", - "openjpeg2", - "openldap", - "openssl", - "opus", - "orc", - "p11-kit", - "p7zip", - "pam", - "pango1.0", - "pci.ids", - "pciutils", - "pcre2", - "pcsc-lite", - "perl", - "pillow", - "pinentry", - "pipewire", - "pixman", - "policykit-1", - "polkit-qt-1", - "poppler", - "popt", - "procps", - "psmisc", - "pulseaudio", - "pycairo", - "pygobject", - "python3.11", - "python3-defaults", - "python3-stdlib-extensions", - "python-apt", - "python-certifi", - "python-distro", - "python-idna", - "python-magic", - "python-setproctitle", - "python-urllib3", - "pyyaml", - "qca2", - "qtbase-opensource-src", - "qtbase-opensource-src-gles", - "qtdeclarative-opensource-src", - "qtdeclarative-opensource-src-gles", - "qtmultimedia-opensource-src", - "qtspeech-opensource-src", - "qtsvg-opensource-src", - "qtwayland-opensource-src", - "qtx11extras-opensource-src", - "raqm", - "readline", - "requests", - "rtmpdump", - "rubberband", - "samba", - "sane-backends", - "sbc", - "scowl", - "sed", - "serd", - "session-migration", - "setuptools", - "sgml-base", - "shadow", - "shared-mime-info", - "shine", - "six", - "slang2", - "snappy", - "sndio", - "solid", - "sonnet", - "sord", - "sound-theme-freedesktop", - "soundtouch", - "spandsp", - "speex", - "speexdsp", - "sqlite3", - "sratom", - "srt", - "svt-av1", - "syndication", - "systemd", - "sysvinit", - "taglib", - "talloc", - "tar", - "tcl8.6", - "tcp-wrappers", - "tdb", - "tesseract", - "tevent", - "tex-gyre", - "texinfo", - "tiff", - "timgm6mb-soundfont", - "tk8.6", - "twolame", - "tzdata", - "ubuntu-keyring", - "ubuntu-themes", - "unixodbc", - "unzip", - "util-linux", - "v4l-utils", - "vkbasalt", - "vo-aacenc", - "vo-amrwbenc", - "vte2.91", - "vulkan-loader", - "vulkan-tools", - "wavpack", - "wayland", - "webkit2gtk", - "webrtc-audio-processing", - "wildmidi", - "woff2", - "x11-xserver-utils", - "x264", - "x265", - "xbitmaps", - "xcb-util", - "xcb-util-image", - "xcb-util-keysyms", - "xcb-util-renderutil", - "xcb-util-wm", - "xdg-dbus-proxy", - "xdg-desktop-portal", - "xdg-desktop-portal-gtk", - "xdg-desktop-portal-kde", - "xfonts-encodings", - "xfonts-utils", - "xft", - "xkeyboard-config", - "xml-core", - "xorg", - "xorgproto", - "xorg-sgml-doctools", - "xterm", - "xtrans", - "xvidcore", - "xxhash", - "xz-utils", - "zbar", - "zenity", - "zeromq3", - "zimg", - "zlib", - "zvbi" - ] -} + "i386_whitelist": [ + "a52dec", + "aalib", + "acl", + "adduser", + "adwaita-icon-theme", + "alsa-lib", + "alsa-plugins-dmo", + "aom-dmo", + "apparmor", + "apt", + "argon2", + "aribb24-dmo", + "aspell", + "at-spi2-core", + "attica-kf5", + "attr", + "audit", + "avahi", + "base-passwd", + "blt", + "brotli", + "bubblewrap", + "bzip2", + "cabextract", + "ca-certificates", + "cairo", + "cdebconf", + "cdparanoia", + "chardet", + "chromaprint-dmo", + "cjson", + "codec2", + "colord", + "coreutils", + "cryptsetup", + "cups", + "curl", + "cyrus-sasl2", + "dav1d-dmo", + "db5.3", + "dbus", + "dbus-broker", + "dbus-python", + "dconf", + "debconf", + "dictionaries-common", + "distro-info-data", + "double-conversion", + "dpkg", + "duktape", + "e2fsprogs", + "elfutils", + "emacsen-common", + "enchant-2", + "expat", + "faad2", + "ffmpeg-dmo", + "ffmpeg-6-dmo", + "fftw3", + "file", + "flac", + "flite", + "fluid-soundfont", + "fluidsynth", + "fontconfig", + "fonts-dejavu", + "fonts-freefont", + "fonts-liberation", + "fonts-noto", + "fonts-urw-base35", + "freetype", + "fribidi", + "fuse3", + "game-music-emu", + "gcc-14", + "gcc-14-cross-ports", + "gcc-14-cross", + "gcc-14-cross-mipsen", + "gcc-defaults", + "gcc-defaults-ports", + "gdbm", + "gdk-pixbuf", + "giflib", + "glib2.0", + "glibc", + "glib-networking", + "gmp", + "gnome-terminal", + "gnupg1", + "gnupg2", + "gnutls28", + "gobject-introspection", + "gpgme1.0", + "gpm", + "graphite2", + "gsettings-desktop-schemas", + "gssdp", + "gst-libav1.0-dmo", + "gst-plugins-bad1.0-dmo", + "gst-plugins-base1.0-dmo", + "gst-plugins-good1.0-dmo", + "gst-plugins-ugly1.0-dmo", + "gstreamer1.0", + "gtk+3.0", + "gupnp", + "gupnp-igd", + "harfbuzz", + "hicolor-icon-theme", + "hunspell", + "hyphen", + "icu", + "imath", + "init-system-helpers", + "intel-gmmlib", + "intel-media-driver-non-free", + "intel-mediasdk", + "intel-vaapi-driver", + "iptables", + "isl", + "iso-codes", + "jackd2", + "jansson", + "jbigkit", + "json-c", + "json-glib", + "karchive", + "kauth", + "kbookmarks", + "kcodecs", + "kcompletion", + "kconfig", + "kconfigwidgets", + "kcoreaddons", + "kcrash", + "kdbusaddons", + "kded", + "kdoctools", + "keyutils", + "kglobalaccel", + "kguiaddons", + "ki18n", + "kiconthemes", + "kio", + "kitemviews", + "kjobwidgets", + "kmod", + "knewstuff", + "knotifications", + "knotifyconfig", + "konsole", + "kpackage", + "kparts", + "kpty", + "krb5", + "kservice", + "ktextwidgets", + "kwallet-kf5", + "kwayland", + "kwidgetsaddons", + "kwindowsystem", + "kxmlgui", + "lame-dmo", + "lcms2", + "leptonlib", + "lerc", + "libarchive", + "libass-dmo", + "libassuan", + "libasyncns", + "libavc1394", + "libavtp", + "libbluray-dmo", + "libbs2b", + "libbsd", + "libcaca", + "libcanberra", + "libcap2", + "libcap-ng", + "libcdio-dmo", + "libcdio-paranoia-dmo", + "libdatrie", + "libdbusmenu-qt", + "libdc1394", + "libdca-dmo", + "libde265", + "libdebian-installer", + "libdecor-0", + "libdeflate", + "libdrm", + "libdv", + "libdvdnav", + "libdvdread", + "libedit", + "libepoxy", + "libevdev", + "libexif", + "libffi", + "libfontenc", + "libgcrypt20", + "libgd2", + "libglvnd", + "libgpg-error", + "libgphoto2", + "libgsm", + "libgudev", + "libhandy-1", + "libice", + "libidn2", + "libiec61883", + "libieee1284", + "libimagequant", + "libimobiledevice", + "libinput", + "libinstpatch", + "libjpeg-turbo", + "libkate", + "libksba", + "libldac", + "libmanette", + "libmd", + "libmodplug", + "libmpc", + "libmspack", + "libmysofa", + "libnice", + "libnotify", + "libnsl", + "libogg", + "libopenmpt", + "libpcap", + "libpciaccess", + "libpgm", + "libplacebo-dmo", + "libplist", + "libpng1.6", + "libproxy", + "libpsl", + "libpthread-stubs", + "librabbitmq", + "libraw1394", + "librist-dmo", + "librsvg", + "libsamplerate", + "libsdl2", + "libseccomp", + "libsecret", + "libselinux", + "libsemanage", + "libsepol", + "libshout", + "libsidplay", + "libsm", + "libsndfile", + "libsodium", + "libsoup2.4", + "libsoup3", + "libsoxr", + "libsrtp2", + "libssh", + "libtasn1-6", + "libtext-iconv-perl", + "libtextwrap", + "libthai", + "libtheora", + "libtirpc", + "libtool", + "libudfread-dmo", + "libunistring", + "libunwind", + "libusb-1.0", + "libusbmuxd", + "libutempter", + "libva", + "libvdpau", + "vid.stab-dmo", + "libvidstab", + "libvisual", + "libvorbis", + "libvpx", + "libwacom", + "libwebp", + "libx11", + "libxau", + "libxaw", + "libxcb", + "libxcomposite", + "libxcrypt", + "libxcursor", + "libxdamage", + "libxdmcp", + "libxext", + "libxfixes", + "libxi", + "libxinerama", + "libxkbcommon", + "libxml2", + "libxmu", + "libxpm", + "libxrandr", + "libxrender", + "libxshmfence", + "libxslt", + "libxss", + "libxt", + "libxv", + "libxxf86vm", + "libyaml", + "libzstd", + "lilv", + "llvm-toolchain-15", + "llvm-toolchain-17", + "llvm-toolchain-18", + "lmdb", + "lm-sensors", + "lsb", + "lsb-release-minimal", + "lutris", + "lvm2", + "lxml", + "lz4", + "mailcap", + "mangohud", + "mbedtls", + "md4c", + "media-types", + "mesa", + "mesa-demos", + "mjpegtools-dmo", + "mpclib3", + "mpeg2dec", + "mpfr4", + "mpg123", + "mtdev", + "ncurses", + "net-snmp", + "nettle", + "newt", + "nghttp2", + "norm", + "npth", + "nspr", + "nss", + "numactl", + "nvidia-cuda-toolkit", + "ocl-icd", + "openal-soft", + "opencore-amr", + "openexr", + "openh264-dmo", + "openjpeg2", + "openldap", + "libldap2", + "openssl", + "opus", + "orc", + "p11-kit", + "p7zip", + "pam", + "pango1.0", + "pci.ids", + "pciutils", + "pcre2", + "pcsc-lite", + "perl", + "pillow", + "pinentry", + "pipewire", + "pixman", + "policykit-1", + "polkit-qt-1", + "poppler", + "popt", + "procps", + "psmisc", + "pulseaudio", + "pycairo", + "pygobject", + "python3.11", + "python3-defaults", + "python3-stdlib-extensions", + "python-apt", + "python-certifi", + "python-distro", + "python-idna", + "python-magic", + "python-setproctitle", + "python-urllib3", + "pyyaml", + "qca2", + "qtbase-opensource-src", + "qtbase-opensource-src-gles", + "qtdeclarative-opensource-src", + "qtdeclarative-opensource-src-gles", + "qtmultimedia-opensource-src", + "qtspeech-opensource-src", + "qtsvg-opensource-src", + "qtwayland-opensource-src", + "qtx11extras-opensource-src", + "raqm", + "readline", + "requests", + "rtmpdump-dmo", + "rubberband-dmo", + "samba", + "sane-backends", + "sbc", + "scowl", + "sed", + "serd", + "setuptools", + "sgml-base", + "shadow", + "shared-mime-info", + "shine", + "six", + "slang2", + "snappy", + "sndio", + "solid", + "sonnet", + "sord", + "sound-theme-freedesktop", + "soundtouch", + "spandsp", + "speex", + "speexdsp", + "sqlite3", + "sratom", + "srt", + "svt-av1-dmo", + "syndication", + "systemd", + "sysvinit", + "taglib", + "talloc", + "tar", + "tcl8.6", + "tcp-wrappers", + "tdb", + "tesseract", + "tevent", + "tex-gyre", + "texinfo", + "tiff", + "timgm6mb-soundfont", + "tk8.6", + "twolame-dmo", + "tzdata", + "ubuntu-keyring", + "unixodbc", + "unzip", + "util-linux", + "v4l-utils", + "vkbasalt", + "vo-aacenc", + "vo-amrwbenc", + "vte2.91", + "vulkan-loader", + "vulkan-tools", + "wavpack", + "wayland", + "webkit2gtk", + "webrtc-audio-processing", + "wildmidi", + "woff2", + "x11-xserver-utils", + "x264-dmo", + "x265-dmo", + "xbitmaps", + "xcb-util", + "xcb-util-image", + "xcb-util-keysyms", + "xcb-util-renderutil", + "xcb-util-wm", + "xdg-dbus-proxy", + "xdg-desktop-portal", + "xdg-desktop-portal-gtk", + "xdg-desktop-portal-kde", + "xfonts-encodings", + "xfonts-utils", + "xft", + "xkeyboard-config", + "xml-core", + "xorg", + "xorgproto", + "xorg-sgml-doctools", + "xterm", + "xtrans", + "xvidcore-dmo", + "xxhash", + "xz-utils", + "zbar", + "zenity", + "zeromq3", + "zimg-dmo", + "zlib", + "zvbi", + "make-dfsg", + "binutils", + "build-essential", + "rust-sequoia-sq", + "rust-sequoia-sop", + "libio-pty-perl", + "libhtml-parser-perl", + "liblwp-protocol-https-perl", + "liburi-perl", + "nodejs", + "libglu", + "rustc" + ] +} \ No newline at end of file