diff --git a/ppp b/ppp index 6879cab..dc38fca 100755 Binary files a/ppp and b/ppp differ diff --git a/src/main.go b/src/main.go index c810b99..b6ecf3a 100644 --- a/src/main.go +++ b/src/main.go @@ -2,26 +2,40 @@ package main import ( "bufio" + "fmt" "io" "log" "net/http" "os" "strings" + "sync" "github.com/ulikunitz/xz" ) func main() { - base := os.Args[1] - target := os.Args[2] + config := config{ + Source: os.Args[1], + Target: os.Args[2], + Download: false, + } - basePackages := processFile(base) - targetPackages := processFile(target) + if len(os.Args) > 3 { + config.Download = true + config.DlUrl = os.Args[3] + config.Output = os.Args[4] + } - compare(basePackages, targetPackages) + basePackages := processFile(config.Source) + targetPackages := processFile(config.Target) + + changed := compare(basePackages, targetPackages, config.Download) + if config.Download { + download(changed, config.DlUrl, config.Output) + } } -func processFile(url string) map[string]string { +func processFile(url string) map[string]packageInfo { resp, err := http.Get(url) if err != nil { @@ -37,7 +51,7 @@ func processFile(url string) map[string]string { rdr = r } - packages := make(map[string]string) + packages := make(map[string]packageInfo) var currentPackage string scanner := bufio.NewScanner(rdr) for scanner.Scan() { @@ -53,30 +67,120 @@ func processFile(url string) map[string]string { _, broken := brokenPackages[pkName] if !broken { currentPackage = pkName + packages[currentPackage] = packageInfo{ + Name: pkName, + } } else { currentPackage = "" } } } else { - if strings.HasPrefix(line, "Version: ") && currentPackage != "" { - packages[currentPackage] = strings.TrimPrefix(line, "Version: ") + if strings.HasPrefix(line, "Version: ") { + packages[currentPackage] = packageInfo{ + Name: currentPackage, + Version: strings.TrimPrefix(line, "Version: "), + } + } + if strings.HasPrefix(line, "Filename: ") { + packages[currentPackage] = packageInfo{ + Name: currentPackage, + Version: packages[currentPackage].Version, + FilePath: strings.TrimPrefix(line, "Filename: "), + } } } } return packages - } -func compare(basePackages map[string]string, targetPackages map[string]string) { - for pack, version := range targetPackages { +func compare(basePackages map[string]packageInfo, targetPackages map[string]packageInfo, download bool) map[string]packageInfo { + output := make(map[string]packageInfo) + for pack, info := range targetPackages { if baseVersion, ok := basePackages[pack]; ok { - if baseVersion != version { - os.Stdout.WriteString(pack) + if baseVersion.Version != info.Version { + output[pack] = info + if !download { + os.Stdout.WriteString(pack) + } } } else { - os.Stdout.WriteString(pack) + output[pack] = info + if !download { + os.Stdout.WriteString(pack) + } } } + return output +} + +func download(packages map[string]packageInfo, url string, output string) { + // Create a buffered channel to store the packages to be downloaded + packageQueue := make(chan packageInfo, 10) + + // Create a worker pool with 10 workers + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case pack, ok := <-packageQueue: + if !ok { + return + } + fmt.Printf("Downloading %s \n", pack.Name) + resp, err := http.Get(url + pack.FilePath) + if err != nil { + fmt.Printf("Failed to download %s: %v \n", pack.Name, err) + continue + } + defer resp.Body.Close() + rdr := io.Reader(resp.Body) + path := output + strings.Split(pack.FilePath, "/")[len(strings.Split(pack.FilePath, "/"))-1] + file, err := os.Create(path) + if err != nil { + fmt.Printf("Failed to create file %s: %v \n", path, err) + continue + } + defer file.Close() + _, err = io.Copy(file, rdr) + if err != nil { + fmt.Printf("Failed to save file %s: %v \n", path, err) + continue + } + default: + // No more packages to download, exit the goroutine + return + } + } + }() + } + + // Add the packages to the queue + for _, pack := range packages { + packageQueue <- pack + } + + // Close the queue to signal the workers to stop + close(packageQueue) + + // Wait for all the workers to finish + wg.Wait() +} + +type config struct { + Source string + Target string + Download bool + DlUrl string + Output string +} + +type packageInfo struct { + Name string + Version string + FilePath string } var brokenPackages = map[string]bool{ diff --git a/ubuntu.sh b/ubuntu.sh index 65a08b4..13186a6 100644 --- a/ubuntu.sh +++ b/ubuntu.sh @@ -6,7 +6,7 @@ chmod 755 ./ppp # output folders mkdir -p ./output/output -cd ./output/output +cd ./output # temp apt update @@ -17,98 +17,27 @@ apt install dpkg-sig wget rsync -y # Get ubuntu main pool -PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz) -if [ ! -z "$PPP32" ] -then - dpkg --add-architecture i386 - apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated - apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y - rm -rfv ./*all.deb -else - echo "i386 Repos are synced" -fi - -PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz) -if [ ! -z "$PPP64" ] -then - apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated - apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y -else - echo "AMD64 Repos are synced" - exit 0 -fi +PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) +rm -rfv ./output/*all.deb +PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) # Get ubuntu multiverse pool -PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz) -if [ ! -z "$PPP32" ] -then - dpkg --add-architecture i386 - apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated - apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y - rm -rfv ./*all.deb -else - echo "i386 multiverse Repos are synced" -fi - -PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz) -if [ ! -z "$PPP64" ] -then - apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated - apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y -else - echo "AMD64 multiverse Repos are synced" - exit 0 -fi +PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) +rm -rfv ./output/*all.deb +PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) # Get ubuntu restricted pool -PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz) -if [ ! -z "$PPP32" ] -then - dpkg --add-architecture i386 - apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated - apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y - rm -rfv ./*all.deb -else - echo "i386 restricted Repos are synced" -fi - -PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz) -if [ ! -z "$PPP64" ] -then - apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated - apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y -else - echo "AMD64 restricted Repos are synced" - exit 0 -fi +PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) +rm -rfv ./output/*all.deb +PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) # Get ubuntu universe pool -PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz) -if [ ! -z "$PPP32" ] -then - dpkg --add-architecture i386 - apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated - apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y - rm -rfv ./*all.deb -else - echo "i386 universe Repos are synced" -fi - -PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz) -if [ ! -z "$PPP64" ] -then - apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated - apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y -else - echo "AMD64 universe Repos are synced" - exit 0 -fi - -# Return to output -cd ../ +PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) +rm -rfv ./output/*all.deb +PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/) # Sign the packages for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done