make ppp download as well

src/main.go

@@ -2,26 +2,40 @@ package main
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/ulikunitz/xz"
 )
 
 func main() {
-	base := os.Args[1]
+	config := config{
-	target := os.Args[2]
+		Source:   os.Args[1],
-
+		Target:   os.Args[2],
-	basePackages := processFile(base)
+		Download: false,
-	targetPackages := processFile(target)
-
-	compare(basePackages, targetPackages)
 	}
 
-func processFile(url string) map[string]string {
+	if len(os.Args) > 3 {
+		config.Download = true
+		config.DlUrl = os.Args[3]
+		config.Output = os.Args[4]
+	}
+
+	basePackages := processFile(config.Source)
+	targetPackages := processFile(config.Target)
+
+	changed := compare(basePackages, targetPackages, config.Download)
+	if config.Download {
+		download(changed, config.DlUrl, config.Output)
+	}
+}
+
+func processFile(url string) map[string]packageInfo {
 
 	resp, err := http.Get(url)
 	if err != nil {
@@ -37,7 +51,7 @@ func processFile(url string) map[string]string {
 		rdr = r
 	}
 
-	packages := make(map[string]string)
+	packages := make(map[string]packageInfo)
 	var currentPackage string
 	scanner := bufio.NewScanner(rdr)
 	for scanner.Scan() {
@@ -53,31 +67,121 @@ func processFile(url string) map[string]string {
 				_, broken := brokenPackages[pkName]
 				if !broken {
 					currentPackage = pkName
+					packages[currentPackage] = packageInfo{
+						Name: pkName,
+					}
 				} else {
 					currentPackage = ""
 				}
 			}
 		} else {
-			if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
+			if strings.HasPrefix(line, "Version: ") {
-				packages[currentPackage] = strings.TrimPrefix(line, "Version: ")
+				packages[currentPackage] = packageInfo{
+					Name:    currentPackage,
+					Version: strings.TrimPrefix(line, "Version: "),
+				}
+			}
+			if strings.HasPrefix(line, "Filename: ") {
+				packages[currentPackage] = packageInfo{
+					Name:     currentPackage,
+					Version:  packages[currentPackage].Version,
+					FilePath: strings.TrimPrefix(line, "Filename: "),
+				}
 			}
 		}
 	}
 	return packages
-
 }
 
-func compare(basePackages map[string]string, targetPackages map[string]string) {
+func compare(basePackages map[string]packageInfo, targetPackages map[string]packageInfo, download bool) map[string]packageInfo {
-	for pack, version := range targetPackages {
+	output := make(map[string]packageInfo)
+	for pack, info := range targetPackages {
 		if baseVersion, ok := basePackages[pack]; ok {
-			if baseVersion != version {
+			if baseVersion.Version != info.Version {
+				output[pack] = info
+				if !download {
 					os.Stdout.WriteString(pack)
 				}
+			}
 		} else {
+			output[pack] = info
+			if !download {
 				os.Stdout.WriteString(pack)
 			}
 		}
 	}
+	return output
+}
+
+func download(packages map[string]packageInfo, url string, output string) {
+	// Create a buffered channel to store the packages to be downloaded
+	packageQueue := make(chan packageInfo, 10)
+
+	// Create a worker pool with 10 workers
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case pack, ok := <-packageQueue:
+					if !ok {
+						return
+					}
+					fmt.Printf("Downloading %s \n", pack.Name)
+					resp, err := http.Get(url + pack.FilePath)
+					if err != nil {
+						fmt.Printf("Failed to download %s: %v \n", pack.Name, err)
+						continue
+					}
+					defer resp.Body.Close()
+					rdr := io.Reader(resp.Body)
+					path := output + strings.Split(pack.FilePath, "/")[len(strings.Split(pack.FilePath, "/"))-1]
+					file, err := os.Create(path)
+					if err != nil {
+						fmt.Printf("Failed to create file %s: %v \n", path, err)
+						continue
+					}
+					defer file.Close()
+					_, err = io.Copy(file, rdr)
+					if err != nil {
+						fmt.Printf("Failed to save file %s: %v \n", path, err)
+						continue
+					}
+				default:
+					// No more packages to download, exit the goroutine
+					return
+				}
+			}
+		}()
+	}
+
+	// Add the packages to the queue
+	for _, pack := range packages {
+		packageQueue <- pack
+	}
+
+	// Close the queue to signal the workers to stop
+	close(packageQueue)
+
+	// Wait for all the workers to finish
+	wg.Wait()
+}
+
+type config struct {
+	Source   string
+	Target   string
+	Download bool
+	DlUrl    string
+	Output   string
+}
+
+type packageInfo struct {
+	Name     string
+	Version  string
+	FilePath string
+}
 
 var brokenPackages = map[string]bool{
 	"libkpim5mbox-data ":               true,

ubuntu.sh

@@ -6,7 +6,7 @@ chmod 755 ./ppp
 
 # output folders
 mkdir -p ./output/output
-cd ./output/output
+cd ./output
 
 # temp
 apt update
@@ -17,98 +17,27 @@ apt install dpkg-sig wget rsync -y
 
 # Get ubuntu main pool
 
-PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz)
+PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-if [ ! -z "$PPP32" ]
+rm -rfv ./output/*all.deb
-then
+PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-    dpkg --add-architecture i386
-    apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated 
-    apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
-    rm -rfv ./*all.deb
-else
-    echo "i386 Repos are synced"
-fi
-
-PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz)
-if [ ! -z "$PPP64" ]
-then
-    apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated 
-    apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
-else
-    echo "AMD64 Repos are synced"
-    exit 0
-fi
 
 # Get ubuntu multiverse pool
 
-PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz)
+PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-if [ ! -z "$PPP32" ]
+rm -rfv ./output/*all.deb
-then
+PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-    dpkg --add-architecture i386
-    apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated 
-    apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
-    rm -rfv ./*all.deb
-else
-    echo "i386 multiverse Repos are synced"
-fi
-
-PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz)
-if [ ! -z "$PPP64" ]
-then
-    apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated 
-    apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
-else
-    echo "AMD64 multiverse Repos are synced"
-    exit 0
-fi
 
 # Get ubuntu restricted pool
 
-PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz)
+PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-if [ ! -z "$PPP32" ]
+rm -rfv ./output/*all.deb
-then
+PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-    dpkg --add-architecture i386
-    apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated 
-    apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
-    rm -rfv ./*all.deb
-else
-    echo "i386 restricted Repos are synced"
-fi
-
-PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz)
-if [ ! -z "$PPP64" ]
-then
-    apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated 
-    apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
-else
-    echo "AMD64 restricted Repos are synced"
-    exit 0
-fi
 
 # Get ubuntu universe pool
 
-PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz)
+PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-if [ ! -z "$PPP32" ]
+rm -rfv ./output/*all.deb
-then
+PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
-    dpkg --add-architecture i386
-    apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated 
-    apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
-    rm -rfv ./*all.deb
-else
-    echo "i386 universe Repos are synced"
-fi
-
-PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz)
-if [ ! -z "$PPP64" ]
-then
-    apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated 
-    apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
-else
-    echo "AMD64 universe Repos are synced"
-    exit 0
-fi
-
-# Return to output
-cd ../
 
 # Sign the packages
 for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done