repo-tools/ppp-sync
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make ppp download as well

Browse Source
This commit is contained in:
ferrreo 2023-09-05 00:02:14 +01:00
parent 4e7e23e4ed
commit 808a11ce45
3 changed files with 132 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
ppp
View File

Binary file not shown.

130
src/main.go
View File

@ -2,26 +2,40 @@ package main
import ( import (
"bufio" "bufio"
"fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"os" "os"
"strings" "strings"
"sync"
"github.com/ulikunitz/xz" "github.com/ulikunitz/xz"
) )
func main() { func main() {
base := os.Args[1] config := config{
target := os.Args[2] Source: os.Args[1],
Target: os.Args[2],
Download: false,
}
basePackages := processFile(base) if len(os.Args) > 3 {
targetPackages := processFile(target) config.Download = true
config.DlUrl = os.Args[3]
config.Output = os.Args[4]
}
compare(basePackages, targetPackages) basePackages := processFile(config.Source)
targetPackages := processFile(config.Target)
changed := compare(basePackages, targetPackages, config.Download)
if config.Download {
download(changed, config.DlUrl, config.Output)
}
} }
func processFile(url string) map[string]string { func processFile(url string) map[string]packageInfo {
resp, err := http.Get(url) resp, err := http.Get(url)
if err != nil { if err != nil {
@ -37,7 +51,7 @@ func processFile(url string) map[string]string {
rdr = r rdr = r
} }
packages := make(map[string]string) packages := make(map[string]packageInfo)
var currentPackage string var currentPackage string
scanner := bufio.NewScanner(rdr) scanner := bufio.NewScanner(rdr)
for scanner.Scan() { for scanner.Scan() {
@ -53,30 +67,120 @@ func processFile(url string) map[string]string {
_, broken := brokenPackages[pkName] _, broken := brokenPackages[pkName]
if !broken { if !broken {
currentPackage = pkName currentPackage = pkName
packages[currentPackage] = packageInfo{
Name: pkName,
}
} else { } else {
currentPackage = "" currentPackage = ""
} }
} }
} else { } else {
if strings.HasPrefix(line, "Version: ") && currentPackage != "" { if strings.HasPrefix(line, "Version: ") {
packages[currentPackage] = strings.TrimPrefix(line, "Version: ") packages[currentPackage] = packageInfo{
Name: currentPackage,
Version: strings.TrimPrefix(line, "Version: "),
}
}
if strings.HasPrefix(line, "Filename: ") {
packages[currentPackage] = packageInfo{
Name: currentPackage,
Version: packages[currentPackage].Version,
FilePath: strings.TrimPrefix(line, "Filename: "),
}
} }
} }
} }
return packages return packages
} }
func compare(basePackages map[string]string, targetPackages map[string]string) { func compare(basePackages map[string]packageInfo, targetPackages map[string]packageInfo, download bool) map[string]packageInfo {
for pack, version := range targetPackages { output := make(map[string]packageInfo)
for pack, info := range targetPackages {
if baseVersion, ok := basePackages[pack]; ok { if baseVersion, ok := basePackages[pack]; ok {
if baseVersion != version { if baseVersion.Version != info.Version {
output[pack] = info
if !download {
os.Stdout.WriteString(pack) os.Stdout.WriteString(pack)
} }
}
} else { } else {
output[pack] = info
if !download {
os.Stdout.WriteString(pack) os.Stdout.WriteString(pack)
} }
} }
}
return output
}
func download(packages map[string]packageInfo, url string, output string) {
// Create a buffered channel to store the packages to be downloaded
packageQueue := make(chan packageInfo, 10)
// Create a worker pool with 10 workers
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case pack, ok := <-packageQueue:
if !ok {
return
}
fmt.Printf("Downloading %s \n", pack.Name)
resp, err := http.Get(url + pack.FilePath)
if err != nil {
fmt.Printf("Failed to download %s: %v \n", pack.Name, err)
continue
}
defer resp.Body.Close()
rdr := io.Reader(resp.Body)
path := output + strings.Split(pack.FilePath, "/")[len(strings.Split(pack.FilePath, "/"))-1]
file, err := os.Create(path)
if err != nil {
fmt.Printf("Failed to create file %s: %v \n", path, err)
continue
}
defer file.Close()
_, err = io.Copy(file, rdr)
if err != nil {
fmt.Printf("Failed to save file %s: %v \n", path, err)
continue
}
default:
// No more packages to download, exit the goroutine
return
}
}
}()
}
// Add the packages to the queue
for _, pack := range packages {
packageQueue <- pack
}
// Close the queue to signal the workers to stop
close(packageQueue)
// Wait for all the workers to finish
wg.Wait()
}
type config struct {
Source string
Target string
Download bool
DlUrl string
Output string
}
type packageInfo struct {
Name string
Version string
FilePath string
} }
var brokenPackages = map[string]bool{ var brokenPackages = map[string]bool{

97
ubuntu.sh
View File

@ -6,7 +6,7 @@ chmod 755 ./ppp
# output folders # output folders
mkdir -p ./output/output mkdir -p ./output/output
cd ./output/output cd ./output
# temp # temp
apt update apt update
@ -17,98 +17,27 @@ apt install dpkg-sig wget rsync -y
# Get ubuntu main pool # Get ubuntu main pool
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz) PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
if [ ! -z "$PPP32" ] rm -rfv ./output/*all.deb
then PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
dpkg --add-architecture i386
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
rm -rfv ./*all.deb
else
echo "i386 Repos are synced"
fi
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz)
if [ ! -z "$PPP64" ]
then
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
else
echo "AMD64 Repos are synced"
exit 0
fi
# Get ubuntu multiverse pool # Get ubuntu multiverse pool
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz) PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
if [ ! -z "$PPP32" ] rm -rfv ./output/*all.deb
then PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
dpkg --add-architecture i386
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
rm -rfv ./*all.deb
else
echo "i386 multiverse Repos are synced"
fi
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz)
if [ ! -z "$PPP64" ]
then
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
else
echo "AMD64 multiverse Repos are synced"
exit 0
fi
# Get ubuntu restricted pool # Get ubuntu restricted pool
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz) PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
if [ ! -z "$PPP32" ] rm -rfv ./output/*all.deb
then PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
dpkg --add-architecture i386
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
rm -rfv ./*all.deb
else
echo "i386 restricted Repos are synced"
fi
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz)
if [ ! -z "$PPP64" ]
then
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
else
echo "AMD64 restricted Repos are synced"
exit 0
fi
# Get ubuntu universe pool # Get ubuntu universe pool
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz) PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
if [ ! -z "$PPP32" ] rm -rfv ./output/*all.deb
then PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
dpkg --add-architecture i386
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
rm -rfv ./*all.deb
else
echo "i386 universe Repos are synced"
fi
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz)
if [ ! -z "$PPP64" ]
then
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
else
echo "AMD64 universe Repos are synced"
exit 0
fi
# Return to output
cd ../
# Sign the packages # Sign the packages
for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done