make ppp download as well
This commit is contained in:
parent
4e7e23e4ed
commit
808a11ce45
134
src/main.go
134
src/main.go
@ -2,26 +2,40 @@ package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
func main() {
|
||||
base := os.Args[1]
|
||||
target := os.Args[2]
|
||||
config := config{
|
||||
Source: os.Args[1],
|
||||
Target: os.Args[2],
|
||||
Download: false,
|
||||
}
|
||||
|
||||
basePackages := processFile(base)
|
||||
targetPackages := processFile(target)
|
||||
if len(os.Args) > 3 {
|
||||
config.Download = true
|
||||
config.DlUrl = os.Args[3]
|
||||
config.Output = os.Args[4]
|
||||
}
|
||||
|
||||
compare(basePackages, targetPackages)
|
||||
basePackages := processFile(config.Source)
|
||||
targetPackages := processFile(config.Target)
|
||||
|
||||
changed := compare(basePackages, targetPackages, config.Download)
|
||||
if config.Download {
|
||||
download(changed, config.DlUrl, config.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func processFile(url string) map[string]string {
|
||||
func processFile(url string) map[string]packageInfo {
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
@ -37,7 +51,7 @@ func processFile(url string) map[string]string {
|
||||
rdr = r
|
||||
}
|
||||
|
||||
packages := make(map[string]string)
|
||||
packages := make(map[string]packageInfo)
|
||||
var currentPackage string
|
||||
scanner := bufio.NewScanner(rdr)
|
||||
for scanner.Scan() {
|
||||
@ -53,30 +67,120 @@ func processFile(url string) map[string]string {
|
||||
_, broken := brokenPackages[pkName]
|
||||
if !broken {
|
||||
currentPackage = pkName
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: pkName,
|
||||
}
|
||||
} else {
|
||||
currentPackage = ""
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
|
||||
packages[currentPackage] = strings.TrimPrefix(line, "Version: ")
|
||||
if strings.HasPrefix(line, "Version: ") {
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: currentPackage,
|
||||
Version: strings.TrimPrefix(line, "Version: "),
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(line, "Filename: ") {
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: currentPackage,
|
||||
Version: packages[currentPackage].Version,
|
||||
FilePath: strings.TrimPrefix(line, "Filename: "),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return packages
|
||||
|
||||
}
|
||||
|
||||
func compare(basePackages map[string]string, targetPackages map[string]string) {
|
||||
for pack, version := range targetPackages {
|
||||
func compare(basePackages map[string]packageInfo, targetPackages map[string]packageInfo, download bool) map[string]packageInfo {
|
||||
output := make(map[string]packageInfo)
|
||||
for pack, info := range targetPackages {
|
||||
if baseVersion, ok := basePackages[pack]; ok {
|
||||
if baseVersion != version {
|
||||
os.Stdout.WriteString(pack)
|
||||
if baseVersion.Version != info.Version {
|
||||
output[pack] = info
|
||||
if !download {
|
||||
os.Stdout.WriteString(pack)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
os.Stdout.WriteString(pack)
|
||||
output[pack] = info
|
||||
if !download {
|
||||
os.Stdout.WriteString(pack)
|
||||
}
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func download(packages map[string]packageInfo, url string, output string) {
|
||||
// Create a buffered channel to store the packages to be downloaded
|
||||
packageQueue := make(chan packageInfo, 10)
|
||||
|
||||
// Create a worker pool with 10 workers
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case pack, ok := <-packageQueue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
fmt.Printf("Downloading %s \n", pack.Name)
|
||||
resp, err := http.Get(url + pack.FilePath)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to download %s: %v \n", pack.Name, err)
|
||||
continue
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rdr := io.Reader(resp.Body)
|
||||
path := output + strings.Split(pack.FilePath, "/")[len(strings.Split(pack.FilePath, "/"))-1]
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create file %s: %v \n", path, err)
|
||||
continue
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(file, rdr)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to save file %s: %v \n", path, err)
|
||||
continue
|
||||
}
|
||||
default:
|
||||
// No more packages to download, exit the goroutine
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Add the packages to the queue
|
||||
for _, pack := range packages {
|
||||
packageQueue <- pack
|
||||
}
|
||||
|
||||
// Close the queue to signal the workers to stop
|
||||
close(packageQueue)
|
||||
|
||||
// Wait for all the workers to finish
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Source string
|
||||
Target string
|
||||
Download bool
|
||||
DlUrl string
|
||||
Output string
|
||||
}
|
||||
|
||||
type packageInfo struct {
|
||||
Name string
|
||||
Version string
|
||||
FilePath string
|
||||
}
|
||||
|
||||
var brokenPackages = map[string]bool{
|
||||
|
||||
97
ubuntu.sh
97
ubuntu.sh
@ -6,7 +6,7 @@ chmod 755 ./ppp
|
||||
|
||||
# output folders
|
||||
mkdir -p ./output/output
|
||||
cd ./output/output
|
||||
cd ./output
|
||||
|
||||
# temp
|
||||
apt update
|
||||
@ -17,98 +17,27 @@ apt install dpkg-sig wget rsync -y
|
||||
|
||||
# Get ubuntu main pool
|
||||
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz)
|
||||
if [ ! -z "$PPP32" ]
|
||||
then
|
||||
dpkg --add-architecture i386
|
||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
||||
rm -rfv ./*all.deb
|
||||
else
|
||||
echo "i386 Repos are synced"
|
||||
fi
|
||||
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz)
|
||||
if [ ! -z "$PPP64" ]
|
||||
then
|
||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
||||
else
|
||||
echo "AMD64 Repos are synced"
|
||||
exit 0
|
||||
fi
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
rm -rfv ./output/*all.deb
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
|
||||
# Get ubuntu multiverse pool
|
||||
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz)
|
||||
if [ ! -z "$PPP32" ]
|
||||
then
|
||||
dpkg --add-architecture i386
|
||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
||||
rm -rfv ./*all.deb
|
||||
else
|
||||
echo "i386 multiverse Repos are synced"
|
||||
fi
|
||||
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz)
|
||||
if [ ! -z "$PPP64" ]
|
||||
then
|
||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
||||
else
|
||||
echo "AMD64 multiverse Repos are synced"
|
||||
exit 0
|
||||
fi
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
rm -rfv ./output/*all.deb
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
|
||||
# Get ubuntu restricted pool
|
||||
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz)
|
||||
if [ ! -z "$PPP32" ]
|
||||
then
|
||||
dpkg --add-architecture i386
|
||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
||||
rm -rfv ./*all.deb
|
||||
else
|
||||
echo "i386 restricted Repos are synced"
|
||||
fi
|
||||
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz)
|
||||
if [ ! -z "$PPP64" ]
|
||||
then
|
||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
||||
else
|
||||
echo "AMD64 restricted Repos are synced"
|
||||
exit 0
|
||||
fi
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
rm -rfv ./output/*all.deb
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
|
||||
# Get ubuntu universe pool
|
||||
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz)
|
||||
if [ ! -z "$PPP32" ]
|
||||
then
|
||||
dpkg --add-architecture i386
|
||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
||||
rm -rfv ./*all.deb
|
||||
else
|
||||
echo "i386 universe Repos are synced"
|
||||
fi
|
||||
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz)
|
||||
if [ ! -z "$PPP64" ]
|
||||
then
|
||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
||||
else
|
||||
echo "AMD64 universe Repos are synced"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Return to output
|
||||
cd ../
|
||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
rm -rfv ./output/*all.deb
|
||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||
|
||||
# Sign the packages
|
||||
for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done
|
||||
|
||||
Loading…
Reference in New Issue
Block a user