make ppp download as well
This commit is contained in:
parent
4e7e23e4ed
commit
808a11ce45
134
src/main.go
134
src/main.go
@ -2,26 +2,40 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
"github.com/ulikunitz/xz"
|
"github.com/ulikunitz/xz"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
base := os.Args[1]
|
config := config{
|
||||||
target := os.Args[2]
|
Source: os.Args[1],
|
||||||
|
Target: os.Args[2],
|
||||||
|
Download: false,
|
||||||
|
}
|
||||||
|
|
||||||
basePackages := processFile(base)
|
if len(os.Args) > 3 {
|
||||||
targetPackages := processFile(target)
|
config.Download = true
|
||||||
|
config.DlUrl = os.Args[3]
|
||||||
|
config.Output = os.Args[4]
|
||||||
|
}
|
||||||
|
|
||||||
compare(basePackages, targetPackages)
|
basePackages := processFile(config.Source)
|
||||||
|
targetPackages := processFile(config.Target)
|
||||||
|
|
||||||
|
changed := compare(basePackages, targetPackages, config.Download)
|
||||||
|
if config.Download {
|
||||||
|
download(changed, config.DlUrl, config.Output)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func processFile(url string) map[string]string {
|
func processFile(url string) map[string]packageInfo {
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
resp, err := http.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -37,7 +51,7 @@ func processFile(url string) map[string]string {
|
|||||||
rdr = r
|
rdr = r
|
||||||
}
|
}
|
||||||
|
|
||||||
packages := make(map[string]string)
|
packages := make(map[string]packageInfo)
|
||||||
var currentPackage string
|
var currentPackage string
|
||||||
scanner := bufio.NewScanner(rdr)
|
scanner := bufio.NewScanner(rdr)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
@ -53,30 +67,120 @@ func processFile(url string) map[string]string {
|
|||||||
_, broken := brokenPackages[pkName]
|
_, broken := brokenPackages[pkName]
|
||||||
if !broken {
|
if !broken {
|
||||||
currentPackage = pkName
|
currentPackage = pkName
|
||||||
|
packages[currentPackage] = packageInfo{
|
||||||
|
Name: pkName,
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
currentPackage = ""
|
currentPackage = ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
|
if strings.HasPrefix(line, "Version: ") {
|
||||||
packages[currentPackage] = strings.TrimPrefix(line, "Version: ")
|
packages[currentPackage] = packageInfo{
|
||||||
|
Name: currentPackage,
|
||||||
|
Version: strings.TrimPrefix(line, "Version: "),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, "Filename: ") {
|
||||||
|
packages[currentPackage] = packageInfo{
|
||||||
|
Name: currentPackage,
|
||||||
|
Version: packages[currentPackage].Version,
|
||||||
|
FilePath: strings.TrimPrefix(line, "Filename: "),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return packages
|
return packages
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func compare(basePackages map[string]string, targetPackages map[string]string) {
|
func compare(basePackages map[string]packageInfo, targetPackages map[string]packageInfo, download bool) map[string]packageInfo {
|
||||||
for pack, version := range targetPackages {
|
output := make(map[string]packageInfo)
|
||||||
|
for pack, info := range targetPackages {
|
||||||
if baseVersion, ok := basePackages[pack]; ok {
|
if baseVersion, ok := basePackages[pack]; ok {
|
||||||
if baseVersion != version {
|
if baseVersion.Version != info.Version {
|
||||||
os.Stdout.WriteString(pack)
|
output[pack] = info
|
||||||
|
if !download {
|
||||||
|
os.Stdout.WriteString(pack)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
os.Stdout.WriteString(pack)
|
output[pack] = info
|
||||||
|
if !download {
|
||||||
|
os.Stdout.WriteString(pack)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
func download(packages map[string]packageInfo, url string, output string) {
|
||||||
|
// Create a buffered channel to store the packages to be downloaded
|
||||||
|
packageQueue := make(chan packageInfo, 10)
|
||||||
|
|
||||||
|
// Create a worker pool with 10 workers
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case pack, ok := <-packageQueue:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("Downloading %s \n", pack.Name)
|
||||||
|
resp, err := http.Get(url + pack.FilePath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed to download %s: %v \n", pack.Name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
rdr := io.Reader(resp.Body)
|
||||||
|
path := output + strings.Split(pack.FilePath, "/")[len(strings.Split(pack.FilePath, "/"))-1]
|
||||||
|
file, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed to create file %s: %v \n", path, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
_, err = io.Copy(file, rdr)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed to save file %s: %v \n", path, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// No more packages to download, exit the goroutine
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the packages to the queue
|
||||||
|
for _, pack := range packages {
|
||||||
|
packageQueue <- pack
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the queue to signal the workers to stop
|
||||||
|
close(packageQueue)
|
||||||
|
|
||||||
|
// Wait for all the workers to finish
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
Source string
|
||||||
|
Target string
|
||||||
|
Download bool
|
||||||
|
DlUrl string
|
||||||
|
Output string
|
||||||
|
}
|
||||||
|
|
||||||
|
type packageInfo struct {
|
||||||
|
Name string
|
||||||
|
Version string
|
||||||
|
FilePath string
|
||||||
}
|
}
|
||||||
|
|
||||||
var brokenPackages = map[string]bool{
|
var brokenPackages = map[string]bool{
|
||||||
|
97
ubuntu.sh
97
ubuntu.sh
@ -6,7 +6,7 @@ chmod 755 ./ppp
|
|||||||
|
|
||||||
# output folders
|
# output folders
|
||||||
mkdir -p ./output/output
|
mkdir -p ./output/output
|
||||||
cd ./output/output
|
cd ./output
|
||||||
|
|
||||||
# temp
|
# temp
|
||||||
apt update
|
apt update
|
||||||
@ -17,98 +17,27 @@ apt install dpkg-sig wget rsync -y
|
|||||||
|
|
||||||
# Get ubuntu main pool
|
# Get ubuntu main pool
|
||||||
|
|
||||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz)
|
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
if [ ! -z "$PPP32" ]
|
rm -rfv ./output/*all.deb
|
||||||
then
|
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
dpkg --add-architecture i386
|
|
||||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
|
||||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
|
||||||
rm -rfv ./*all.deb
|
|
||||||
else
|
|
||||||
echo "i386 Repos are synced"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/main/binary-amd64/Packages.xz)
|
|
||||||
if [ ! -z "$PPP64" ]
|
|
||||||
then
|
|
||||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
|
||||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
|
||||||
else
|
|
||||||
echo "AMD64 Repos are synced"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Get ubuntu multiverse pool
|
# Get ubuntu multiverse pool
|
||||||
|
|
||||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz)
|
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
if [ ! -z "$PPP32" ]
|
rm -rfv ./output/*all.deb
|
||||||
then
|
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
dpkg --add-architecture i386
|
|
||||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
|
||||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
|
||||||
rm -rfv ./*all.deb
|
|
||||||
else
|
|
||||||
echo "i386 multiverse Repos are synced"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/multiverse/binary-amd64/Packages.xz)
|
|
||||||
if [ ! -z "$PPP64" ]
|
|
||||||
then
|
|
||||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
|
||||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
|
||||||
else
|
|
||||||
echo "AMD64 multiverse Repos are synced"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Get ubuntu restricted pool
|
# Get ubuntu restricted pool
|
||||||
|
|
||||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz)
|
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
if [ ! -z "$PPP32" ]
|
rm -rfv ./output/*all.deb
|
||||||
then
|
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
dpkg --add-architecture i386
|
|
||||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
|
||||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
|
||||||
rm -rfv ./*all.deb
|
|
||||||
else
|
|
||||||
echo "i386 restricted Repos are synced"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/restricted/binary-amd64/Packages.xz)
|
|
||||||
if [ ! -z "$PPP64" ]
|
|
||||||
then
|
|
||||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
|
||||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
|
||||||
else
|
|
||||||
echo "AMD64 restricted Repos are synced"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Get ubuntu universe pool
|
# Get ubuntu universe pool
|
||||||
|
|
||||||
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz)
|
PPP32=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-i386/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-i386/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
if [ ! -z "$PPP32" ]
|
rm -rfv ./output/*all.deb
|
||||||
then
|
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz http://archive.ubuntu.com/ubuntu/ ./output/)
|
||||||
dpkg --add-architecture i386
|
|
||||||
apt update -o APT::Architecture="i386" -o APT::Architectures="i386" -y --allow-unauthenticated
|
|
||||||
apt download $PPP32 -o APT::Architecture="i386" -o APT::Architectures="i386" -y
|
|
||||||
rm -rfv ./*all.deb
|
|
||||||
else
|
|
||||||
echo "i386 universe Repos are synced"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PPP64=$(../../ppp https://ppa.pika-os.com/dists/lunar/ubuntu/binary-amd64/Packages http://archive.ubuntu.com/ubuntu/dists/lunar/universe/binary-amd64/Packages.xz)
|
|
||||||
if [ ! -z "$PPP64" ]
|
|
||||||
then
|
|
||||||
apt update -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y --allow-unauthenticated
|
|
||||||
apt download $PPP64 -o APT::Architecture="amd64" -o APT::Architectures="amd64" -y
|
|
||||||
else
|
|
||||||
echo "AMD64 universe Repos are synced"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Return to output
|
|
||||||
cd ../
|
|
||||||
|
|
||||||
# Sign the packages
|
# Sign the packages
|
||||||
for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done
|
for f in ./output/*.deb; do dpkg-sig --sign builder "$f"; done
|
||||||
|
Loading…
Reference in New Issue
Block a user