530 lines
13 KiB
Go
530 lines
13 KiB
Go
package packages
|
|
|
|
import (
|
|
"brunel/config"
|
|
"brunel/deb"
|
|
"brunel/domain"
|
|
"brunel/helpers"
|
|
"compress/bzip2"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"pault.ag/go/debian/version"
|
|
|
|
"github.com/alphadose/haxmap"
|
|
"github.com/klauspost/compress/gzip"
|
|
"github.com/ulikunitz/xz"
|
|
)
|
|
|
|
var LastUpdateTime time.Time
|
|
var currentPackages = haxmap.New[string, domain.SourcePackage]()
|
|
|
|
func ProcessPackages() error {
|
|
var internalPackages = haxmap.New[string, domain.SourcePackage]()
|
|
var externalPackages = haxmap.New[string, domain.SourcePackage]()
|
|
err := LoadInternalPackages(internalPackages)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = LoadExternalPackages(externalPackages)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
combinePackages(internalPackages)
|
|
combinePackages(externalPackages)
|
|
|
|
ProcessMissingPackages(internalPackages, externalPackages)
|
|
ProcessStalePackages(internalPackages, externalPackages)
|
|
|
|
updatedPackages := haxmap.New[string, domain.SourcePackage]()
|
|
|
|
internalPackages.ForEach(func(k string, v domain.SourcePackage) bool {
|
|
curr, exists := currentPackages.Get(k)
|
|
if !exists {
|
|
updatedPackages.Set(k, v)
|
|
return true
|
|
}
|
|
|
|
mergedPackage := domain.SourcePackage{
|
|
Name: curr.Name,
|
|
Packages: haxmap.New[string, domain.PackageInfo](),
|
|
}
|
|
|
|
v.Packages.ForEach(func(pkgName string, newPkg domain.PackageInfo) bool {
|
|
if existingPkg, ok := curr.Packages.Get(pkgName); ok {
|
|
if newPkg.Version != existingPkg.Version ||
|
|
newPkg.Status != existingPkg.Status ||
|
|
newPkg.NewVersion != existingPkg.NewVersion {
|
|
if newPkg.Status == domain.Current {
|
|
mergedPackage.Packages.Set(pkgName, newPkg)
|
|
return true
|
|
}
|
|
newPkg.LastBuildStatus = existingPkg.LastBuildStatus
|
|
newPkg.BuildAttempts = existingPkg.BuildAttempts
|
|
mergedPackage.Packages.Set(pkgName, newPkg)
|
|
} else {
|
|
mergedPackage.Packages.Set(pkgName, existingPkg)
|
|
}
|
|
} else {
|
|
mergedPackage.Packages.Set(pkgName, newPkg)
|
|
}
|
|
return true
|
|
})
|
|
|
|
curr.Packages.ForEach(func(pkgName string, pkg domain.PackageInfo) bool {
|
|
if _, exists := mergedPackage.Packages.Get(pkgName); !exists {
|
|
mergedPackage.Packages.Set(pkgName, pkg)
|
|
}
|
|
return true
|
|
})
|
|
|
|
updatedPackages.Set(k, mergedPackage)
|
|
return true
|
|
})
|
|
|
|
updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool {
|
|
for _, pkg := range config.Configs.I386List {
|
|
if v.Name == pkg || v.Name == pkg+"-dmo" {
|
|
v.Has32bit = true
|
|
updatedPackages.Set(k, v)
|
|
return true
|
|
}
|
|
}
|
|
v.Has32bit = false
|
|
updatedPackages.Set(k, v)
|
|
return true
|
|
})
|
|
|
|
for _, pkg := range config.Configs.I386List {
|
|
if _, ok := updatedPackages.Get(pkg); !ok {
|
|
_, ok := updatedPackages.Get(pkg + "-dmo")
|
|
if ok {
|
|
continue
|
|
}
|
|
updatedPackages.Set(pkg, domain.SourcePackage{
|
|
Name: pkg,
|
|
Has32bit: true,
|
|
Packages: haxmap.New[string, domain.PackageInfo](),
|
|
})
|
|
}
|
|
}
|
|
|
|
currentPackages = updatedPackages
|
|
|
|
LastUpdateTime = time.Now()
|
|
err = helpers.DBInst.DropPackages()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = SaveToDb()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func GetPackages() *haxmap.Map[string, domain.SourcePackage] {
|
|
return currentPackages
|
|
}
|
|
|
|
func UpdatePackage(pkg domain.PackageInfo) error {
|
|
curr, ok := currentPackages.Get(pkg.Source)
|
|
if !ok {
|
|
return fmt.Errorf("package %s not found", pkg.Source)
|
|
}
|
|
curr.Packages.Set(pkg.PackageName, pkg)
|
|
currentPackages.Set(pkg.Source, curr)
|
|
return saveSingleToDb(curr)
|
|
}
|
|
|
|
func UpdateSourcePackage(pkg domain.SourcePackage) error {
|
|
currentPackages.Set(pkg.Name, pkg)
|
|
return saveSingleToDb(pkg)
|
|
}
|
|
|
|
func IsBuilt(pkg domain.PackageInfo) bool {
|
|
curr, ok := currentPackages.Get(pkg.Source)
|
|
if !ok {
|
|
return false
|
|
}
|
|
pk, ok := curr.Packages.Get(pkg.PackageName)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return pk.Status == domain.Built || pk.Status == domain.Current
|
|
}
|
|
|
|
func saveSingleToDb(pkg domain.SourcePackage) error {
|
|
err := helpers.DBInst.UpdatePackage(pkg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
LastUpdateTime = time.Now()
|
|
err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func SaveToDb() error {
|
|
err := helpers.DBInst.SavePackages(currentPackages)
|
|
if err != nil {
|
|
slog.Error(err.Error())
|
|
return err
|
|
}
|
|
LastUpdateTime = time.Now()
|
|
return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
|
|
}
|
|
|
|
func LoadFromDb() error {
|
|
packages, err := helpers.DBInst.GetPackages()
|
|
if err != nil {
|
|
slog.Error(err.Error())
|
|
return nil
|
|
}
|
|
slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int {
|
|
if a.Name == b.Name {
|
|
return 0
|
|
}
|
|
if a.Name > b.Name {
|
|
return 1
|
|
}
|
|
return -1
|
|
})
|
|
currentPackages.Clear()
|
|
for _, pkg := range packages {
|
|
currentPackages.Set(pkg.Name, pkg)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func LoadInternalPackages(internalPackages *haxmap.Map[string, domain.SourcePackage]) error {
|
|
localPackageFile := config.Configs.LocalPackageFiles
|
|
slices.SortStableFunc(localPackageFile, func(a, b config.PackageFile) int {
|
|
if a.Priority == b.Priority {
|
|
return 0
|
|
}
|
|
if a.Priority < b.Priority {
|
|
return 1
|
|
}
|
|
return -1
|
|
})
|
|
|
|
for _, pkg := range config.Configs.LocalPackageFiles {
|
|
for _, repo := range pkg.Subrepos {
|
|
packages, err := fetchPackageFile(pkg, repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
packages.ForEach(func(newKey string, newPkg domain.PackageInfo) bool {
|
|
pk, ok := internalPackages.Get(newPkg.Source)
|
|
if !ok {
|
|
newMap := haxmap.New[string, domain.PackageInfo]()
|
|
newMap.Set(newKey, newPkg)
|
|
internalPackages.Set(newPkg.Source, domain.SourcePackage{
|
|
Name: newPkg.Source,
|
|
Packages: newMap,
|
|
})
|
|
return true
|
|
}
|
|
pkg, ok := pk.Packages.Get(newKey)
|
|
if !ok {
|
|
pk.Packages.Set(newKey, newPkg)
|
|
return true
|
|
}
|
|
mVer, _ := version.Parse(pkg.Version)
|
|
extVer, _ := version.Parse(newPkg.Version)
|
|
cmpVal := version.Compare(extVer, mVer)
|
|
if cmpVal >= 0 {
|
|
pk.Packages.Set(newKey, newPkg)
|
|
return true
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func LoadExternalPackages(externalPackages *haxmap.Map[string, domain.SourcePackage]) error {
|
|
externalPackageFile := config.Configs.ExternalPackageFiles
|
|
slices.SortStableFunc(externalPackageFile, func(a, b config.PackageFile) int {
|
|
if a.Priority == b.Priority {
|
|
return 0
|
|
}
|
|
if a.Priority < b.Priority {
|
|
return -1
|
|
}
|
|
return 1
|
|
})
|
|
|
|
for _, pkg := range config.Configs.ExternalPackageFiles {
|
|
for _, repo := range pkg.Subrepos {
|
|
packages, err := fetchPackageFile(pkg, repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
packages.ForEach(func(k string, v domain.PackageInfo) bool {
|
|
pk, ok := externalPackages.Get(v.Source)
|
|
if !ok {
|
|
newMap := haxmap.New[string, domain.PackageInfo]()
|
|
newMap.Set(k, v)
|
|
externalPackages.Set(v.Source, domain.SourcePackage{
|
|
Name: v.Source,
|
|
Packages: newMap,
|
|
})
|
|
return true
|
|
}
|
|
pkg, ok := pk.Packages.Get(k)
|
|
if !ok {
|
|
pk.Packages.Set(k, v)
|
|
return true
|
|
}
|
|
mVer, _ := version.Parse(pkg.Version)
|
|
extVer, _ := version.Parse(v.Version)
|
|
cmpVal := version.Compare(extVer, mVer)
|
|
if cmpVal >= 0 {
|
|
pk.Packages.Set(k, v)
|
|
return true
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func ProcessMissingPackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
|
|
externalPackages.ForEach(func(k string, src domain.SourcePackage) bool {
|
|
_, ok := internalPackages.Get(k)
|
|
if !ok && src.Packages.Len() > 0 {
|
|
newStatus := domain.Missing
|
|
src.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
|
|
v.Status = newStatus
|
|
v.Version = strings.Split(v.Version, "+b")[0]
|
|
src.Packages.Set(k, v)
|
|
return true
|
|
})
|
|
internalPackages.Set(k, src)
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
func ProcessStalePackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
|
|
externalPackages.ForEach(func(newPackage string, newSource domain.SourcePackage) bool {
|
|
matchedPackage, ok := internalPackages.Get(newPackage)
|
|
if !ok || matchedPackage.Packages.Len() == 0 {
|
|
return true
|
|
}
|
|
matchedPackage.Packages.ForEach(func(currentKey string, currentPackage domain.PackageInfo) bool {
|
|
if currentPackage.Status == domain.Missing {
|
|
return true
|
|
}
|
|
newSource.Packages.ForEach(func(newKey string, newPackage domain.PackageInfo) bool {
|
|
if currentKey != newKey {
|
|
return true
|
|
}
|
|
newVersion := strings.Split(newPackage.Version, "+b")[0]
|
|
mVer, _ := version.Parse(currentPackage.Version)
|
|
extVer, _ := version.Parse(newVersion)
|
|
cmpVal := version.Compare(mVer, extVer)
|
|
if cmpVal < 0 {
|
|
if currentPackage.Status != domain.Error {
|
|
currentPackage.Status = domain.Stale
|
|
}
|
|
currentPackage.NewVersion = extVer.String()
|
|
matchedPackage.Packages.Set(currentKey, currentPackage)
|
|
}
|
|
return false
|
|
})
|
|
return true
|
|
})
|
|
return true
|
|
})
|
|
}
|
|
|
|
func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[string, domain.PackageInfo], error) {
|
|
resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
rdr := io.Reader(resp.Body)
|
|
if pkg.Compression == "bz2" {
|
|
r := bzip2.NewReader(resp.Body)
|
|
rdr = r
|
|
}
|
|
if pkg.Compression == "xz" {
|
|
r, err := xz.NewReader(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rdr = r
|
|
}
|
|
if pkg.Compression == "gz" {
|
|
r, err := gzip.NewReader(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rdr = r
|
|
}
|
|
|
|
packages := haxmap.New[string, domain.PackageInfo]()
|
|
sreader := deb.NewControlFileReader(rdr, false, false)
|
|
for {
|
|
stanza, err := sreader.ReadStanza()
|
|
if err != nil || stanza == nil {
|
|
break
|
|
}
|
|
|
|
if stanza["Section"] == "debian-installer" {
|
|
continue
|
|
}
|
|
|
|
name := stanza["Package"]
|
|
|
|
useWhitelist := pkg.UseWhitelist && len(pkg.Whitelist) > 0
|
|
if useWhitelist {
|
|
contained := nameContains(name, pkg.Whitelist)
|
|
if !contained {
|
|
continue
|
|
}
|
|
}
|
|
|
|
broken := nameContains(name, pkg.Blacklist)
|
|
if broken {
|
|
continue
|
|
}
|
|
|
|
sourceSplit := strings.Split(stanza["Source"], " ")
|
|
source := sourceSplit[0]
|
|
if source == "" {
|
|
source = name
|
|
}
|
|
|
|
// Extract version from Source if available
|
|
sourceVersion := ""
|
|
if len(sourceSplit) > 1 {
|
|
sourceVersion = strings.Trim(sourceSplit[1], "()")
|
|
}
|
|
|
|
// Use sourceVersion if available, otherwise use stanza["Version"]
|
|
versionStr := stanza["Version"]
|
|
if sourceVersion != "" {
|
|
versionStr = sourceVersion
|
|
}
|
|
|
|
ver, err := version.Parse(versionStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
pk, ok := packages.Get(name)
|
|
if ok {
|
|
matchedVer, _ := version.Parse(pk.Version)
|
|
cmpVal := version.Compare(ver, matchedVer)
|
|
if cmpVal < 0 {
|
|
continue
|
|
}
|
|
}
|
|
|
|
packages.Set(name, domain.PackageInfo{
|
|
PackageName: name,
|
|
Version: ver.String(),
|
|
Source: source,
|
|
Architecture: stanza["Architecture"],
|
|
Description: stanza["Description"],
|
|
Status: domain.Current,
|
|
})
|
|
}
|
|
|
|
return packages, nil
|
|
}
|
|
|
|
func nameContains(name string, match []string) bool {
|
|
for _, m := range match {
|
|
if strings.Contains(name, m) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func GetPackagesCount() domain.PackagesCount {
|
|
count := domain.PackagesCount{
|
|
Stale: 0,
|
|
Missing: 0,
|
|
Current: 0,
|
|
Error: 0,
|
|
Queued: 0,
|
|
Building: 0,
|
|
}
|
|
currentPackages.ForEach(func(k string, v domain.SourcePackage) bool {
|
|
v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool {
|
|
switch pkg.Status {
|
|
case domain.Stale:
|
|
count.Stale++
|
|
case domain.Missing:
|
|
count.Missing++
|
|
case domain.Built:
|
|
count.Current++
|
|
case domain.Current:
|
|
count.Current++
|
|
case domain.Error:
|
|
count.Error++
|
|
}
|
|
return false
|
|
})
|
|
return true
|
|
})
|
|
|
|
return count
|
|
}
|
|
|
|
func combinePackages(packages *haxmap.Map[string, domain.SourcePackage]) {
|
|
dmoPackages := haxmap.New[string, domain.SourcePackage]()
|
|
|
|
// First pass: identify and collect -dmo packages
|
|
packages.ForEach(func(k string, v domain.SourcePackage) bool {
|
|
if strings.HasSuffix(k, "-dmo") {
|
|
dmoPackages.Set(k, v)
|
|
packages.Del(k)
|
|
}
|
|
return true
|
|
})
|
|
|
|
// Second pass: combine -dmo packages with base packages or add them back
|
|
dmoPackages.ForEach(func(dmoName string, dmoPkg domain.SourcePackage) bool {
|
|
baseName := strings.TrimSuffix(dmoName, "-dmo")
|
|
basePkg, hasBase := packages.Get(baseName)
|
|
|
|
if hasBase {
|
|
// Combine packages, prioritizing -dmo
|
|
combinedPkg := dmoPkg // Start with the -dmo package
|
|
basePkg.Packages.ForEach(func(pkgName string, pkgInfo domain.PackageInfo) bool {
|
|
if _, exists := combinedPkg.Packages.Get(pkgName); !exists {
|
|
combinedPkg.Packages.Set(pkgName, pkgInfo)
|
|
}
|
|
return true
|
|
})
|
|
packages.Set(dmoName, combinedPkg) // Store under the -dmo name
|
|
packages.Del(baseName) // Remove the base package
|
|
} else {
|
|
// If there's no base package, just add the dmo package back
|
|
packages.Set(dmoName, dmoPkg)
|
|
}
|
|
return true
|
|
})
|
|
}
|