brunel/packages/packages.go

536 lines
13 KiB
Go
Raw Normal View History

2024-07-28 20:59:50 +02:00
package packages
import (
"brunel/config"
"brunel/deb"
"brunel/domain"
"brunel/helpers"
"compress/bzip2"
"fmt"
"io"
"log/slog"
"net/http"
"slices"
"strings"
"time"
"pault.ag/go/debian/version"
2024-07-30 02:24:18 +02:00
"github.com/alphadose/haxmap"
2024-07-28 20:59:50 +02:00
"github.com/klauspost/compress/gzip"
"github.com/ulikunitz/xz"
)
var LastUpdateTime time.Time
2024-07-30 02:24:18 +02:00
var currentPackagesFastMap = haxmap.New[string, domain.SourcePackage]()
2024-07-28 20:59:50 +02:00
func ProcessPackages() error {
2024-07-30 02:24:18 +02:00
var internalPackages = haxmap.New[string, domain.SourcePackage]()
var externalPackages = haxmap.New[string, domain.SourcePackage]()
2024-07-28 20:59:50 +02:00
err := LoadInternalPackages(internalPackages)
if err != nil {
return err
}
err = LoadExternalPackages(externalPackages)
if err != nil {
return err
}
2024-07-30 02:24:18 +02:00
combinePackages(internalPackages)
combinePackages(externalPackages)
2024-07-28 20:59:50 +02:00
ProcessStalePackages(internalPackages, externalPackages)
ProcessMissingPackages(internalPackages, externalPackages)
2024-07-30 09:20:32 +02:00
updatedPackages := haxmap.New[string, domain.SourcePackage]()
2024-07-30 02:24:18 +02:00
internalPackages.ForEach(func(k string, v domain.SourcePackage) bool {
2024-07-30 09:20:32 +02:00
curr, exists := currentPackagesFastMap.Get(k)
if !exists {
updatedPackages.Set(k, v)
2024-07-30 09:08:00 +02:00
return true
}
2024-07-30 09:20:32 +02:00
mergedPackage := domain.SourcePackage{
Name: curr.Name,
Packages: haxmap.New[string, domain.PackageInfo](),
}
v.Packages.ForEach(func(pkgName string, newPkg domain.PackageInfo) bool {
if existingPkg, ok := curr.Packages.Get(pkgName); ok {
if newPkg.Version != existingPkg.Version ||
newPkg.Status != existingPkg.Status ||
newPkg.NewVersion != existingPkg.NewVersion {
2024-07-30 09:22:29 +02:00
if newPkg.Status == domain.Current {
mergedPackage.Packages.Set(pkgName, newPkg)
return true
}
newPkg.LastBuildStatus = existingPkg.LastBuildStatus
2024-07-30 09:20:32 +02:00
newPkg.BuildAttempts = existingPkg.BuildAttempts
mergedPackage.Packages.Set(pkgName, newPkg)
} else {
mergedPackage.Packages.Set(pkgName, existingPkg)
2024-07-30 09:08:00 +02:00
}
2024-07-30 09:20:32 +02:00
} else {
mergedPackage.Packages.Set(pkgName, newPkg)
}
2024-07-30 09:08:00 +02:00
return true
})
2024-07-30 09:20:32 +02:00
curr.Packages.ForEach(func(pkgName string, pkg domain.PackageInfo) bool {
if _, exists := mergedPackage.Packages.Get(pkgName); !exists {
mergedPackage.Packages.Set(pkgName, pkg)
}
return true
})
updatedPackages.Set(k, mergedPackage)
2024-07-28 20:59:50 +02:00
return true
})
2024-07-30 09:20:32 +02:00
currentPackagesFastMap = updatedPackages
2024-07-28 20:59:50 +02:00
LastUpdateTime = time.Now()
2024-07-29 17:39:11 +02:00
helpers.ReloadCache()
err = helpers.DBInst.DropPackages()
if err != nil {
return err
}
2024-07-28 20:59:50 +02:00
err = SaveToDb()
if err != nil {
return err
}
return nil
}
2024-07-30 02:24:18 +02:00
func GetPackages() *haxmap.Map[string, domain.SourcePackage] {
2024-07-28 20:59:50 +02:00
return currentPackagesFastMap
}
func UpdatePackage(pkg domain.PackageInfo) error {
curr, ok := currentPackagesFastMap.Get(pkg.Source)
if !ok {
return fmt.Errorf("package %s not found", pkg.Source)
}
curr.Packages.Set(pkg.PackageName, pkg)
currentPackagesFastMap.Set(pkg.Source, curr)
return saveSingleToDb(curr)
}
2024-07-29 23:40:26 +02:00
func UpdateSourcePackage(pkg domain.SourcePackage) error {
currentPackagesFastMap.Set(pkg.Name, pkg)
return saveSingleToDb(pkg)
}
2024-07-28 20:59:50 +02:00
func IsBuilt(pkg domain.PackageInfo) bool {
curr, ok := currentPackagesFastMap.Get(pkg.Source)
if !ok {
return false
}
pk, ok := curr.Packages.Get(pkg.PackageName)
if !ok {
return false
}
return pk.Status == domain.Built || pk.Status == domain.Current
}
func saveSingleToDb(pkg domain.SourcePackage) error {
err := helpers.DBInst.UpdatePackage(pkg)
if err != nil {
return err
}
LastUpdateTime = time.Now()
2024-07-29 17:39:11 +02:00
helpers.ReloadCache()
2024-07-28 20:59:50 +02:00
err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
if err != nil {
return err
}
return nil
}
func SaveToDb() error {
err := helpers.DBInst.SavePackages(currentPackagesFastMap)
if err != nil {
slog.Error(err.Error())
return err
}
LastUpdateTime = time.Now()
2024-07-29 17:39:11 +02:00
helpers.ReloadCache()
2024-07-28 20:59:50 +02:00
return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
}
func LoadFromDb() error {
packages, err := helpers.DBInst.GetPackages()
if err != nil {
slog.Error(err.Error())
return nil
}
slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int {
if a.Name == b.Name {
return 0
}
if a.Name > b.Name {
return 1
}
return -1
})
currentPackagesFastMap.Clear()
for _, pkg := range packages {
currentPackagesFastMap.Set(pkg.Name, pkg)
}
return nil
}
2024-07-30 02:24:18 +02:00
func LoadInternalPackages(internalPackages *haxmap.Map[string, domain.SourcePackage]) error {
2024-07-28 20:59:50 +02:00
localPackageFile := config.Configs.LocalPackageFiles
slices.SortStableFunc(localPackageFile, func(a, b config.PackageFile) int {
if a.Priority == b.Priority {
return 0
}
if a.Priority < b.Priority {
return 1
}
return -1
})
for _, pkg := range config.Configs.LocalPackageFiles {
for _, repo := range pkg.Subrepos {
packages, err := fetchPackageFile(pkg, repo)
if err != nil {
return err
}
2024-07-30 02:24:18 +02:00
packages.ForEach(func(newKey string, newPkg domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
pk, ok := internalPackages.Get(newPkg.Source)
if !ok {
2024-07-30 02:24:18 +02:00
newMap := haxmap.New[string, domain.PackageInfo]()
2024-07-28 20:59:50 +02:00
newMap.Set(newKey, newPkg)
internalPackages.Set(newPkg.Source, domain.SourcePackage{
Name: newPkg.Source,
Packages: newMap,
})
return true
}
pkg, ok := pk.Packages.Get(newKey)
if !ok {
pk.Packages.Set(newKey, newPkg)
return true
}
mVer, _ := version.Parse(pkg.Version)
extVer, _ := version.Parse(newPkg.Version)
cmpVal := version.Compare(extVer, mVer)
if cmpVal >= 0 {
pk.Packages.Set(newKey, newPkg)
return true
}
return true
})
}
}
return nil
}
2024-07-30 02:24:18 +02:00
func LoadExternalPackages(externalPackages *haxmap.Map[string, domain.SourcePackage]) error {
2024-07-28 20:59:50 +02:00
externalPackageFile := config.Configs.ExternalPackageFiles
slices.SortStableFunc(externalPackageFile, func(a, b config.PackageFile) int {
if a.Priority == b.Priority {
return 0
}
if a.Priority < b.Priority {
return -1
}
return 1
})
for _, pkg := range config.Configs.ExternalPackageFiles {
for _, repo := range pkg.Subrepos {
packages, err := fetchPackageFile(pkg, repo)
if err != nil {
return err
}
2024-07-30 02:24:18 +02:00
packages.ForEach(func(k string, v domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
pk, ok := externalPackages.Get(v.Source)
if !ok {
2024-07-30 02:24:18 +02:00
newMap := haxmap.New[string, domain.PackageInfo]()
2024-07-28 20:59:50 +02:00
newMap.Set(k, v)
externalPackages.Set(v.Source, domain.SourcePackage{
Name: v.Source,
Packages: newMap,
})
return true
}
pkg, ok := pk.Packages.Get(k)
if !ok {
pk.Packages.Set(k, v)
return true
}
mVer, _ := version.Parse(pkg.Version)
extVer, _ := version.Parse(v.Version)
cmpVal := version.Compare(extVer, mVer)
if cmpVal >= 0 {
pk.Packages.Set(k, v)
return true
}
return true
})
}
}
return nil
}
2024-07-30 02:24:18 +02:00
func ProcessMissingPackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
externalPackages.ForEach(func(k string, src domain.SourcePackage) bool {
2024-07-28 20:59:50 +02:00
_, ok := internalPackages.Get(k)
if !ok && src.Packages.Len() > 0 {
newStatus := domain.Missing
2024-07-30 02:24:18 +02:00
src.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
v.Status = newStatus
2024-07-30 00:59:30 +02:00
v.Version = strings.Split(v.Version, "+b")[0]
2024-07-28 20:59:50 +02:00
src.Packages.Set(k, v)
return true
})
internalPackages.Set(k, src)
}
return true
})
}
2024-07-30 02:24:18 +02:00
func ProcessStalePackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
externalPackages.ForEach(func(newPackage string, newSource domain.SourcePackage) bool {
2024-07-28 20:59:50 +02:00
matchedPackage, ok := internalPackages.Get(newPackage)
if !ok || matchedPackage.Packages.Len() == 0 {
return true
}
2024-07-30 02:24:18 +02:00
matchedPackage.Packages.ForEach(func(currentKey string, currentPackage domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
if currentPackage.Status == domain.Missing {
return true
}
2024-07-30 02:24:18 +02:00
newSource.Packages.ForEach(func(newKey string, newPackage domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
if currentKey != newKey {
return true
}
newVersion := strings.Split(newPackage.Version, "+b")[0]
mVer, _ := version.Parse(currentPackage.Version)
extVer, _ := version.Parse(newVersion)
cmpVal := version.Compare(mVer, extVer)
if cmpVal < 0 {
2024-07-30 09:08:00 +02:00
if currentPackage.Status != domain.Error {
currentPackage.Status = domain.Stale
}
2024-07-28 20:59:50 +02:00
currentPackage.NewVersion = extVer.String()
matchedPackage.Packages.Set(currentKey, currentPackage)
}
return false
})
return true
})
wasMissing := false
2024-07-30 02:24:18 +02:00
newSource.Packages.ForEach(func(newKey string, newPackage domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
found := false
2024-07-30 02:24:18 +02:00
matchedPackage.Packages.ForEach(func(currentKey string, currentPackage domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
if currentKey != newKey {
return true
}
found = true
return false
})
if !found {
wasMissing = true
newPackage.Status = domain.Missing
newPackage.NewVersion = newPackage.Version
matchedPackage.Packages.Set(newKey, newPackage)
}
return true
})
if wasMissing {
2024-07-30 02:24:18 +02:00
matchedPackage.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
2024-07-30 09:08:00 +02:00
if v.Status == domain.Error {
return true
}
2024-07-28 20:59:50 +02:00
if v.Status == domain.Missing {
return true
}
v.Status = domain.Missing
return true
})
}
return true
})
}
2024-07-30 02:24:18 +02:00
func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[string, domain.PackageInfo], error) {
2024-07-28 20:59:50 +02:00
resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression)
if err != nil {
return nil, err
}
defer resp.Body.Close()
rdr := io.Reader(resp.Body)
if pkg.Compression == "bz2" {
r := bzip2.NewReader(resp.Body)
rdr = r
}
if pkg.Compression == "xz" {
r, err := xz.NewReader(resp.Body)
if err != nil {
return nil, err
}
rdr = r
}
if pkg.Compression == "gz" {
r, err := gzip.NewReader(resp.Body)
if err != nil {
return nil, err
}
rdr = r
}
2024-07-30 02:24:18 +02:00
packages := haxmap.New[string, domain.PackageInfo]()
2024-07-28 20:59:50 +02:00
sreader := deb.NewControlFileReader(rdr, false, false)
for {
stanza, err := sreader.ReadStanza()
if err != nil || stanza == nil {
break
}
if stanza["Section"] == "debian-installer" {
continue
}
name := stanza["Package"]
useWhitelist := pkg.UseWhitelist && len(pkg.Whitelist) > 0
if useWhitelist {
contained := nameContains(name, pkg.Whitelist)
if !contained {
continue
}
}
broken := nameContains(name, pkg.Blacklist)
if broken {
continue
}
sourceSplit := strings.Split(stanza["Source"], " ")
source := sourceSplit[0]
if source == "" {
source = name
}
// Extract version from Source if available
sourceVersion := ""
if len(sourceSplit) > 1 {
sourceVersion = strings.Trim(sourceSplit[1], "()")
}
// Use sourceVersion if available, otherwise use stanza["Version"]
versionStr := stanza["Version"]
if sourceVersion != "" {
versionStr = sourceVersion
}
ver, err := version.Parse(versionStr)
2024-07-28 20:59:50 +02:00
if err != nil {
return nil, err
}
pk, ok := packages.Get(name)
if ok {
matchedVer, _ := version.Parse(pk.Version)
cmpVal := version.Compare(ver, matchedVer)
if cmpVal < 0 {
continue
}
}
packages.Set(name, domain.PackageInfo{
PackageName: name,
Version: ver.String(),
Source: source,
Architecture: stanza["Architecture"],
Description: stanza["Description"],
Status: domain.Current,
})
}
return packages, nil
}
func nameContains(name string, match []string) bool {
for _, m := range match {
if strings.Contains(name, m) {
return true
}
}
return false
}
func GetPackagesCount() domain.PackagesCount {
count := domain.PackagesCount{
Stale: 0,
Missing: 0,
Current: 0,
Error: 0,
Queued: 0,
Building: 0,
}
2024-07-30 02:24:18 +02:00
currentPackagesFastMap.ForEach(func(k string, v domain.SourcePackage) bool {
v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool {
2024-07-28 20:59:50 +02:00
switch pkg.Status {
case domain.Stale:
count.Stale++
case domain.Missing:
count.Missing++
case domain.Built:
count.Current++
case domain.Current:
count.Current++
case domain.Error:
count.Error++
}
return false
})
return true
})
2024-07-29 23:49:20 +02:00
2024-07-28 20:59:50 +02:00
return count
}
2024-07-30 02:24:18 +02:00
func combinePackages(packages *haxmap.Map[string, domain.SourcePackage]) {
dmoPackages := haxmap.New[string, domain.SourcePackage]()
// First pass: identify and collect -dmo packages
packages.ForEach(func(k string, v domain.SourcePackage) bool {
if strings.HasSuffix(k, "-dmo") {
dmoPackages.Set(k, v)
packages.Del(k)
}
return true
})
// Second pass: combine -dmo packages with base packages or add them back
dmoPackages.ForEach(func(dmoName string, dmoPkg domain.SourcePackage) bool {
baseName := strings.TrimSuffix(dmoName, "-dmo")
basePkg, hasBase := packages.Get(baseName)
if hasBase {
// Combine packages, prioritizing -dmo
combinedPkg := dmoPkg // Start with the -dmo package
basePkg.Packages.ForEach(func(pkgName string, pkgInfo domain.PackageInfo) bool {
if _, exists := combinedPkg.Packages.Get(pkgName); !exists {
combinedPkg.Packages.Set(pkgName, pkgInfo)
}
return true
})
packages.Set(dmoName, combinedPkg) // Store under the -dmo name
packages.Del(baseName) // Remove the base package
} else {
// If there's no base package, just add the dmo package back
packages.Set(dmoName, dmoPkg)
}
return true
})
}