package packages import ( "brunel/config" "brunel/deb" "brunel/domain" "brunel/helpers" "compress/bzip2" "fmt" "io" "log/slog" "net/http" "slices" "strings" "time" "pault.ag/go/debian/version" "github.com/alphadose/haxmap" "github.com/klauspost/compress/gzip" "github.com/ulikunitz/xz" ) var LastUpdateTime time.Time var currentPackagesFastMap = haxmap.New[string, domain.SourcePackage]() func ProcessPackages() error { var internalPackages = haxmap.New[string, domain.SourcePackage]() var externalPackages = haxmap.New[string, domain.SourcePackage]() err := LoadInternalPackages(internalPackages) if err != nil { return err } err = LoadExternalPackages(externalPackages) if err != nil { return err } combinePackages(internalPackages) combinePackages(externalPackages) ProcessStalePackages(internalPackages, externalPackages) ProcessMissingPackages(internalPackages, externalPackages) updatedPackages := haxmap.New[string, domain.SourcePackage]() internalPackages.ForEach(func(k string, v domain.SourcePackage) bool { curr, exists := currentPackagesFastMap.Get(k) if !exists { updatedPackages.Set(k, v) return true } mergedPackage := domain.SourcePackage{ Name: curr.Name, Packages: haxmap.New[string, domain.PackageInfo](), } v.Packages.ForEach(func(pkgName string, newPkg domain.PackageInfo) bool { if existingPkg, ok := curr.Packages.Get(pkgName); ok { if newPkg.Version != existingPkg.Version || newPkg.Status != existingPkg.Status || newPkg.NewVersion != existingPkg.NewVersion { if newPkg.Status == domain.Current { mergedPackage.Packages.Set(pkgName, newPkg) return true } newPkg.LastBuildStatus = existingPkg.LastBuildStatus newPkg.BuildAttempts = existingPkg.BuildAttempts mergedPackage.Packages.Set(pkgName, newPkg) } else { mergedPackage.Packages.Set(pkgName, existingPkg) } } else { mergedPackage.Packages.Set(pkgName, newPkg) } return true }) curr.Packages.ForEach(func(pkgName string, pkg domain.PackageInfo) bool { if _, exists := mergedPackage.Packages.Get(pkgName); !exists { mergedPackage.Packages.Set(pkgName, pkg) } return true }) updatedPackages.Set(k, mergedPackage) return true }) currentPackagesFastMap = updatedPackages LastUpdateTime = time.Now() helpers.ReloadCache() err = SaveToDb() if err != nil { return err } return nil } func GetPackages() *haxmap.Map[string, domain.SourcePackage] { return currentPackagesFastMap } func UpdatePackage(pkg domain.PackageInfo) error { curr, ok := currentPackagesFastMap.Get(pkg.Source) if !ok { return fmt.Errorf("package %s not found", pkg.Source) } curr.Packages.Set(pkg.PackageName, pkg) currentPackagesFastMap.Set(pkg.Source, curr) return saveSingleToDb(curr) } func UpdateSourcePackage(pkg domain.SourcePackage) error { currentPackagesFastMap.Set(pkg.Name, pkg) return saveSingleToDb(pkg) } func IsBuilt(pkg domain.PackageInfo) bool { curr, ok := currentPackagesFastMap.Get(pkg.Source) if !ok { return false } pk, ok := curr.Packages.Get(pkg.PackageName) if !ok { return false } return pk.Status == domain.Built || pk.Status == domain.Current } func saveSingleToDb(pkg domain.SourcePackage) error { err := helpers.DBInst.UpdatePackage(pkg) if err != nil { return err } LastUpdateTime = time.Now() helpers.ReloadCache() err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) if err != nil { return err } return nil } func SaveToDb() error { err := helpers.DBInst.SavePackages(currentPackagesFastMap) if err != nil { slog.Error(err.Error()) return err } LastUpdateTime = time.Now() helpers.ReloadCache() return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) } func LoadFromDb() error { packages, err := helpers.DBInst.GetPackages() if err != nil { slog.Error(err.Error()) return nil } slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int { if a.Name == b.Name { return 0 } if a.Name > b.Name { return 1 } return -1 }) currentPackagesFastMap.Clear() for _, pkg := range packages { currentPackagesFastMap.Set(pkg.Name, pkg) } return nil } func LoadInternalPackages(internalPackages *haxmap.Map[string, domain.SourcePackage]) error { localPackageFile := config.Configs.LocalPackageFiles slices.SortStableFunc(localPackageFile, func(a, b config.PackageFile) int { if a.Priority == b.Priority { return 0 } if a.Priority < b.Priority { return 1 } return -1 }) for _, pkg := range config.Configs.LocalPackageFiles { for _, repo := range pkg.Subrepos { packages, err := fetchPackageFile(pkg, repo) if err != nil { return err } packages.ForEach(func(newKey string, newPkg domain.PackageInfo) bool { pk, ok := internalPackages.Get(newPkg.Source) if !ok { newMap := haxmap.New[string, domain.PackageInfo]() newMap.Set(newKey, newPkg) internalPackages.Set(newPkg.Source, domain.SourcePackage{ Name: newPkg.Source, Packages: newMap, }) return true } pkg, ok := pk.Packages.Get(newKey) if !ok { pk.Packages.Set(newKey, newPkg) return true } mVer, _ := version.Parse(pkg.Version) extVer, _ := version.Parse(newPkg.Version) cmpVal := version.Compare(extVer, mVer) if cmpVal >= 0 { pk.Packages.Set(newKey, newPkg) return true } return true }) } } return nil } func LoadExternalPackages(externalPackages *haxmap.Map[string, domain.SourcePackage]) error { externalPackageFile := config.Configs.ExternalPackageFiles slices.SortStableFunc(externalPackageFile, func(a, b config.PackageFile) int { if a.Priority == b.Priority { return 0 } if a.Priority < b.Priority { return -1 } return 1 }) for _, pkg := range config.Configs.ExternalPackageFiles { for _, repo := range pkg.Subrepos { packages, err := fetchPackageFile(pkg, repo) if err != nil { return err } packages.ForEach(func(k string, v domain.PackageInfo) bool { pk, ok := externalPackages.Get(v.Source) if !ok { newMap := haxmap.New[string, domain.PackageInfo]() newMap.Set(k, v) externalPackages.Set(v.Source, domain.SourcePackage{ Name: v.Source, Packages: newMap, }) return true } pkg, ok := pk.Packages.Get(k) if !ok { pk.Packages.Set(k, v) return true } mVer, _ := version.Parse(pkg.Version) extVer, _ := version.Parse(v.Version) cmpVal := version.Compare(extVer, mVer) if cmpVal >= 0 { pk.Packages.Set(k, v) return true } return true }) } } return nil } func ProcessMissingPackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) { externalPackages.ForEach(func(k string, src domain.SourcePackage) bool { _, ok := internalPackages.Get(k) if !ok && src.Packages.Len() > 0 { newStatus := domain.Missing src.Packages.ForEach(func(k string, v domain.PackageInfo) bool { v.Status = newStatus v.Version = strings.Split(v.Version, "+b")[0] src.Packages.Set(k, v) return true }) internalPackages.Set(k, src) } return true }) } func ProcessStalePackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) { externalPackages.ForEach(func(newPackage string, newSource domain.SourcePackage) bool { matchedPackage, ok := internalPackages.Get(newPackage) if !ok || matchedPackage.Packages.Len() == 0 { return true } matchedPackage.Packages.ForEach(func(currentKey string, currentPackage domain.PackageInfo) bool { if currentPackage.Status == domain.Missing { return true } newSource.Packages.ForEach(func(newKey string, newPackage domain.PackageInfo) bool { if currentKey != newKey { return true } newVersion := strings.Split(newPackage.Version, "+b")[0] mVer, _ := version.Parse(currentPackage.Version) extVer, _ := version.Parse(newVersion) cmpVal := version.Compare(mVer, extVer) if cmpVal < 0 { if currentPackage.Status != domain.Error { currentPackage.Status = domain.Stale } currentPackage.NewVersion = extVer.String() matchedPackage.Packages.Set(currentKey, currentPackage) } return false }) return true }) wasMissing := false newSource.Packages.ForEach(func(newKey string, newPackage domain.PackageInfo) bool { found := false matchedPackage.Packages.ForEach(func(currentKey string, currentPackage domain.PackageInfo) bool { if currentKey != newKey { return true } found = true return false }) if !found { wasMissing = true newPackage.Status = domain.Missing newPackage.NewVersion = newPackage.Version matchedPackage.Packages.Set(newKey, newPackage) } return true }) if wasMissing { matchedPackage.Packages.ForEach(func(k string, v domain.PackageInfo) bool { if v.Status == domain.Error { return true } if v.Status == domain.Missing { return true } v.Status = domain.Missing return true }) } return true }) } func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[string, domain.PackageInfo], error) { resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression) if err != nil { return nil, err } defer resp.Body.Close() rdr := io.Reader(resp.Body) if pkg.Compression == "bz2" { r := bzip2.NewReader(resp.Body) rdr = r } if pkg.Compression == "xz" { r, err := xz.NewReader(resp.Body) if err != nil { return nil, err } rdr = r } if pkg.Compression == "gz" { r, err := gzip.NewReader(resp.Body) if err != nil { return nil, err } rdr = r } packages := haxmap.New[string, domain.PackageInfo]() sreader := deb.NewControlFileReader(rdr, false, false) for { stanza, err := sreader.ReadStanza() if err != nil || stanza == nil { break } if stanza["Section"] == "debian-installer" { continue } name := stanza["Package"] useWhitelist := pkg.UseWhitelist && len(pkg.Whitelist) > 0 if useWhitelist { contained := nameContains(name, pkg.Whitelist) if !contained { continue } } broken := nameContains(name, pkg.Blacklist) if broken { continue } sourceSplit := strings.Split(stanza["Source"], " ") source := sourceSplit[0] if source == "" { source = name } // Extract version from Source if available sourceVersion := "" if len(sourceSplit) > 1 { sourceVersion = strings.Trim(sourceSplit[1], "()") } // Use sourceVersion if available, otherwise use stanza["Version"] versionStr := stanza["Version"] if sourceVersion != "" { versionStr = sourceVersion } ver, err := version.Parse(versionStr) if err != nil { return nil, err } pk, ok := packages.Get(name) if ok { matchedVer, _ := version.Parse(pk.Version) cmpVal := version.Compare(ver, matchedVer) if cmpVal < 0 { continue } } packages.Set(name, domain.PackageInfo{ PackageName: name, Version: ver.String(), Source: source, Architecture: stanza["Architecture"], Description: stanza["Description"], Status: domain.Current, }) } return packages, nil } func nameContains(name string, match []string) bool { for _, m := range match { if strings.Contains(name, m) { return true } } return false } func GetPackagesCount() domain.PackagesCount { count := domain.PackagesCount{ Stale: 0, Missing: 0, Current: 0, Error: 0, Queued: 0, Building: 0, } currentPackagesFastMap.ForEach(func(k string, v domain.SourcePackage) bool { v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool { switch pkg.Status { case domain.Stale: count.Stale++ case domain.Missing: count.Missing++ case domain.Built: count.Current++ case domain.Current: count.Current++ case domain.Error: count.Error++ } return false }) return true }) return count } func combinePackages(packages *haxmap.Map[string, domain.SourcePackage]) { dmoPackages := haxmap.New[string, domain.SourcePackage]() // First pass: identify and collect -dmo packages packages.ForEach(func(k string, v domain.SourcePackage) bool { if strings.HasSuffix(k, "-dmo") { dmoPackages.Set(k, v) packages.Del(k) } return true }) // Second pass: combine -dmo packages with base packages or add them back dmoPackages.ForEach(func(dmoName string, dmoPkg domain.SourcePackage) bool { baseName := strings.TrimSuffix(dmoName, "-dmo") basePkg, hasBase := packages.Get(baseName) if hasBase { // Combine packages, prioritizing -dmo combinedPkg := dmoPkg // Start with the -dmo package basePkg.Packages.ForEach(func(pkgName string, pkgInfo domain.PackageInfo) bool { if _, exists := combinedPkg.Packages.Get(pkgName); !exists { combinedPkg.Packages.Set(pkgName, pkgInfo) } return true }) packages.Set(dmoName, combinedPkg) // Store under the -dmo name packages.Del(baseName) // Remove the base package } else { // If there's no base package, just add the dmo package back packages.Set(dmoName, dmoPkg) } return true }) }