package packages import ( "brunel/config" "brunel/deb" "brunel/domain" "brunel/helpers" "compress/bzip2" "fmt" "io" "log/slog" "net/http" "slices" "strings" "sync" "time" "golang.org/x/sync/errgroup" "pault.ag/go/debian/version" "github.com/alphadose/haxmap" "github.com/klauspost/compress/gzip" "github.com/ulikunitz/xz" ) // Constants for frequently used strings const ( DMOSuffix = "-dmo" DebianInstallerSection = "debian-installer" PackageKey = "Package" SourceKey = "Source" VersionKey = "Version" ArchitectureKey = "Architecture" DescriptionKey = "Description" SectionKey = "Section" ) var LastUpdateTime time.Time var currentPackages = haxmap.New[string, domain.SourcePackage]() func ProcessPackages() error { start := time.Now() internalPackages := haxmap.New[string, domain.SourcePackage]() externalPackages := haxmap.New[string, domain.SourcePackage]() var eg errgroup.Group eg.Go(func() error { return LoadPackages(internalPackages, config.Configs.LocalPackageFiles, true) }) eg.Go(func() error { return LoadPackages(externalPackages, config.Configs.ExternalPackageFiles, false) }) if err := eg.Wait(); err != nil { return err } slog.Info("packages processed in " + time.Since(start).String()) combinePackages(internalPackages) combinePackages(externalPackages) ProcessMissingPackages(internalPackages, externalPackages) ProcessStalePackages(internalPackages, externalPackages) updatedPackages := haxmap.New[string, domain.SourcePackage]() internalPackages.ForEach(func(k string, v domain.SourcePackage) bool { curr, exists := currentPackages.Get(k) if !exists { updatedPackages.Set(k, v) return true } mergedPackage := domain.SourcePackage{ Name: curr.Name, Has32bit: curr.Has32bit, Version: v.Version, NewVersion: v.NewVersion, Status: v.Status, BuildAttempts: curr.BuildAttempts, LastBuildStatus: curr.LastBuildStatus, Packages: haxmap.New[string, domain.PackageInfo](), } updatedPackages.Set(k, mergedPackage) return true }) updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool { for _, pkg := range config.Configs.I386List { if v.Name == pkg || v.Name == pkg+DMOSuffix { v.Has32bit = true updatedPackages.Set(k, v) return true } } v.Has32bit = false updatedPackages.Set(k, v) return true }) currentPackages = updatedPackages LastUpdateTime = time.Now() if err := helpers.DBInst.DropPackages(); err != nil { return fmt.Errorf("dropping packages: %w", err) } if err := SaveToDb(); err != nil { return fmt.Errorf("saving to database: %w", err) } return nil } func GetPackages() *haxmap.Map[string, domain.SourcePackage] { return currentPackages } func UpdatePackage(pkg domain.PackageInfo) error { curr, ok := currentPackages.Get(pkg.Source) if !ok { return fmt.Errorf("package %s not found", pkg.Source) } curr.Packages.Set(pkg.PackageName, pkg) currentPackages.Set(pkg.Source, curr) return saveSingleToDb(curr) } func UpdateSourcePackage(pkg domain.SourcePackage) error { currentPackages.Set(pkg.Name, pkg) return saveSingleToDb(pkg) } func saveSingleToDb(pkg domain.SourcePackage) error { if err := helpers.DBInst.UpdatePackage(pkg); err != nil { return fmt.Errorf("updating package: %w", err) } LastUpdateTime = time.Now() if err := helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime); err != nil { return fmt.Errorf("updating last update time: %w", err) } return nil } func SaveToDb() error { if err := helpers.DBInst.SavePackages(currentPackages); err != nil { slog.Error("Error saving packages to database", "error", err) return fmt.Errorf("saving packages to database: %w", err) } LastUpdateTime = time.Now() return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) } func LoadFromDb() error { packages, err := helpers.DBInst.GetPackages() if err != nil { slog.Error("Error getting packages from database", "error", err) return nil } slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int { if a.Name == b.Name { return 0 } if a.Name > b.Name { return 1 } return -1 }) currentPackages.Clear() for _, pkg := range packages { currentPackages.Set(pkg.Name, pkg) } return nil } func LoadPackages(packages *haxmap.Map[string, domain.SourcePackage], packageFiles []config.PackageFile, isInternal bool) error { slices.SortStableFunc(packageFiles, func(a, b config.PackageFile) int { if a.Priority == b.Priority { return 0 } if (isInternal && a.Priority < b.Priority) || (!isInternal && a.Priority > b.Priority) { return 1 } return -1 }) var eg errgroup.Group var mu sync.Mutex packageResults := make([][]fetchResult, len(packageFiles)) for i, pkg := range packageFiles { eg.Go(func() error { results, err := fetchPackagesForFile(pkg) if err != nil { return err } mu.Lock() packageResults[i] = results mu.Unlock() return nil }) } if err := eg.Wait(); err != nil { return fmt.Errorf("fetching package files: %w", err) } for _, results := range packageResults { for _, result := range results { processPackageResult(packages, result) } } return nil } type fetchResult struct { repo string packages *haxmap.Map[string, domain.PackageInfo] } func fetchPackagesForFile(pkg config.PackageFile) ([]fetchResult, error) { var results []fetchResult for _, repo := range pkg.Subrepos { packages, err := fetchPackageFile(pkg, repo) if err != nil { return nil, fmt.Errorf("fetching package file for repo %s: %w", repo, err) } results = append(results, fetchResult{repo: repo, packages: packages}) } return results, nil } func processPackageResult(packages *haxmap.Map[string, domain.SourcePackage], result fetchResult) { result.packages.ForEach(func(newKey string, newPkg domain.PackageInfo) bool { pk, ok := packages.Get(newPkg.Source) if !ok { newMap := haxmap.New[string, domain.PackageInfo]() newMap.Set(newKey, newPkg) packages.Set(newPkg.Source, domain.SourcePackage{ Name: newPkg.Source, Packages: newMap, Status: domain.Current, Version: newPkg.Version, }) return true } pk.Version = getHighestVer(pk.Version, newPkg.Version) pkg, ok := pk.Packages.Get(newKey) if !ok { pk.Packages.Set(newKey, newPkg) } else { mVer, _ := version.Parse(pkg.Version) extVer, _ := version.Parse(strings.Split(newPkg.Version, "+b")[0]) cmpVal := version.Compare(extVer, mVer) if cmpVal >= 0 { pk.Packages.Set(newKey, newPkg) } } packages.Set(newPkg.Source, pk) return true }) } func ProcessMissingPackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) { externalPackages.ForEach(func(k string, src domain.SourcePackage) bool { _, ok := internalPackages.Get(k) if !ok && src.Packages.Len() > 0 { src.Status = domain.Missing internalPackages.Set(k, src) } return true }) } func ProcessStalePackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) { externalPackages.ForEach(func(newPackage string, newSource domain.SourcePackage) bool { matchedPackage, ok := internalPackages.Get(newPackage) if !ok || matchedPackage.Packages.Len() == 0 { return true } ver := getHighestVer(newSource.Version, matchedPackage.Version) if ver != matchedPackage.Version { matchedPackage.NewVersion = ver matchedPackage.Status = domain.Stale internalPackages.Set(newPackage, matchedPackage) } return true }) } func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[string, domain.PackageInfo], error) { resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression) if err != nil { return nil, fmt.Errorf("fetching package file: %w", err) } defer resp.Body.Close() rdr, err := getCompressedReader(resp.Body, pkg.Compression) if err != nil { return nil, fmt.Errorf("creating compressed reader: %w", err) } packages := haxmap.New[string, domain.PackageInfo]() sreader := deb.NewControlFileReader(rdr, false, false) for { stanza, err := sreader.ReadStanza() if err != nil || stanza == nil { break } if stanza[SectionKey] == DebianInstallerSection { continue } name := stanza[PackageKey] if shouldSkipPackage(name, pkg) { continue } source, sourceVersion := parseSource(stanza[SourceKey]) if source == "" { source = name } versionStr := chooseVersion(sourceVersion, stanza[VersionKey]) ver, err := version.Parse(versionStr) if err != nil { return nil, fmt.Errorf("parsing version %s: %w", versionStr, err) } if shouldUpdatePackage(packages, name, ver) { packages.Set(name, domain.PackageInfo{ PackageName: name, Version: ver.String(), Source: source, Architecture: stanza[ArchitectureKey], Description: stanza[DescriptionKey], Status: domain.Current, }) } } return packages, nil } func getCompressedReader(body io.Reader, compression string) (io.Reader, error) { switch compression { case "bz2": return bzip2.NewReader(body), nil case "xz": return xz.NewReader(body) case "gz": return gzip.NewReader(body) default: return body, nil } } func shouldSkipPackage(name string, pkg config.PackageFile) bool { if pkg.UseWhitelist && len(pkg.Whitelist) > 0 && !nameContains(name, pkg.Whitelist) { return true } return nameContains(name, pkg.Blacklist) } func parseSource(source string) (string, string) { sourceSplit := strings.Split(source, " ") if len(sourceSplit) > 1 { return sourceSplit[0], strings.Trim(sourceSplit[1], "()") } return source, "" } func chooseVersion(sourceVersion, stanzaVersion string) string { if sourceVersion != "" { return sourceVersion } return stanzaVersion } func shouldUpdatePackage(packages *haxmap.Map[string, domain.PackageInfo], name string, ver version.Version) bool { existingPkg, exists := packages.Get(name) if !exists { return true } existingVer, _ := version.Parse(existingPkg.Version) return version.Compare(ver, existingVer) >= 0 } func nameContains(name string, match []string) bool { for _, m := range match { if strings.Contains(name, m) { return true } } return false } func GetPackagesCount() domain.PackagesCount { count := domain.PackagesCount{ Stale: 0, Missing: 0, Current: 0, Error: 0, Queued: 0, Building: 0, } currentPackages.ForEach(func(k string, pkg domain.SourcePackage) bool { switch pkg.Status { case domain.Stale: count.Stale++ case domain.Missing: count.Missing++ case domain.Built: count.Current++ case domain.Current: count.Current++ case domain.Error: count.Error++ } return true }) return count } func combinePackages(packages *haxmap.Map[string, domain.SourcePackage]) { dmoPackages := haxmap.New[string, domain.SourcePackage]() // Identify and collect -dmo packages packages.ForEach(func(k string, v domain.SourcePackage) bool { if strings.HasSuffix(k, DMOSuffix) { dmoPackages.Set(k, v) packages.Del(k) } return true }) // Combine -dmo packages with base packages or add them back dmoPackages.ForEach(func(dmoName string, dmoPkg domain.SourcePackage) bool { baseName := strings.TrimSuffix(dmoName, DMOSuffix) if basePkg, hasBase := packages.Get(baseName); hasBase { combinedPkg := combineDMOPackages(dmoPkg, basePkg) packages.Set(dmoName, combinedPkg) packages.Del(baseName) } else { packages.Set(dmoName, dmoPkg) } return true }) } func combineDMOPackages(dmoPkg, basePkg domain.SourcePackage) domain.SourcePackage { combinedPkg := dmoPkg basePkg.Packages.ForEach(func(pkgName string, pkgInfo domain.PackageInfo) bool { if _, exists := combinedPkg.Packages.Get(pkgName); !exists { combinedPkg.Packages.Set(pkgName, pkgInfo) } return true }) return combinedPkg } func getHighestVer(ver string, newVer string) string { mVer, _ := version.Parse(ver) extVer, _ := version.Parse(newVer) cmpVal := version.Compare(mVer, extVer) if cmpVal < 0 { return newVer } return ver }