package packages import ( "brunel/config" "brunel/deb" "brunel/domain" "brunel/fastmap" "brunel/helpers" "compress/bzip2" "fmt" "io" "log/slog" "net/http" "slices" "strings" "time" "pault.ag/go/debian/version" "github.com/klauspost/compress/gzip" "github.com/ulikunitz/xz" ) var LastUpdateTime time.Time var currentPackagesFastMap = fastmap.New[string, domain.SourcePackage]() var buildQueue domain.BuildQueue = fastmap.New[string, domain.BuildQueueItem]() func ProcessPackages() error { var internalPackages = fastmap.New[string, domain.SourcePackage]() var externalPackages = fastmap.New[string, domain.SourcePackage]() err := LoadInternalPackages(internalPackages) if err != nil { return err } err = LoadExternalPackages(externalPackages) if err != nil { return err } ProcessStalePackages(internalPackages, externalPackages) ProcessMissingPackages(internalPackages, externalPackages) currentPackagesFastMap.Clear() internalPackages.Iter(func(k string, v domain.SourcePackage) bool { currentPackagesFastMap.Set(k, v) return true }) currentPackagesFastMap.StableSortByKey() LastUpdateTime = time.Now() helpers.ReloadCache() err = SaveToDb() if err != nil { return err } return nil } func GetBuildQueue() domain.BuildQueue { return buildQueue } func GetPackages() *fastmap.Fastmap[string, domain.SourcePackage] { return currentPackagesFastMap } func UpdatePackage(pkg domain.PackageInfo) error { curr, ok := currentPackagesFastMap.Get(pkg.Source) if !ok { return fmt.Errorf("package %s not found", pkg.Source) } curr.Packages.Set(pkg.PackageName, pkg) currentPackagesFastMap.Set(pkg.Source, curr) return saveSingleToDb(curr) } func IsBuilt(pkg domain.PackageInfo) bool { curr, ok := currentPackagesFastMap.Get(pkg.Source) if !ok { return false } pk, ok := curr.Packages.Get(pkg.PackageName) if !ok { return false } return pk.Status == domain.Built || pk.Status == domain.Current } func saveSingleToDb(pkg domain.SourcePackage) error { err := helpers.DBInst.UpdatePackage(pkg) if err != nil { return err } LastUpdateTime = time.Now() helpers.ReloadCache() err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) if err != nil { return err } return nil } func SaveToDb() error { err := helpers.DBInst.SavePackages(currentPackagesFastMap) if err != nil { slog.Error(err.Error()) return err } LastUpdateTime = time.Now() helpers.ReloadCache() return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) } func LoadFromDb() error { packages, err := helpers.DBInst.GetPackages() if err != nil { slog.Error(err.Error()) return nil } slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int { if a.Name == b.Name { return 0 } if a.Name > b.Name { return 1 } return -1 }) currentPackagesFastMap.Clear() for _, pkg := range packages { currentPackagesFastMap.Set(pkg.Name, pkg) } currentPackagesFastMap.StableSortByKey() return nil } func LoadInternalPackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage]) error { localPackageFile := config.Configs.LocalPackageFiles slices.SortStableFunc(localPackageFile, func(a, b config.PackageFile) int { if a.Priority == b.Priority { return 0 } if a.Priority < b.Priority { return 1 } return -1 }) for _, pkg := range config.Configs.LocalPackageFiles { for _, repo := range pkg.Subrepos { packages, err := fetchPackageFile(pkg, repo) if err != nil { return err } packages.Iter(func(newKey string, newPkg domain.PackageInfo) bool { pk, ok := internalPackages.Get(newPkg.Source) if !ok { newMap := fastmap.New[string, domain.PackageInfo]() newMap.Set(newKey, newPkg) internalPackages.Set(newPkg.Source, domain.SourcePackage{ Name: newPkg.Source, Packages: newMap, }) return true } pkg, ok := pk.Packages.Get(newKey) if !ok { pk.Packages.Set(newKey, newPkg) return true } mVer, _ := version.Parse(pkg.Version) extVer, _ := version.Parse(newPkg.Version) cmpVal := version.Compare(extVer, mVer) if cmpVal >= 0 { pk.Packages.Set(newKey, newPkg) return true } return true }) } } return nil } func LoadExternalPackages(externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) error { externalPackageFile := config.Configs.ExternalPackageFiles slices.SortStableFunc(externalPackageFile, func(a, b config.PackageFile) int { if a.Priority == b.Priority { return 0 } if a.Priority < b.Priority { return -1 } return 1 }) for _, pkg := range config.Configs.ExternalPackageFiles { for _, repo := range pkg.Subrepos { packages, err := fetchPackageFile(pkg, repo) if err != nil { return err } packages.Iter(func(k string, v domain.PackageInfo) bool { pk, ok := externalPackages.Get(v.Source) if !ok { newMap := fastmap.New[string, domain.PackageInfo]() newMap.Set(k, v) externalPackages.Set(v.Source, domain.SourcePackage{ Name: v.Source, Packages: newMap, }) return true } pkg, ok := pk.Packages.Get(k) if !ok { pk.Packages.Set(k, v) return true } mVer, _ := version.Parse(pkg.Version) extVer, _ := version.Parse(v.Version) cmpVal := version.Compare(extVer, mVer) if cmpVal >= 0 { pk.Packages.Set(k, v) return true } return true }) } } return nil } func ProcessMissingPackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage], externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) { externalPackages.Iter(func(k string, src domain.SourcePackage) bool { _, ok := internalPackages.Get(k) if !ok && src.Packages.Len() > 0 { newStatus := domain.Missing src.Packages.Iter(func(k string, v domain.PackageInfo) bool { v.Status = newStatus src.Packages.Set(k, v) return true }) internalPackages.Set(k, src) } return true }) } func ProcessStalePackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage], externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) { externalPackages.Iter(func(newPackage string, newSource domain.SourcePackage) bool { matchedPackage, ok := internalPackages.Get(newPackage) if !ok || matchedPackage.Packages.Len() == 0 { return true } matchedPackage.Packages.Iter(func(currentKey string, currentPackage domain.PackageInfo) bool { if currentPackage.Status == domain.Missing { return true } newSource.Packages.Iter(func(newKey string, newPackage domain.PackageInfo) bool { if currentKey != newKey { return true } newVersion := strings.Split(newPackage.Version, "+b")[0] mVer, _ := version.Parse(currentPackage.Version) extVer, _ := version.Parse(newVersion) cmpVal := version.Compare(mVer, extVer) if cmpVal < 0 { currentPackage.Status = domain.Stale currentPackage.NewVersion = extVer.String() matchedPackage.Packages.Set(currentKey, currentPackage) } return false }) return true }) wasMissing := false newSource.Packages.Iter(func(newKey string, newPackage domain.PackageInfo) bool { found := false matchedPackage.Packages.Iter(func(currentKey string, currentPackage domain.PackageInfo) bool { if currentKey != newKey { return true } found = true return false }) if !found { wasMissing = true newPackage.Status = domain.Missing newPackage.NewVersion = newPackage.Version matchedPackage.Packages.Set(newKey, newPackage) } return true }) if wasMissing { matchedPackage.Packages.Iter(func(k string, v domain.PackageInfo) bool { if v.Status == domain.Missing { return true } v.Status = domain.Missing return true }) } return true }) } func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*fastmap.Fastmap[string, domain.PackageInfo], error) { resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression) if err != nil { return nil, err } defer resp.Body.Close() rdr := io.Reader(resp.Body) if pkg.Compression == "bz2" { r := bzip2.NewReader(resp.Body) rdr = r } if pkg.Compression == "xz" { r, err := xz.NewReader(resp.Body) if err != nil { return nil, err } rdr = r } if pkg.Compression == "gz" { r, err := gzip.NewReader(resp.Body) if err != nil { return nil, err } rdr = r } packages := fastmap.New[string, domain.PackageInfo]() sreader := deb.NewControlFileReader(rdr, false, false) for { stanza, err := sreader.ReadStanza() if err != nil || stanza == nil { break } if stanza["Section"] == "debian-installer" { continue } name := stanza["Package"] useWhitelist := pkg.UseWhitelist && len(pkg.Whitelist) > 0 if useWhitelist { contained := nameContains(name, pkg.Whitelist) if !contained { continue } } broken := nameContains(name, pkg.Blacklist) if broken { continue } ver, err := version.Parse(stanza["Version"]) if err != nil { return nil, err } pk, ok := packages.Get(name) if ok { matchedVer, _ := version.Parse(pk.Version) cmpVal := version.Compare(ver, matchedVer) if cmpVal < 0 { continue } } sourceSplit := strings.Split(stanza["Source"], " ") source := sourceSplit[0] if source == "" { source = name } packages.Set(name, domain.PackageInfo{ PackageName: name, Version: ver.String(), Source: source, Architecture: stanza["Architecture"], Description: stanza["Description"], Status: domain.Current, }) } return packages, nil } func nameContains(name string, match []string) bool { for _, m := range match { if strings.Contains(name, m) { return true } } return false } func GetPackagesCount() domain.PackagesCount { count := domain.PackagesCount{ Stale: 0, Missing: 0, Current: 0, Error: 0, Queued: 0, Building: 0, } currentPackagesFastMap.Iter(func(k string, v domain.SourcePackage) bool { v.Packages.Iter(func(k string, pkg domain.PackageInfo) bool { switch pkg.Status { case domain.Stale: count.Stale++ case domain.Missing: count.Missing++ case domain.Built: count.Current++ case domain.Current: count.Current++ case domain.Error: count.Error++ } if pkg.LastBuildStatus == domain.Error { count.Error++ } return false }) return true }) return count }