brunel/packages/packages.go

package packages

import (
	"brunel/config"
	"brunel/deb"
	"brunel/domain"
	"brunel/helpers"
	"compress/bzip2"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"slices"
	"strings"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"
	"pault.ag/go/debian/version"

	"github.com/alphadose/haxmap"
	"github.com/klauspost/compress/gzip"
	"github.com/ulikunitz/xz"
)

// Constants for frequently used strings
const (
	DMOSuffix              = "-dmo"
	DebianInstallerSection = "debian-installer"
	PackageKey             = "Package"
	SourceKey              = "Source"
	VersionKey             = "Version"
	ArchitectureKey        = "Architecture"
	DescriptionKey         = "Description"
	SectionKey             = "Section"
)

var LastUpdateTime time.Time
var currentPackages = haxmap.New[string, domain.SourcePackage]()

func ProcessPackages() error {
	start := time.Now()
	internalPackages := haxmap.New[string, domain.SourcePackage]()
	externalPackages := haxmap.New[string, domain.SourcePackage]()

	var eg errgroup.Group
	eg.Go(func() error {
		return LoadPackages(internalPackages, config.Configs.LocalPackageFiles, true)
	})

	eg.Go(func() error {
		return LoadPackages(externalPackages, config.Configs.ExternalPackageFiles, false)
	})

	if err := eg.Wait(); err != nil {
		return err
	}
	slog.Info("packages processed in " + time.Since(start).String())

	combinePackages(internalPackages)
	combinePackages(externalPackages)

	ProcessMissingPackages(internalPackages, externalPackages)
	ProcessStalePackages(internalPackages, externalPackages)

	updatedPackages := haxmap.New[string, domain.SourcePackage]()

	internalPackages.ForEach(func(k string, v domain.SourcePackage) bool {
		curr, exists := currentPackages.Get(k)
		if !exists {
			updatedPackages.Set(k, v)
			return true
		}

		mergedPackage := domain.SourcePackage{
			Name:            curr.Name,
			Has32bit:        curr.Has32bit,
			Version:         v.Version,
			NewVersion:      v.NewVersion,
			Status:          v.Status,
			BuildAttempts:   curr.BuildAttempts,
			LastBuildStatus: curr.LastBuildStatus,
			Packages:        haxmap.New[string, domain.PackageInfo](),
		}

		updatedPackages.Set(k, mergedPackage)
		return true
	})

	updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool {
		for _, pkg := range config.Configs.I386List {
			if v.Name == pkg || v.Name == pkg+DMOSuffix {
				v.Has32bit = true
				updatedPackages.Set(k, v)
				return true
			}
		}
		v.Has32bit = false
		updatedPackages.Set(k, v)
		return true
	})

	currentPackages = updatedPackages

	LastUpdateTime = time.Now()
	if err := helpers.DBInst.DropPackages(); err != nil {
		return fmt.Errorf("dropping packages: %w", err)
	}
	if err := SaveToDb(); err != nil {
		return fmt.Errorf("saving to database: %w", err)
	}

	return nil
}

func GetPackages() *haxmap.Map[string, domain.SourcePackage] {
	return currentPackages
}

func UpdatePackage(pkg domain.PackageInfo) error {
	curr, ok := currentPackages.Get(pkg.Source)
	if !ok {
		return fmt.Errorf("package %s not found", pkg.Source)
	}
	curr.Packages.Set(pkg.PackageName, pkg)
	currentPackages.Set(pkg.Source, curr)
	return saveSingleToDb(curr)
}

func UpdateSourcePackage(pkg domain.SourcePackage) error {
	currentPackages.Set(pkg.Name, pkg)
	return saveSingleToDb(pkg)
}

func saveSingleToDb(pkg domain.SourcePackage) error {
	if err := helpers.DBInst.UpdatePackage(pkg); err != nil {
		return fmt.Errorf("updating package: %w", err)
	}
	LastUpdateTime = time.Now()
	if err := helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime); err != nil {
		return fmt.Errorf("updating last update time: %w", err)
	}
	return nil
}

func SaveToDb() error {
	if err := helpers.DBInst.SavePackages(currentPackages); err != nil {
		slog.Error("Error saving packages to database", "error", err)
		return fmt.Errorf("saving packages to database: %w", err)
	}
	LastUpdateTime = time.Now()
	return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
}

func LoadFromDb() error {
	packages, err := helpers.DBInst.GetPackages()
	if err != nil {
		slog.Error("Error getting packages from database", "error", err)
		return nil
	}
	slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int {
		if a.Name == b.Name {
			return 0
		}
		if a.Name > b.Name {
			return 1
		}
		return -1
	})
	currentPackages.Clear()
	for _, pkg := range packages {
		currentPackages.Set(pkg.Name, pkg)
	}
	return nil
}

func LoadPackages(packages *haxmap.Map[string, domain.SourcePackage], packageFiles []config.PackageFile, isInternal bool) error {
	slices.SortStableFunc(packageFiles, func(a, b config.PackageFile) int {
		if a.Priority == b.Priority {
			return 0
		}
		if (isInternal && a.Priority < b.Priority) || (!isInternal && a.Priority > b.Priority) {
			return 1
		}
		return -1
	})

	var eg errgroup.Group
	var mu sync.Mutex
	packageResults := make([][]fetchResult, len(packageFiles))

	for i, pkg := range packageFiles {
		eg.Go(func() error {
			results, err := fetchPackagesForFile(pkg)
			if err != nil {
				return err
			}
			mu.Lock()
			packageResults[i] = results
			mu.Unlock()
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return fmt.Errorf("fetching package files: %w", err)
	}

	for _, results := range packageResults {
		for _, result := range results {
			processPackageResult(packages, result)
		}
	}

	return nil
}

type fetchResult struct {
	repo     string
	packages *haxmap.Map[string, domain.PackageInfo]
}

func fetchPackagesForFile(pkg config.PackageFile) ([]fetchResult, error) {
	var results []fetchResult
	for _, repo := range pkg.Subrepos {
		packages, err := fetchPackageFile(pkg, repo)
		if err != nil {
			return nil, fmt.Errorf("fetching package file for repo %s: %w", repo, err)
		}
		results = append(results, fetchResult{repo: repo, packages: packages})
	}
	return results, nil
}

func processPackageResult(packages *haxmap.Map[string, domain.SourcePackage], result fetchResult) {
	result.packages.ForEach(func(newKey string, newPkg domain.PackageInfo) bool {
		pk, ok := packages.Get(newPkg.Source)
		if !ok {
			newMap := haxmap.New[string, domain.PackageInfo]()
			newMap.Set(newKey, newPkg)
			packages.Set(newPkg.Source, domain.SourcePackage{
				Name:     newPkg.Source,
				Packages: newMap,
				Status:   domain.Current,
				Version:  newPkg.Version,
			})
			return true
		}

		pk.Version = getHighestVer(pk.Version, newPkg.Version)
		pkg, ok := pk.Packages.Get(newKey)
		if !ok {
			pk.Packages.Set(newKey, newPkg)
		} else {
			mVer, _ := version.Parse(pkg.Version)
			extVer, _ := version.Parse(strings.Split(newPkg.Version, "+b")[0])
			cmpVal := version.Compare(extVer, mVer)
			if cmpVal >= 0 {
				pk.Packages.Set(newKey, newPkg)
			}
		}

		packages.Set(newPkg.Source, pk)
		return true
	})
}

func ProcessMissingPackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
	externalPackages.ForEach(func(k string, src domain.SourcePackage) bool {
		_, ok := internalPackages.Get(k)
		if !ok && src.Packages.Len() > 0 {
			src.Status = domain.Missing
			internalPackages.Set(k, src)
		}
		return true
	})
}

func ProcessStalePackages(internalPackages *haxmap.Map[string, domain.SourcePackage], externalPackages *haxmap.Map[string, domain.SourcePackage]) {
	externalPackages.ForEach(func(newPackage string, newSource domain.SourcePackage) bool {
		matchedPackage, ok := internalPackages.Get(newPackage)
		if !ok || matchedPackage.Packages.Len() == 0 {
			return true
		}
		ver := getHighestVer(newSource.Version, matchedPackage.Version)
		if ver != matchedPackage.Version {
			matchedPackage.NewVersion = ver
			matchedPackage.Status = domain.Stale
			internalPackages.Set(newPackage, matchedPackage)
		}

		return true
	})
}

func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[string, domain.PackageInfo], error) {
	resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression)
	if err != nil {
		return nil, fmt.Errorf("fetching package file: %w", err)
	}
	defer resp.Body.Close()

	rdr, err := getCompressedReader(resp.Body, pkg.Compression)
	if err != nil {
		return nil, fmt.Errorf("creating compressed reader: %w", err)
	}

	packages := haxmap.New[string, domain.PackageInfo]()
	sreader := deb.NewControlFileReader(rdr, false, false)

	for {
		stanza, err := sreader.ReadStanza()
		if err != nil || stanza == nil {
			break
		}

		if stanza[SectionKey] == DebianInstallerSection {
			continue
		}

		name := stanza[PackageKey]

		if shouldSkipPackage(name, pkg) {
			continue
		}

		source, sourceVersion := parseSource(stanza[SourceKey])
		if source == "" {
			source = name
		}

		versionStr := chooseVersion(sourceVersion, stanza[VersionKey])

		ver, err := version.Parse(versionStr)
		if err != nil {
			return nil, fmt.Errorf("parsing version %s: %w", versionStr, err)
		}

		if shouldUpdatePackage(packages, name, ver) {
			packages.Set(name, domain.PackageInfo{
				PackageName:  name,
				Version:      ver.String(),
				Source:       source,
				Architecture: stanza[ArchitectureKey],
				Description:  stanza[DescriptionKey],
				Status:       domain.Current,
			})
		}
	}

	return packages, nil
}

func getCompressedReader(body io.Reader, compression string) (io.Reader, error) {
	switch compression {
	case "bz2":
		return bzip2.NewReader(body), nil
	case "xz":
		return xz.NewReader(body)
	case "gz":
		return gzip.NewReader(body)
	default:
		return body, nil
	}
}

func shouldSkipPackage(name string, pkg config.PackageFile) bool {
	if pkg.UseWhitelist && len(pkg.Whitelist) > 0 && !nameContains(name, pkg.Whitelist) {
		return true
	}
	return nameContains(name, pkg.Blacklist)
}

func parseSource(source string) (string, string) {
	sourceSplit := strings.Split(source, " ")
	if len(sourceSplit) > 1 {
		return sourceSplit[0], strings.Trim(sourceSplit[1], "()")
	}
	return source, ""
}

func chooseVersion(sourceVersion, stanzaVersion string) string {
	if sourceVersion != "" {
		return sourceVersion
	}
	return stanzaVersion
}

func shouldUpdatePackage(packages *haxmap.Map[string, domain.PackageInfo], name string, ver version.Version) bool {
	existingPkg, exists := packages.Get(name)
	if !exists {
		return true
	}
	existingVer, _ := version.Parse(existingPkg.Version)
	return version.Compare(ver, existingVer) >= 0
}

func nameContains(name string, match []string) bool {
	for _, m := range match {
		if strings.Contains(name, m) {
			return true
		}
	}
	return false
}

func GetPackagesCount() domain.PackagesCount {
	count := domain.PackagesCount{
		Stale:    0,
		Missing:  0,
		Current:  0,
		Error:    0,
		Queued:   0,
		Building: 0,
	}
	currentPackages.ForEach(func(k string, pkg domain.SourcePackage) bool {
		switch pkg.Status {
		case domain.Stale:
			count.Stale++
		case domain.Missing:
			count.Missing++
		case domain.Built:
			count.Current++
		case domain.Current:
			count.Current++
		case domain.Error:
			count.Error++
		}

		return true
	})

	return count
}

func combinePackages(packages *haxmap.Map[string, domain.SourcePackage]) {
	dmoPackages := haxmap.New[string, domain.SourcePackage]()

	// Identify and collect -dmo packages
	packages.ForEach(func(k string, v domain.SourcePackage) bool {
		if strings.HasSuffix(k, DMOSuffix) {
			dmoPackages.Set(k, v)
			packages.Del(k)
		}
		return true
	})

	// Combine -dmo packages with base packages or add them back
	dmoPackages.ForEach(func(dmoName string, dmoPkg domain.SourcePackage) bool {
		baseName := strings.TrimSuffix(dmoName, DMOSuffix)
		if basePkg, hasBase := packages.Get(baseName); hasBase {
			combinedPkg := combineDMOPackages(dmoPkg, basePkg)
			packages.Set(dmoName, combinedPkg)
			packages.Del(baseName)
		} else {
			packages.Set(dmoName, dmoPkg)
		}
		return true
	})
}

func combineDMOPackages(dmoPkg, basePkg domain.SourcePackage) domain.SourcePackage {
	combinedPkg := dmoPkg
	basePkg.Packages.ForEach(func(pkgName string, pkgInfo domain.PackageInfo) bool {
		if _, exists := combinedPkg.Packages.Get(pkgName); !exists {
			combinedPkg.Packages.Set(pkgName, pkgInfo)
		}
		return true
	})
	return combinedPkg
}

func getHighestVer(ver string, newVer string) string {
	mVer, _ := version.Parse(ver)
	extVer, _ := version.Parse(newVer)
	cmpVal := version.Compare(mVer, extVer)
	if cmpVal < 0 {
		return newVer
	}
	return ver
}