Add bulk rebuild and i386 support

buildqueue/queue.go

@@ -82,6 +82,22 @@ func ProcessNext() error {
 	if item == nil {
 		return nil
 	}
+	if item.Source.Has32bit {
+		i386Item := domain.BuildQueueItem{
+			Source:       item.Source,
+			Status:       domain.Building,
+			Type:         domain.BuildTypeI386,
+			Patch:        item.Patch,
+			Rebuild:      item.Rebuild,
+			BuildNumber:  item.BuildNumber,
+			BuildVersion: item.BuildVersion,
+		}
+		err := UpdateBuildFile(i386Item)
+		if err != nil {
+			return err
+		}
+	}
+
 	err := UpdateBuildFile(*item)
 	if err != nil {
 		return err

config/config.go

@@ -34,6 +34,7 @@ type Config struct {
 	LocalPackageFiles    []PackageFile `json:"localPackageFiles"`
 	ExternalPackageFiles []PackageFile `json:"externalPackageFiles"`
 	LTOBlocklist         []string      `json:"ltoBlocklist"`
+	I386List             []string      `json:"i386List"`
 	DeboutputDir         string        `json:"deboutputDir"`
 	Salt                 string        `json:"salt"`
 	Buildrepo            string        `json:"buildRepo"`
@@ -48,6 +49,10 @@ type expPkgNames struct {
 	PackageNames []string `json:"package_names"`
 }
 
+type i386List struct {
+	PackageNames []string `json:"i386_whitelist"`
+}
+
 func Init() error {
 	err := loadUsers()
 	if err != nil {
@@ -106,9 +111,14 @@ func loadConfig() error {
 		return err
 	}
 
+	Configs.I386List, err = loadI386List()
+	if err != nil {
+		return err
+	}
+
 	Configs = config
 
-	xpPkgNames, err := LoadExperimentalPackageNames()
+	xpPkgNames, err := loadExperimentalPackageNames()
 	if err != nil {
 		return err
 	}
@@ -125,7 +135,26 @@ func loadConfig() error {
 	return nil
 }
 
-func LoadExperimentalPackageNames() ([]string, error) {
+func loadI386List() ([]string, error) {
+	var i386List i386List
+	response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/i386_src_whitelist.json")
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	byteValue, _ := io.ReadAll(response.Body)
+
+	err = json.Unmarshal(byteValue, &i386List)
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+
+	return i386List.PackageNames, nil
+}
+
+func loadExperimentalPackageNames() ([]string, error) {
 	var epPkgNames expPkgNames
 	response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/exp_pkg_names.json")
 	if err != nil {

domain/packages.go

@@ -17,11 +17,13 @@ type PackagesCount struct {
 
 type SourcePackage struct {
 	Name     string `gorm:"primarykey"`
+	Has32bit bool
 	Packages *haxmap.Map[string, PackageInfo] `gorm:"foreignKey:PackageInfo;references:PackageName"`
 }
 
 type SourcePackageDTO struct {
 	Name string `gorm:"primarykey"`
+
 	Packages []PackageInfo `gorm:"foreignKey:PackageName"`
 }
 

fastmap/fastmap.go

@@ -1,155 +0,0 @@
-package fastmap
-
-import (
-	"fmt"
-	"strings"
-
-	"slices"
-
-	"github.com/goccy/go-json"
-)
-
-type Fastmap[K comparable, V any] struct {
-	idx   map[K]int
-	store []fastmapValue[K, V]
-}
-
-type fastmapValue[K comparable, V any] struct {
-	Key   K
-	Value V
-}
-
-func New[K comparable, V any]() *Fastmap[K, V] {
-	return &Fastmap[K, V]{
-		idx:   make(map[K]int),
-		store: make([]fastmapValue[K, V], 0),
-	}
-}
-
-func (m *Fastmap[K, V]) Set(key K, value V) {
-	if _, ok := m.idx[key]; ok {
-		m.store[m.idx[key]].Value = value
-		return
-	}
-	m.idx[key] = len(m.store)
-	m.store = append(m.store, fastmapValue[K, V]{Key: key, Value: value})
-}
-
-func (m *Fastmap[K, V]) Get(key K) (value V, ok bool) {
-	idx, ok := m.idx[key]
-	if !ok {
-		return
-	}
-	return m.store[idx].Value, true
-}
-
-func (m *Fastmap[K, V]) Delete(key K) {
-	idx, ok := m.idx[key]
-	if !ok {
-		return
-	}
-	delete(m.idx, key)
-	m.store[idx] = m.store[len(m.store)-1]
-	m.store = m.store[:len(m.store)-1]
-}
-
-func (m *Fastmap[K, V]) Has(key K) bool {
-	_, ok := m.idx[key]
-	return ok
-}
-
-func (m *Fastmap[K, V]) Len() int {
-	return len(m.idx)
-}
-
-func (m *Fastmap[K, V]) GetPage(pageNum int, pageSize int) *Fastmap[K, V] {
-	start := pageSize * pageNum
-	end := start + pageSize
-	if end > len(m.store) {
-		end = len(m.store)
-	}
-
-	returnVal := New[K, V]()
-	for i := start; i < end; i++ {
-		returnVal.Set(m.store[i].Key, m.store[i].Value)
-	}
-	return returnVal
-}
-
-func (m *Fastmap[K, V]) Clear() {
-	m.idx = make(map[K]int)
-	m.store = make([]fastmapValue[K, V], 0)
-}
-
-func (m *Fastmap[K, V]) Iter(fn func(key K, value V) bool) {
-	for _, v := range m.store {
-		if !fn(v.Key, v.Value) {
-			break
-		}
-	}
-}
-
-func (m *Fastmap[K, V]) StableSortByKey() {
-	slices.SortStableFunc(m.store, func(a, b fastmapValue[K, V]) int {
-		aKey := fmt.Sprint(a.Key)
-		bKey := fmt.Sprint(b.Key)
-		return strings.Compare(aKey, bKey)
-	})
-
-	// Update the index map after sorting
-	for i, v := range m.store {
-		m.idx[v.Key] = i
-	}
-}
-
-func (m *Fastmap[K, V]) MarshalText() ([]byte, error) {
-	var builder strings.Builder
-	for _, v := range m.store {
-		builder.WriteString(fmt.Sprintf("%v:%v\n", v.Key, v.Value))
-	}
-	return []byte(builder.String()), nil
-}
-
-func (m *Fastmap[K, V]) UnmarshalText(text []byte) error {
-	m.Clear()
-	lines := strings.Split(string(text), "\n")
-	for _, line := range lines {
-		if line == "" {
-			continue
-		}
-		parts := strings.SplitN(line, ":", 2)
-		if len(parts) != 2 {
-			return fmt.Errorf("invalid format: %s", line)
-		}
-		var key K
-		var value V
-		if _, err := fmt.Sscan(parts[0], &key); err != nil {
-			return fmt.Errorf("error parsing key: %v", err)
-		}
-		if _, err := fmt.Sscan(parts[1], &value); err != nil {
-			return fmt.Errorf("error parsing value: %v", err)
-		}
-		m.Set(key, value)
-	}
-	return nil
-}
-
-func (m *Fastmap[K, V]) MarshalJSON() ([]byte, error) {
-	temp := make(map[K]V)
-	for _, v := range m.store {
-		temp[v.Key] = v.Value
-	}
-	return json.Marshal(temp)
-}
-
-func (m *Fastmap[K, V]) UnmarshalJSON(data []byte) error {
-	temp := make(map[K]V)
-	if err := json.Unmarshal(data, &temp); err != nil {
-		return err
-	}
-	m.Clear()
-	for k, v := range temp {
-		m.Set(k, v)
-	}
-	return nil
-}

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/jinzhu/now v1.1.5
 	github.com/klauspost/compress v1.17.9
-	github.com/maypok86/otter v1.2.1
 	github.com/samber/slog-fiber v1.16.0
 	github.com/ulikunitz/xz v0.5.12
 	golang.org/x/crypto v0.21.0
@@ -27,9 +26,7 @@ require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
-	github.com/dolthub/maphash v0.1.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

go.sum

@@ -22,14 +22,10 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
-github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
-github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
 github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -76,8 +72,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/maypok86/otter v1.2.1 h1:xyvMW+t0vE1sKt/++GTkznLitEl7D/msqXkAbLwiC1M=
-github.com/maypok86/otter v1.2.1/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=

handlers/build/builkRebuild.go

@@ -0,0 +1,71 @@
+package handlers_build
+
+import (
+	"brunel/buildqueue"
+	"brunel/domain"
+	"brunel/packages"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+type BulkRebuildRequestItem struct {
+	PackageName string `json:"packageName"`
+	BuildType   string `json:"buildType"`
+}
+
+func BulkRebuild(c *fiber.Ctx) error {
+	var req []BulkRebuildRequestItem
+
+	if err := c.BodyParser(&req); err != nil {
+		return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+			"error": "Invalid request body",
+		})
+	}
+	if len(req) == 0 {
+		return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+			"error": "No packages specified",
+		})
+	}
+
+	packs := packages.GetPackages()
+
+	for _, reqItem := range req {
+		pack, ok := packs.Get(reqItem.PackageName)
+		if !ok {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+				"error": "Package not found",
+			})
+		}
+
+		buildVersion := ""
+		pack.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
+			if v.Status == domain.Current {
+				buildVersion = v.Version
+				return false
+			}
+			return true
+		})
+
+		buildItem := domain.BuildQueueItem{
+			Source:       pack,
+			BuildVersion: buildVersion,
+			Type:         domain.BuildType(reqItem.BuildType),
+			Rebuild:      true,
+			Status:       domain.Queued,
+			Patch:        false,
+			BuildNumber:  33,
+		}
+
+		err := buildqueue.Add(buildItem)
+		if err != nil {
+			if err.Error() != "package already in queue" {
+				return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+					"error": "Failed to add build to queue",
+				})
+			}
+		}
+	}
+	return c.Status(fiber.StatusOK).JSON(fiber.Map{
+		"message": "Build job added to queue",
+	})
+}

handlers/packages/i386.go

@@ -0,0 +1,20 @@
+package handlers_packages
+
+import (
+	"brunel/domain"
+	"brunel/packages"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func I386(c *fiber.Ctx) error {
+	pkgs := packages.GetPackages()
+	i386Pkgs := make([]domain.SourcePackage, 0)
+	pkgs.ForEach(func(k string, v domain.SourcePackage) bool {
+		if v.Has32bit {
+			i386Pkgs = append(i386Pkgs, v)
+		}
+		return true
+	})
+	return c.Status(fiber.StatusOK).JSON(i386Pkgs)
+}

helpers/cache.go

@@ -1,13 +0,0 @@
-package helpers
-
-import (
-	"github.com/maypok86/otter"
-)
-
-var (
-	Cache otter.Cache[string, any]
-)
-
-func ReloadCache() {
-	Cache.Clear()
-}

packages/packages.go

@@ -22,7 +22,7 @@ import (
 )
 
 var LastUpdateTime time.Time
-var currentPackagesFastMap = haxmap.New[string, domain.SourcePackage]()
+var currentPackages = haxmap.New[string, domain.SourcePackage]()
 
 func ProcessPackages() error {
 	var internalPackages = haxmap.New[string, domain.SourcePackage]()
@@ -45,7 +45,7 @@ func ProcessPackages() error {
 	updatedPackages := haxmap.New[string, domain.SourcePackage]()
 
 	internalPackages.ForEach(func(k string, v domain.SourcePackage) bool {
-		curr, exists := currentPackagesFastMap.Get(k)
+		curr, exists := currentPackages.Get(k)
 		if !exists {
 			updatedPackages.Set(k, v)
 			return true
@@ -88,10 +88,30 @@ func ProcessPackages() error {
 		return true
 	})
 
-	currentPackagesFastMap = updatedPackages
+	updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool {
+		for _, pkg := range config.Configs.I386List {
+			if v.Name == pkg {
+				v.Has32bit = true
+				return true
+			}
+		}
+		v.Has32bit = false
+		return true
+	})
+
+	for _, pkg := range config.Configs.I386List {
+		if _, ok := updatedPackages.Get(pkg); !ok {
+			updatedPackages.Set(pkg, domain.SourcePackage{
+				Name:     pkg,
+				Has32bit: true,
+				Packages: haxmap.New[string, domain.PackageInfo](),
+			})
+		}
+	}
+
+	currentPackages = updatedPackages
 
 	LastUpdateTime = time.Now()
-	helpers.ReloadCache()
 	err = helpers.DBInst.DropPackages()
 	if err != nil {
 		return err
@@ -105,26 +125,26 @@ func ProcessPackages() error {
 }
 
 func GetPackages() *haxmap.Map[string, domain.SourcePackage] {
-	return currentPackagesFastMap
+	return currentPackages
 }
 
 func UpdatePackage(pkg domain.PackageInfo) error {
-	curr, ok := currentPackagesFastMap.Get(pkg.Source)
+	curr, ok := currentPackages.Get(pkg.Source)
 	if !ok {
 		return fmt.Errorf("package %s not found", pkg.Source)
 	}
 	curr.Packages.Set(pkg.PackageName, pkg)
-	currentPackagesFastMap.Set(pkg.Source, curr)
+	currentPackages.Set(pkg.Source, curr)
 	return saveSingleToDb(curr)
 }
 
 func UpdateSourcePackage(pkg domain.SourcePackage) error {
-	currentPackagesFastMap.Set(pkg.Name, pkg)
+	currentPackages.Set(pkg.Name, pkg)
 	return saveSingleToDb(pkg)
 }
 
 func IsBuilt(pkg domain.PackageInfo) bool {
-	curr, ok := currentPackagesFastMap.Get(pkg.Source)
+	curr, ok := currentPackages.Get(pkg.Source)
 	if !ok {
 		return false
 	}
@@ -141,7 +161,6 @@ func saveSingleToDb(pkg domain.SourcePackage) error {
 		return err
 	}
 	LastUpdateTime = time.Now()
-	helpers.ReloadCache()
 	err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
 	if err != nil {
 		return err
@@ -150,13 +169,12 @@ func saveSingleToDb(pkg domain.SourcePackage) error {
 }
 
 func SaveToDb() error {
-	err := helpers.DBInst.SavePackages(currentPackagesFastMap)
+	err := helpers.DBInst.SavePackages(currentPackages)
 	if err != nil {
 		slog.Error(err.Error())
 		return err
 	}
 	LastUpdateTime = time.Now()
-	helpers.ReloadCache()
 	return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
 }
 
@@ -175,9 +193,9 @@ func LoadFromDb() error {
 		}
 		return -1
 	})
-	currentPackagesFastMap.Clear()
+	currentPackages.Clear()
 	for _, pkg := range packages {
-		currentPackagesFastMap.Set(pkg.Name, pkg)
+		currentPackages.Set(pkg.Name, pkg)
 	}
 	return nil
 }
@@ -476,7 +494,7 @@ func GetPackagesCount() domain.PackagesCount {
 		Queued:   0,
 		Building: 0,
 	}
-	currentPackagesFastMap.ForEach(func(k string, v domain.SourcePackage) bool {
+	currentPackages.ForEach(func(k string, v domain.SourcePackage) bool {
 		v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool {
 			switch pkg.Status {
 			case domain.Stale:

server.go

@@ -19,23 +19,13 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/compress"
 	"github.com/gofiber/fiber/v2/middleware/recover"
-	"github.com/maypok86/otter"
 	slogfiber "github.com/samber/slog-fiber"
 )
 
 // runServer runs a new HTTP server with the loaded environment variables.
 func runServer(ctx context.Context) error {
 	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
-	var err error
+	err := config.Init()
-
-	ott := otter.MustBuilder[string, any](10000)
-	ott.WithTTL(time.Hour * 24)
-	helpers.Cache, err = ott.Build()
-	if err != nil {
-		panic(err)
-	}
-
-	err = config.Init()
 	if err != nil {
 		slog.Error("unable to load configuration: " + err.Error())
 		return err
@@ -96,9 +86,11 @@ func runServer(ctx context.Context) error {
 	server.Get("/api/queue", handlers_build.Queue)
 	server.Get("/api/errored", handlers_build.Errored)
 	server.Get("/api/isloggedin", handlers_auth.IsLoggedIn)
+	server.Get("/api/i386", handlers_packages.I386)
 
 	server.Post("/api/login", handlers_auth.Login)
 	adminRoutes.Post("/triggerbuild", handlers_build.TriggerBuild)
+	adminRoutes.Post("/bulkRebuild", handlers_build.BulkRebuild)
 	adminRoutes.Post("/register", handlers_auth.Register)
 	adminRoutes.Post("/updatePassword", handlers_auth.UpdatePassword)