From 8e5303ed7a3b3eb0c59ae89e9038472e15db3c74 Mon Sep 17 00:00:00 2001 From: ferreo Date: Fri, 2 Aug 2024 21:07:39 +0100 Subject: [PATCH] Add bulk rebuild and i386 support --- buildqueue/queue.go | 16 ++++ config/config.go | 33 ++++++- domain/packages.go | 6 +- fastmap/fastmap.go | 155 --------------------------------- go.mod | 3 - go.sum | 6 -- handlers/build/builkRebuild.go | 71 +++++++++++++++ handlers/packages/i386.go | 20 +++++ helpers/cache.go | 13 --- packages/packages.go | 48 ++++++---- server.go | 14 +-- 11 files changed, 178 insertions(+), 207 deletions(-) delete mode 100644 fastmap/fastmap.go create mode 100644 handlers/build/builkRebuild.go create mode 100644 handlers/packages/i386.go delete mode 100644 helpers/cache.go diff --git a/buildqueue/queue.go b/buildqueue/queue.go index 18911d5..1976329 100644 --- a/buildqueue/queue.go +++ b/buildqueue/queue.go @@ -82,6 +82,22 @@ func ProcessNext() error { if item == nil { return nil } + if item.Source.Has32bit { + i386Item := domain.BuildQueueItem{ + Source: item.Source, + Status: domain.Building, + Type: domain.BuildTypeI386, + Patch: item.Patch, + Rebuild: item.Rebuild, + BuildNumber: item.BuildNumber, + BuildVersion: item.BuildVersion, + } + err := UpdateBuildFile(i386Item) + if err != nil { + return err + } + } + err := UpdateBuildFile(*item) if err != nil { return err diff --git a/config/config.go b/config/config.go index 6c64c51..d660711 100644 --- a/config/config.go +++ b/config/config.go @@ -34,6 +34,7 @@ type Config struct { LocalPackageFiles []PackageFile `json:"localPackageFiles"` ExternalPackageFiles []PackageFile `json:"externalPackageFiles"` LTOBlocklist []string `json:"ltoBlocklist"` + I386List []string `json:"i386List"` DeboutputDir string `json:"deboutputDir"` Salt string `json:"salt"` Buildrepo string `json:"buildRepo"` @@ -48,6 +49,10 @@ type expPkgNames struct { PackageNames []string `json:"package_names"` } +type i386List struct { + PackageNames []string `json:"i386_whitelist"` +} + func Init() error { err := loadUsers() if err != nil { @@ -106,9 +111,14 @@ func loadConfig() error { return err } + Configs.I386List, err = loadI386List() + if err != nil { + return err + } + Configs = config - xpPkgNames, err := LoadExperimentalPackageNames() + xpPkgNames, err := loadExperimentalPackageNames() if err != nil { return err } @@ -125,7 +135,26 @@ func loadConfig() error { return nil } -func LoadExperimentalPackageNames() ([]string, error) { +func loadI386List() ([]string, error) { + var i386List i386List + response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/i386_src_whitelist.json") + if err != nil { + return nil, err + } + defer response.Body.Close() + + byteValue, _ := io.ReadAll(response.Body) + + err = json.Unmarshal(byteValue, &i386List) + if err != nil { + fmt.Println(err) + return nil, err + } + + return i386List.PackageNames, nil +} + +func loadExperimentalPackageNames() ([]string, error) { var epPkgNames expPkgNames response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/exp_pkg_names.json") if err != nil { diff --git a/domain/packages.go b/domain/packages.go index 0373e08..76c4e0f 100644 --- a/domain/packages.go +++ b/domain/packages.go @@ -16,12 +16,14 @@ type PackagesCount struct { } type SourcePackage struct { - Name string `gorm:"primarykey"` + Name string `gorm:"primarykey"` + Has32bit bool Packages *haxmap.Map[string, PackageInfo] `gorm:"foreignKey:PackageInfo;references:PackageName"` } type SourcePackageDTO struct { - Name string `gorm:"primarykey"` + Name string `gorm:"primarykey"` + Packages []PackageInfo `gorm:"foreignKey:PackageName"` } diff --git a/fastmap/fastmap.go b/fastmap/fastmap.go deleted file mode 100644 index 0d24c95..0000000 --- a/fastmap/fastmap.go +++ /dev/null @@ -1,155 +0,0 @@ -package fastmap - -import ( - "fmt" - "strings" - - "slices" - - "github.com/goccy/go-json" -) - -type Fastmap[K comparable, V any] struct { - idx map[K]int - store []fastmapValue[K, V] -} - -type fastmapValue[K comparable, V any] struct { - Key K - Value V -} - -func New[K comparable, V any]() *Fastmap[K, V] { - return &Fastmap[K, V]{ - idx: make(map[K]int), - store: make([]fastmapValue[K, V], 0), - } -} - -func (m *Fastmap[K, V]) Set(key K, value V) { - if _, ok := m.idx[key]; ok { - m.store[m.idx[key]].Value = value - return - } - m.idx[key] = len(m.store) - m.store = append(m.store, fastmapValue[K, V]{Key: key, Value: value}) -} - -func (m *Fastmap[K, V]) Get(key K) (value V, ok bool) { - idx, ok := m.idx[key] - if !ok { - return - } - return m.store[idx].Value, true -} - -func (m *Fastmap[K, V]) Delete(key K) { - idx, ok := m.idx[key] - if !ok { - return - } - delete(m.idx, key) - m.store[idx] = m.store[len(m.store)-1] - m.store = m.store[:len(m.store)-1] -} - -func (m *Fastmap[K, V]) Has(key K) bool { - _, ok := m.idx[key] - return ok -} - -func (m *Fastmap[K, V]) Len() int { - return len(m.idx) -} - -func (m *Fastmap[K, V]) GetPage(pageNum int, pageSize int) *Fastmap[K, V] { - start := pageSize * pageNum - end := start + pageSize - if end > len(m.store) { - end = len(m.store) - } - - returnVal := New[K, V]() - for i := start; i < end; i++ { - returnVal.Set(m.store[i].Key, m.store[i].Value) - } - return returnVal -} - -func (m *Fastmap[K, V]) Clear() { - m.idx = make(map[K]int) - m.store = make([]fastmapValue[K, V], 0) -} - -func (m *Fastmap[K, V]) Iter(fn func(key K, value V) bool) { - for _, v := range m.store { - if !fn(v.Key, v.Value) { - break - } - } -} - -func (m *Fastmap[K, V]) StableSortByKey() { - slices.SortStableFunc(m.store, func(a, b fastmapValue[K, V]) int { - aKey := fmt.Sprint(a.Key) - bKey := fmt.Sprint(b.Key) - return strings.Compare(aKey, bKey) - }) - - // Update the index map after sorting - for i, v := range m.store { - m.idx[v.Key] = i - } -} - -func (m *Fastmap[K, V]) MarshalText() ([]byte, error) { - var builder strings.Builder - for _, v := range m.store { - builder.WriteString(fmt.Sprintf("%v:%v\n", v.Key, v.Value)) - } - return []byte(builder.String()), nil -} - -func (m *Fastmap[K, V]) UnmarshalText(text []byte) error { - m.Clear() - lines := strings.Split(string(text), "\n") - for _, line := range lines { - if line == "" { - continue - } - parts := strings.SplitN(line, ":", 2) - if len(parts) != 2 { - return fmt.Errorf("invalid format: %s", line) - } - var key K - var value V - if _, err := fmt.Sscan(parts[0], &key); err != nil { - return fmt.Errorf("error parsing key: %v", err) - } - if _, err := fmt.Sscan(parts[1], &value); err != nil { - return fmt.Errorf("error parsing value: %v", err) - } - m.Set(key, value) - } - return nil -} - -func (m *Fastmap[K, V]) MarshalJSON() ([]byte, error) { - temp := make(map[K]V) - for _, v := range m.store { - temp[v.Key] = v.Value - } - return json.Marshal(temp) -} - -func (m *Fastmap[K, V]) UnmarshalJSON(data []byte) error { - temp := make(map[K]V) - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - m.Clear() - for k, v := range temp { - m.Set(k, v) - } - return nil -} diff --git a/go.mod b/go.mod index 4e0692f..0be702c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/gofiber/fiber/v2 v2.52.5 github.com/jinzhu/now v1.1.5 github.com/klauspost/compress v1.17.9 - github.com/maypok86/otter v1.2.1 github.com/samber/slog-fiber v1.16.0 github.com/ulikunitz/xz v0.5.12 golang.org/x/crypto v0.21.0 @@ -27,9 +26,7 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/dolthub/maphash v0.1.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/gammazero/deque v0.2.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index af8c21a..0148d50 100644 --- a/go.sum +++ b/go.sum @@ -22,14 +22,10 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= -github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= -github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -76,8 +72,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/maypok86/otter v1.2.1 h1:xyvMW+t0vE1sKt/++GTkznLitEl7D/msqXkAbLwiC1M= -github.com/maypok86/otter v1.2.1/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= diff --git a/handlers/build/builkRebuild.go b/handlers/build/builkRebuild.go new file mode 100644 index 0000000..402c2d0 --- /dev/null +++ b/handlers/build/builkRebuild.go @@ -0,0 +1,71 @@ +package handlers_build + +import ( + "brunel/buildqueue" + "brunel/domain" + "brunel/packages" + + "github.com/gofiber/fiber/v2" +) + +type BulkRebuildRequestItem struct { + PackageName string `json:"packageName"` + BuildType string `json:"buildType"` +} + +func BulkRebuild(c *fiber.Ctx) error { + var req []BulkRebuildRequestItem + + if err := c.BodyParser(&req); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Invalid request body", + }) + } + if len(req) == 0 { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "No packages specified", + }) + } + + packs := packages.GetPackages() + + for _, reqItem := range req { + pack, ok := packs.Get(reqItem.PackageName) + if !ok { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Package not found", + }) + } + + buildVersion := "" + pack.Packages.ForEach(func(k string, v domain.PackageInfo) bool { + if v.Status == domain.Current { + buildVersion = v.Version + return false + } + return true + }) + + buildItem := domain.BuildQueueItem{ + Source: pack, + BuildVersion: buildVersion, + Type: domain.BuildType(reqItem.BuildType), + Rebuild: true, + Status: domain.Queued, + Patch: false, + BuildNumber: 33, + } + + err := buildqueue.Add(buildItem) + if err != nil { + if err.Error() != "package already in queue" { + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": "Failed to add build to queue", + }) + } + } + } + return c.Status(fiber.StatusOK).JSON(fiber.Map{ + "message": "Build job added to queue", + }) +} diff --git a/handlers/packages/i386.go b/handlers/packages/i386.go new file mode 100644 index 0000000..e6a479b --- /dev/null +++ b/handlers/packages/i386.go @@ -0,0 +1,20 @@ +package handlers_packages + +import ( + "brunel/domain" + "brunel/packages" + + "github.com/gofiber/fiber/v2" +) + +func I386(c *fiber.Ctx) error { + pkgs := packages.GetPackages() + i386Pkgs := make([]domain.SourcePackage, 0) + pkgs.ForEach(func(k string, v domain.SourcePackage) bool { + if v.Has32bit { + i386Pkgs = append(i386Pkgs, v) + } + return true + }) + return c.Status(fiber.StatusOK).JSON(i386Pkgs) +} diff --git a/helpers/cache.go b/helpers/cache.go deleted file mode 100644 index 49ec55b..0000000 --- a/helpers/cache.go +++ /dev/null @@ -1,13 +0,0 @@ -package helpers - -import ( - "github.com/maypok86/otter" -) - -var ( - Cache otter.Cache[string, any] -) - -func ReloadCache() { - Cache.Clear() -} diff --git a/packages/packages.go b/packages/packages.go index 1d6b340..b71e0d3 100644 --- a/packages/packages.go +++ b/packages/packages.go @@ -22,7 +22,7 @@ import ( ) var LastUpdateTime time.Time -var currentPackagesFastMap = haxmap.New[string, domain.SourcePackage]() +var currentPackages = haxmap.New[string, domain.SourcePackage]() func ProcessPackages() error { var internalPackages = haxmap.New[string, domain.SourcePackage]() @@ -45,7 +45,7 @@ func ProcessPackages() error { updatedPackages := haxmap.New[string, domain.SourcePackage]() internalPackages.ForEach(func(k string, v domain.SourcePackage) bool { - curr, exists := currentPackagesFastMap.Get(k) + curr, exists := currentPackages.Get(k) if !exists { updatedPackages.Set(k, v) return true @@ -88,10 +88,30 @@ func ProcessPackages() error { return true }) - currentPackagesFastMap = updatedPackages + updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool { + for _, pkg := range config.Configs.I386List { + if v.Name == pkg { + v.Has32bit = true + return true + } + } + v.Has32bit = false + return true + }) + + for _, pkg := range config.Configs.I386List { + if _, ok := updatedPackages.Get(pkg); !ok { + updatedPackages.Set(pkg, domain.SourcePackage{ + Name: pkg, + Has32bit: true, + Packages: haxmap.New[string, domain.PackageInfo](), + }) + } + } + + currentPackages = updatedPackages LastUpdateTime = time.Now() - helpers.ReloadCache() err = helpers.DBInst.DropPackages() if err != nil { return err @@ -105,26 +125,26 @@ func ProcessPackages() error { } func GetPackages() *haxmap.Map[string, domain.SourcePackage] { - return currentPackagesFastMap + return currentPackages } func UpdatePackage(pkg domain.PackageInfo) error { - curr, ok := currentPackagesFastMap.Get(pkg.Source) + curr, ok := currentPackages.Get(pkg.Source) if !ok { return fmt.Errorf("package %s not found", pkg.Source) } curr.Packages.Set(pkg.PackageName, pkg) - currentPackagesFastMap.Set(pkg.Source, curr) + currentPackages.Set(pkg.Source, curr) return saveSingleToDb(curr) } func UpdateSourcePackage(pkg domain.SourcePackage) error { - currentPackagesFastMap.Set(pkg.Name, pkg) + currentPackages.Set(pkg.Name, pkg) return saveSingleToDb(pkg) } func IsBuilt(pkg domain.PackageInfo) bool { - curr, ok := currentPackagesFastMap.Get(pkg.Source) + curr, ok := currentPackages.Get(pkg.Source) if !ok { return false } @@ -141,7 +161,6 @@ func saveSingleToDb(pkg domain.SourcePackage) error { return err } LastUpdateTime = time.Now() - helpers.ReloadCache() err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) if err != nil { return err @@ -150,13 +169,12 @@ func saveSingleToDb(pkg domain.SourcePackage) error { } func SaveToDb() error { - err := helpers.DBInst.SavePackages(currentPackagesFastMap) + err := helpers.DBInst.SavePackages(currentPackages) if err != nil { slog.Error(err.Error()) return err } LastUpdateTime = time.Now() - helpers.ReloadCache() return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) } @@ -175,9 +193,9 @@ func LoadFromDb() error { } return -1 }) - currentPackagesFastMap.Clear() + currentPackages.Clear() for _, pkg := range packages { - currentPackagesFastMap.Set(pkg.Name, pkg) + currentPackages.Set(pkg.Name, pkg) } return nil } @@ -476,7 +494,7 @@ func GetPackagesCount() domain.PackagesCount { Queued: 0, Building: 0, } - currentPackagesFastMap.ForEach(func(k string, v domain.SourcePackage) bool { + currentPackages.ForEach(func(k string, v domain.SourcePackage) bool { v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool { switch pkg.Status { case domain.Stale: diff --git a/server.go b/server.go index 0585878..5027530 100644 --- a/server.go +++ b/server.go @@ -19,23 +19,13 @@ import ( "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/compress" "github.com/gofiber/fiber/v2/middleware/recover" - "github.com/maypok86/otter" slogfiber "github.com/samber/slog-fiber" ) // runServer runs a new HTTP server with the loaded environment variables. func runServer(ctx context.Context) error { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - var err error - - ott := otter.MustBuilder[string, any](10000) - ott.WithTTL(time.Hour * 24) - helpers.Cache, err = ott.Build() - if err != nil { - panic(err) - } - - err = config.Init() + err := config.Init() if err != nil { slog.Error("unable to load configuration: " + err.Error()) return err @@ -96,9 +86,11 @@ func runServer(ctx context.Context) error { server.Get("/api/queue", handlers_build.Queue) server.Get("/api/errored", handlers_build.Errored) server.Get("/api/isloggedin", handlers_auth.IsLoggedIn) + server.Get("/api/i386", handlers_packages.I386) server.Post("/api/login", handlers_auth.Login) adminRoutes.Post("/triggerbuild", handlers_build.TriggerBuild) + adminRoutes.Post("/bulkRebuild", handlers_build.BulkRebuild) adminRoutes.Post("/register", handlers_auth.Register) adminRoutes.Post("/updatePassword", handlers_auth.UpdatePassword)