From 6c31af01669f970871ee82de61330ce514e14592 Mon Sep 17 00:00:00 2001 From: ferreo Date: Tue, 30 Jul 2024 15:41:35 +0100 Subject: [PATCH] Add errored builds api. fix package parsing of source version numbers --- buildqueue/worker.go | 51 +++++++++++++++++--------- db/db.go | 1 + db/repository.go | 17 +++++++++ domain/buildqueue.go | 7 ++++ go.mod | 2 +- handlers/build/errored.go | 29 +++++++++++++++ handlers/build/triggerBuild.go | 65 ++++++++++++++++++++++++++++++++++ packages/packages.go | 26 ++++++++++---- server.go | 2 ++ 9 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 handlers/build/errored.go create mode 100644 handlers/build/triggerBuild.go diff --git a/buildqueue/worker.go b/buildqueue/worker.go index 192d96d..22fbe9b 100644 --- a/buildqueue/worker.go +++ b/buildqueue/worker.go @@ -3,6 +3,7 @@ package buildqueue import ( "brunel/config" "brunel/domain" + "brunel/helpers" "brunel/packages" "context" "fmt" @@ -30,7 +31,6 @@ func StartPackageQueueWorker(ctx context.Context) { packs.ForEach(func(k string, v domain.SourcePackage) bool { needsBuild := false buildVersion := "" - buildAttempt := 0 v.Packages.ForEach(func(k string, v domain.PackageInfo) bool { if v.Status == domain.Current { return true @@ -39,11 +39,6 @@ func StartPackageQueueWorker(ctx context.Context) { if vs == "" { vs = v.Version } - if v.LastBuildStatus == domain.Error { - if v.BuildAttempts > 0 { - buildAttempt = v.BuildAttempts - } - } if v.Status == domain.Missing || v.Status == domain.Stale { needsBuild = true } @@ -66,11 +61,17 @@ func StartPackageQueueWorker(ctx context.Context) { return true }) if needsBuild { - if buildAttempt > 1 { + state, err := helpers.DBInst.GetBuildState(v.Name) + if err != nil { + state = domain.BuildState{ + BuildNumber: 0, + } + } + if state.BuildNumber > 1 { return true } typ := domain.BuildTypeLTO - if buildAttempt == 1 { + if state.BuildNumber == 1 { typ = domain.BuildTypeNormal } buildItem := domain.BuildQueueItem{ @@ -79,10 +80,10 @@ func StartPackageQueueWorker(ctx context.Context) { Type: typ, Patch: false, Rebuild: false, - BuildNumber: buildAttempt, + BuildNumber: state.BuildNumber, BuildVersion: buildVersion, } - err := Add(buildItem) + err = Add(buildItem) if err != nil { slog.Info("unable to add package to queue: " + err.Error()) } @@ -113,10 +114,13 @@ func processQueueAndStatus(ctx context.Context) { slog.Error("unable to check if build is complete: " + err.Error()) } if complete { + if err != nil { - updatePackageStatus(&item, domain.Error, domain.Error) + updateBuildState(item, domain.Error) + updatePackageStatus(&item, domain.Error) } else { - updatePackageStatus(&item, domain.Current, domain.Built) + updateBuildState(item, domain.Built) + updatePackageStatus(&item, domain.Current) } packages.UpdateSourcePackage(item.Source) itemsToRemove = append(itemsToRemove, k) @@ -136,19 +140,16 @@ func processQueueAndStatus(ctx context.Context) { } } - time.Sleep(1500 * time.Millisecond) + time.Sleep(10 * time.Second) } } } -func updatePackageStatus(item *domain.BuildQueueItem, status domain.PackageStatus, buildStatus domain.PackageStatus) { +func updatePackageStatus(item *domain.BuildQueueItem, status domain.PackageStatus) { item.Source.Packages.ForEach(func(k string, v domain.PackageInfo) bool { v.Status = status - v.BuildAttempts++ - v.LastBuildStatus = domain.PackageStatus(buildStatus) if status == domain.Current { v.Version = item.BuildVersion - v.BuildAttempts = 0 v.NewVersion = "" } item.Source.Packages.Set(k, v) @@ -156,6 +157,22 @@ func updatePackageStatus(item *domain.BuildQueueItem, status domain.PackageStatu }) } +func updateBuildState(item domain.BuildQueueItem, buildStatus domain.PackageStatus) { + state, err := helpers.DBInst.GetBuildState(item.Source.Name) + if err != nil { + state = domain.BuildState{ + Name: item.Source.Name, + Status: string(domain.Queued), + BuildVersion: item.BuildVersion, + BuildNumber: 0, + } + } + state.Status = string(buildStatus) + state.BuildVersion = item.BuildVersion + state.BuildNumber++ + helpers.DBInst.UpdateBuildState(state) +} + func StartQueueAndStatusWorker(ctx context.Context) { go processQueueAndStatus(ctx) } diff --git a/db/db.go b/db/db.go index 97a67bc..71e6e9c 100644 --- a/db/db.go +++ b/db/db.go @@ -32,6 +32,7 @@ func New() (*gorm.DB, error) { panic("failed to connect database") } + db.AutoMigrate(&domain.BuildState{}) db.AutoMigrate(&domain.User{}) db.AutoMigrate(&domain.Session{}) db.AutoMigrate(&domain.PackageInfo{}) diff --git a/db/repository.go b/db/repository.go index 9612a63..afb6009 100644 --- a/db/repository.go +++ b/db/repository.go @@ -138,6 +138,23 @@ func (r *Repository) UpdateLastUpdateTime(time time.Time) error { return tx.Error } +func (r *Repository) UpdateBuildState(state domain.BuildState) error { + tx := r.db.Save(&state) + return tx.Error +} + +func (r *Repository) GetBuildState(name string) (domain.BuildState, error) { + var state domain.BuildState + tx := r.db.Where("name = ?", name).First(&state) + return state, tx.Error +} + +func (r *Repository) GetFailedBuilds() ([]domain.BuildState, error) { + var states []domain.BuildState + tx := r.db.Where("status = ? AND build_number > 1", string(domain.Error)).Find(&states) + return states, tx.Error +} + func sourcePackageToDto(pkg domain.SourcePackage) domain.SourcePackageDTO { dto := domain.SourcePackageDTO{ Name: pkg.Name, diff --git a/domain/buildqueue.go b/domain/buildqueue.go index 5f254b6..be66a8b 100644 --- a/domain/buildqueue.go +++ b/domain/buildqueue.go @@ -33,3 +33,10 @@ type BuildQueueCount struct { Queued int `json:"queued"` Building int `json:"building"` } + +type BuildState struct { + Name string `gorm:"primarykey"` + Status string + BuildVersion string + BuildNumber int +} diff --git a/go.mod b/go.mod index 0e58f2b..4e0692f 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/samber/slog-fiber v1.16.0 github.com/ulikunitz/xz v0.5.12 golang.org/x/crypto v0.21.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/net v0.22.0 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.11 @@ -50,7 +51,6 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/handlers/build/errored.go b/handlers/build/errored.go new file mode 100644 index 0000000..66eeca2 --- /dev/null +++ b/handlers/build/errored.go @@ -0,0 +1,29 @@ +package handlers_build + +import ( + "brunel/domain" + "brunel/helpers" + + "github.com/gofiber/fiber/v2" + "golang.org/x/exp/slog" +) + +type ErroredResponse struct { + Total int `json:"total"` + Packages []domain.BuildState `json:"packages"` +} + +func Errored(c *fiber.Ctx) error { + states, err := helpers.DBInst.GetFailedBuilds() + if err != nil { + slog.Error(err.Error()) + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + response := ErroredResponse{ + Total: len(states), + Packages: states, + } + + return c.Status(fiber.StatusOK).JSON(response) +} diff --git a/handlers/build/triggerBuild.go b/handlers/build/triggerBuild.go new file mode 100644 index 0000000..442c62b --- /dev/null +++ b/handlers/build/triggerBuild.go @@ -0,0 +1,65 @@ +package handlers_build + +import ( + "brunel/buildqueue" + "brunel/domain" + "brunel/packages" + + "github.com/gofiber/fiber/v2" +) + +func TriggerBuild(c *fiber.Ctx) error { + var req struct { + PackageName string `json:"packageName"` + Version string `json:"version"` + BuildType string `json:"buildType"` + Rebuild bool `json:"rebuild"` + } + + if err := c.BodyParser(&req); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Invalid request body", + }) + } + + if req.PackageName == "" || req.Version == "" || req.BuildType == "" { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Missing required fields", + }) + } + + packs := packages.GetPackages() + pack, ok := packs.Get(req.PackageName) + if !ok { + return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{ + "error": "Package not found", + }) + } + + buildItem := domain.BuildQueueItem{ + Source: pack, + BuildVersion: req.Version, + Type: domain.BuildType(req.BuildType), + Rebuild: req.Rebuild, + Status: domain.Queued, + Patch: false, + BuildNumber: 0, + } + + err := buildqueue.Add(buildItem) + if err != nil { + if err.Error() == "package already in queue" { + return c.Status(fiber.StatusConflict).JSON(fiber.Map{ + "error": "Package already in queue", + }) + } + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": "Failed to add build to queue", + }) + } + + return c.Status(fiber.StatusAccepted).JSON(fiber.Map{ + "message": "Build job added to queue", + "package": req.PackageName, + }) +} diff --git a/packages/packages.go b/packages/packages.go index c2bbc15..85bf258 100644 --- a/packages/packages.go +++ b/packages/packages.go @@ -409,7 +409,25 @@ func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[ continue } - ver, err := version.Parse(stanza["Version"]) + sourceSplit := strings.Split(stanza["Source"], " ") + source := sourceSplit[0] + if source == "" { + source = name + } + + // Extract version from Source if available + sourceVersion := "" + if len(sourceSplit) > 1 { + sourceVersion = strings.Trim(sourceSplit[1], "()") + } + + // Use sourceVersion if available, otherwise use stanza["Version"] + versionStr := stanza["Version"] + if sourceVersion != "" { + versionStr = sourceVersion + } + + ver, err := version.Parse(versionStr) if err != nil { return nil, err } @@ -423,12 +441,6 @@ func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*haxmap.Map[ } } - sourceSplit := strings.Split(stanza["Source"], " ") - source := sourceSplit[0] - if source == "" { - source = name - } - packages.Set(name, domain.PackageInfo{ PackageName: name, Version: ver.String(), diff --git a/server.go b/server.go index 4fe9c0f..b4f28a0 100644 --- a/server.go +++ b/server.go @@ -93,8 +93,10 @@ func runServer(ctx context.Context) error { server.Get("/api/counts", handlers_packages.Counts) server.Get("/api/packages", handlers_packages.Packages) server.Get("/api/queue", handlers_build.Queue) + server.Get("/api/errored", handlers_build.Errored) server.Post("/api/login", handlers_auth.Login) + adminRoutes.Post("/triggerBuild", handlers_build.TriggerBuild) adminRoutes.Post("/register", handlers_auth.Register) adminRoutes.Post("/updatePassword", handlers_auth.UpdatePassword)