repo-tools/brunel
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add bulk rebuild and i386 support

Browse Source
This commit is contained in:
ferreo 2024-08-02 21:07:39 +01:00
parent fe65a0b9eb
commit 8e5303ed7a
11 changed files with 178 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
buildqueue/queue.go
View File

@ -82,6 +82,22 @@ func ProcessNext() error {
if item == nil {
return nil
}
if item.Source.Has32bit {
i386Item := domain.BuildQueueItem{
Source: item.Source,
Status: domain.Building,
Type: domain.BuildTypeI386,
Patch: item.Patch,
Rebuild: item.Rebuild,
BuildNumber: item.BuildNumber,
BuildVersion: item.BuildVersion,
}
err := UpdateBuildFile(i386Item)
if err != nil {
return err
}
}
err := UpdateBuildFile(*item)
if err != nil {
return err

33
config/config.go
View File

@ -34,6 +34,7 @@ type Config struct {
LocalPackageFiles []PackageFile `json:"localPackageFiles"`
ExternalPackageFiles []PackageFile `json:"externalPackageFiles"`
LTOBlocklist []string `json:"ltoBlocklist"`
I386List []string `json:"i386List"`
DeboutputDir string `json:"deboutputDir"`
Salt string `json:"salt"`
Buildrepo string `json:"buildRepo"`
@ -48,6 +49,10 @@ type expPkgNames struct {
PackageNames []string `json:"package_names"`
}
type i386List struct {
PackageNames []string `json:"i386_whitelist"`
}
func Init() error {
err := loadUsers()
if err != nil {
@ -106,9 +111,14 @@ func loadConfig() error {
return err
}
Configs.I386List, err = loadI386List()
if err != nil {
return err
}
Configs = config
xpPkgNames, err := LoadExperimentalPackageNames()
xpPkgNames, err := loadExperimentalPackageNames()
if err != nil {
return err
}
@ -125,7 +135,26 @@ func loadConfig() error {
return nil
}
func LoadExperimentalPackageNames() ([]string, error) {
func loadI386List() ([]string, error) {
var i386List i386List
response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/i386_src_whitelist.json")
if err != nil {
return nil, err
}
defer response.Body.Close()
byteValue, _ := io.ReadAll(response.Body)
err = json.Unmarshal(byteValue, &i386List)
if err != nil {
fmt.Println(err)
return nil, err
}
return i386List.PackageNames, nil
}
func loadExperimentalPackageNames() ([]string, error) {
var epPkgNames expPkgNames
response, err := http.Get("https://raw.githubusercontent.com/PikaOS-Linux/pika-base-debian-container/main/exp_pkg_names.json")
if err != nil {

2
domain/packages.go
View File

@ -17,11 +17,13 @@ type PackagesCount struct {
type SourcePackage struct {
Name string `gorm:"primarykey"`
Has32bit bool
Packages *haxmap.Map[string, PackageInfo] `gorm:"foreignKey:PackageInfo;references:PackageName"`
}
type SourcePackageDTO struct {
Name string `gorm:"primarykey"`
Packages []PackageInfo `gorm:"foreignKey:PackageName"`
}

155
fastmap/fastmap.go
View File

@ -1,155 +0,0 @@
package fastmap
import (
"fmt"
"strings"
"slices"
"github.com/goccy/go-json"
)
type Fastmap[K comparable, V any] struct {
idx map[K]int
store []fastmapValue[K, V]
}
type fastmapValue[K comparable, V any] struct {
Key K
Value V
}
func New[K comparable, V any]() *Fastmap[K, V] {
return &Fastmap[K, V]{
idx: make(map[K]int),
store: make([]fastmapValue[K, V], 0),
}
}
func (m *Fastmap[K, V]) Set(key K, value V) {
if _, ok := m.idx[key]; ok {
m.store[m.idx[key]].Value = value
return
}
m.idx[key] = len(m.store)
m.store = append(m.store, fastmapValue[K, V]{Key: key, Value: value})
}
func (m *Fastmap[K, V]) Get(key K) (value V, ok bool) {
idx, ok := m.idx[key]
if !ok {
return
}
return m.store[idx].Value, true
}
func (m *Fastmap[K, V]) Delete(key K) {
idx, ok := m.idx[key]
if !ok {
return
}
delete(m.idx, key)
m.store[idx] = m.store[len(m.store)-1]
m.store = m.store[:len(m.store)-1]
}
func (m *Fastmap[K, V]) Has(key K) bool {
_, ok := m.idx[key]
return ok
}
func (m *Fastmap[K, V]) Len() int {
return len(m.idx)
}
func (m *Fastmap[K, V]) GetPage(pageNum int, pageSize int) *Fastmap[K, V] {
start := pageSize * pageNum
end := start + pageSize
if end > len(m.store) {
end = len(m.store)
}
returnVal := New[K, V]()
for i := start; i < end; i++ {
returnVal.Set(m.store[i].Key, m.store[i].Value)
}
return returnVal
}
func (m *Fastmap[K, V]) Clear() {
m.idx = make(map[K]int)
m.store = make([]fastmapValue[K, V], 0)
}
func (m *Fastmap[K, V]) Iter(fn func(key K, value V) bool) {
for _, v := range m.store {
if !fn(v.Key, v.Value) {
break
}
}
}
func (m *Fastmap[K, V]) StableSortByKey() {
slices.SortStableFunc(m.store, func(a, b fastmapValue[K, V]) int {
aKey := fmt.Sprint(a.Key)
bKey := fmt.Sprint(b.Key)
return strings.Compare(aKey, bKey)
})
// Update the index map after sorting
for i, v := range m.store {
m.idx[v.Key] = i
}
}
func (m *Fastmap[K, V]) MarshalText() ([]byte, error) {
var builder strings.Builder
for _, v := range m.store {
builder.WriteString(fmt.Sprintf("%v:%v\n", v.Key, v.Value))
}
return []byte(builder.String()), nil
}
func (m *Fastmap[K, V]) UnmarshalText(text []byte) error {
m.Clear()
lines := strings.Split(string(text), "\n")
for _, line := range lines {
if line == "" {
continue
}
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
return fmt.Errorf("invalid format: %s", line)
}
var key K
var value V
if _, err := fmt.Sscan(parts[0], &key); err != nil {
return fmt.Errorf("error parsing key: %v", err)
}
if _, err := fmt.Sscan(parts[1], &value); err != nil {
return fmt.Errorf("error parsing value: %v", err)
}
m.Set(key, value)
}
return nil
}
func (m *Fastmap[K, V]) MarshalJSON() ([]byte, error) {
temp := make(map[K]V)
for _, v := range m.store {
temp[v.Key] = v.Value
}
return json.Marshal(temp)
}
func (m *Fastmap[K, V]) UnmarshalJSON(data []byte) error {
temp := make(map[K]V)
if err := json.Unmarshal(data, &temp); err != nil {
return err
}
m.Clear()
for k, v := range temp {
m.Set(k, v)
}
return nil
}

3
go.mod
View File

@ -9,7 +9,6 @@ require (
github.com/gofiber/fiber/v2 v2.52.5
github.com/jinzhu/now v1.1.5
github.com/klauspost/compress v1.17.9
github.com/maypok86/otter v1.2.1
github.com/samber/slog-fiber v1.16.0
github.com/ulikunitz/xz v0.5.12
golang.org/x/crypto v0.21.0
@ -27,9 +26,7 @@ require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/dolthub/maphash v0.1.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/gammazero/deque v0.2.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

6
go.sum
View File

@ -22,14 +22,10 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@ -76,8 +72,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/maypok86/otter v1.2.1 h1:xyvMW+t0vE1sKt/++GTkznLitEl7D/msqXkAbLwiC1M=
github.com/maypok86/otter v1.2.1/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=

71
handlers/build/builkRebuild.go Normal file
View File

@ -0,0 +1,71 @@
package handlers_build
import (
"brunel/buildqueue"
"brunel/domain"
"brunel/packages"
"github.com/gofiber/fiber/v2"
)
type BulkRebuildRequestItem struct {
PackageName string `json:"packageName"`
BuildType string `json:"buildType"`
}
func BulkRebuild(c *fiber.Ctx) error {
var req []BulkRebuildRequestItem
if err := c.BodyParser(&req); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
"error": "Invalid request body",
})
}
if len(req) == 0 {
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
"error": "No packages specified",
})
}
packs := packages.GetPackages()
for _, reqItem := range req {
pack, ok := packs.Get(reqItem.PackageName)
if !ok {
return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
"error": "Package not found",
})
}
buildVersion := ""
pack.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
if v.Status == domain.Current {
buildVersion = v.Version
return false
}
return true
})
buildItem := domain.BuildQueueItem{
Source: pack,
BuildVersion: buildVersion,
Type: domain.BuildType(reqItem.BuildType),
Rebuild: true,
Status: domain.Queued,
Patch: false,
BuildNumber: 33,
}
err := buildqueue.Add(buildItem)
if err != nil {
if err.Error() != "package already in queue" {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
"error": "Failed to add build to queue",
})
}
}
}
return c.Status(fiber.StatusOK).JSON(fiber.Map{
"message": "Build job added to queue",
})
}

20
handlers/packages/i386.go Normal file
View File

@ -0,0 +1,20 @@
package handlers_packages
import (
"brunel/domain"
"brunel/packages"
"github.com/gofiber/fiber/v2"
)
func I386(c *fiber.Ctx) error {
pkgs := packages.GetPackages()
i386Pkgs := make([]domain.SourcePackage, 0)
pkgs.ForEach(func(k string, v domain.SourcePackage) bool {
if v.Has32bit {
i386Pkgs = append(i386Pkgs, v)
}
return true
})
return c.Status(fiber.StatusOK).JSON(i386Pkgs)
}

13
helpers/cache.go
View File

@ -1,13 +0,0 @@
package helpers
import (
"github.com/maypok86/otter"
)
var (
Cache otter.Cache[string, any]
)
func ReloadCache() {
Cache.Clear()
}

48
packages/packages.go
View File

@ -22,7 +22,7 @@ import (
)
var LastUpdateTime time.Time
var currentPackagesFastMap = haxmap.New[string, domain.SourcePackage]()
var currentPackages = haxmap.New[string, domain.SourcePackage]()
func ProcessPackages() error {
var internalPackages = haxmap.New[string, domain.SourcePackage]()
@ -45,7 +45,7 @@ func ProcessPackages() error {
updatedPackages := haxmap.New[string, domain.SourcePackage]()
internalPackages.ForEach(func(k string, v domain.SourcePackage) bool {
curr, exists := currentPackagesFastMap.Get(k)
curr, exists := currentPackages.Get(k)
if !exists {
updatedPackages.Set(k, v)
return true
@ -88,10 +88,30 @@ func ProcessPackages() error {
return true
})
currentPackagesFastMap = updatedPackages
updatedPackages.ForEach(func(k string, v domain.SourcePackage) bool {
for _, pkg := range config.Configs.I386List {
if v.Name == pkg {
v.Has32bit = true
return true
}
}
v.Has32bit = false
return true
})
for _, pkg := range config.Configs.I386List {
if _, ok := updatedPackages.Get(pkg); !ok {
updatedPackages.Set(pkg, domain.SourcePackage{
Name: pkg,
Has32bit: true,
Packages: haxmap.New[string, domain.PackageInfo](),
})
}
}
currentPackages = updatedPackages
LastUpdateTime = time.Now()
helpers.ReloadCache()
err = helpers.DBInst.DropPackages()
if err != nil {
return err
@ -105,26 +125,26 @@ func ProcessPackages() error {
}
func GetPackages() *haxmap.Map[string, domain.SourcePackage] {
return currentPackagesFastMap
return currentPackages
}
func UpdatePackage(pkg domain.PackageInfo) error {
curr, ok := currentPackagesFastMap.Get(pkg.Source)
curr, ok := currentPackages.Get(pkg.Source)
if !ok {
return fmt.Errorf("package %s not found", pkg.Source)
}
curr.Packages.Set(pkg.PackageName, pkg)
currentPackagesFastMap.Set(pkg.Source, curr)
currentPackages.Set(pkg.Source, curr)
return saveSingleToDb(curr)
}
func UpdateSourcePackage(pkg domain.SourcePackage) error {
currentPackagesFastMap.Set(pkg.Name, pkg)
currentPackages.Set(pkg.Name, pkg)
return saveSingleToDb(pkg)
}
func IsBuilt(pkg domain.PackageInfo) bool {
curr, ok := currentPackagesFastMap.Get(pkg.Source)
curr, ok := currentPackages.Get(pkg.Source)
if !ok {
return false
}
@ -141,7 +161,6 @@ func saveSingleToDb(pkg domain.SourcePackage) error {
return err
}
LastUpdateTime = time.Now()
helpers.ReloadCache()
err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
if err != nil {
return err
@ -150,13 +169,12 @@ func saveSingleToDb(pkg domain.SourcePackage) error {
}
func SaveToDb() error {
err := helpers.DBInst.SavePackages(currentPackagesFastMap)
err := helpers.DBInst.SavePackages(currentPackages)
if err != nil {
slog.Error(err.Error())
return err
}
LastUpdateTime = time.Now()
helpers.ReloadCache()
return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime)
}
@ -175,9 +193,9 @@ func LoadFromDb() error {
}
return -1
})
currentPackagesFastMap.Clear()
currentPackages.Clear()
for _, pkg := range packages {
currentPackagesFastMap.Set(pkg.Name, pkg)
currentPackages.Set(pkg.Name, pkg)
}
return nil
}
@ -476,7 +494,7 @@ func GetPackagesCount() domain.PackagesCount {
Queued: 0,
Building: 0,
}
currentPackagesFastMap.ForEach(func(k string, v domain.SourcePackage) bool {
currentPackages.ForEach(func(k string, v domain.SourcePackage) bool {
v.Packages.ForEach(func(k string, pkg domain.PackageInfo) bool {
switch pkg.Status {
case domain.Stale:

14
server.go
View File

@ -19,23 +19,13 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/compress"
"github.com/gofiber/fiber/v2/middleware/recover"
"github.com/maypok86/otter"
slogfiber "github.com/samber/slog-fiber"
)
// runServer runs a new HTTP server with the loaded environment variables.
func runServer(ctx context.Context) error {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
var err error
ott := otter.MustBuilder[string, any](10000)
ott.WithTTL(time.Hour * 24)
helpers.Cache, err = ott.Build()
if err != nil {
panic(err)
}
err = config.Init()
err := config.Init()
if err != nil {
slog.Error("unable to load configuration: " + err.Error())
return err
@ -96,9 +86,11 @@ func runServer(ctx context.Context) error {
server.Get("/api/queue", handlers_build.Queue)
server.Get("/api/errored", handlers_build.Errored)
server.Get("/api/isloggedin", handlers_auth.IsLoggedIn)
server.Get("/api/i386", handlers_packages.I386)
server.Post("/api/login", handlers_auth.Login)
adminRoutes.Post("/triggerbuild", handlers_build.TriggerBuild)
adminRoutes.Post("/bulkRebuild", handlers_build.BulkRebuild)
adminRoutes.Post("/register", handlers_auth.Register)
adminRoutes.Post("/updatePassword", handlers_auth.UpdatePassword)