brunel/buildqueue/worker.go

327 lines
7.0 KiB
Go
Raw Normal View History

2024-07-29 23:40:26 +02:00
package buildqueue
import (
"brunel/config"
"brunel/domain"
"brunel/helpers"
2024-07-29 23:40:26 +02:00
"brunel/packages"
"context"
"fmt"
"log/slog"
"net/http"
2024-07-30 02:24:18 +02:00
"sort"
2024-07-29 23:40:26 +02:00
"time"
"golang.org/x/net/html"
)
func StartPackageQueueWorker(ctx context.Context) {
go func() {
for {
select {
case <-ctx.Done():
return
default:
2024-08-04 19:01:27 +02:00
processPackages()
2024-07-29 23:40:26 +02:00
time.Sleep(1 * time.Hour)
}
}
}()
}
2024-08-04 19:01:27 +02:00
func StartQueueAndStatusWorker(ctx context.Context) {
go processQueueAndStatus(ctx)
}
func processPackages() {
if err := packages.ProcessPackages(); err != nil {
slog.Error("unable to process packages", "error", err)
return
}
packages.GetPackages().ForEach(func(k string, v domain.SourcePackage) bool {
if !shouldBuild(v) {
return true
}
state, _ := helpers.DBInst.GetBuildState(v.Name)
2024-08-10 15:06:31 +02:00
if state.BuildNumber > 3 {
2024-08-04 19:01:27 +02:00
return true
}
2024-08-10 15:06:31 +02:00
if state.BuildNumber > 1 && state.Type == string(domain.BuildTypeI386) {
return true
}
queueBuildItem(v, state, false)
if v.Has32bit {
queueBuildItem(v, state, true)
2024-08-04 19:01:27 +02:00
}
return true
})
}
2024-08-10 15:06:31 +02:00
func queueBuildItem(v domain.SourcePackage, state domain.BuildState, isI386 bool) {
buildItem := createBuildItem(v, state, isI386)
if err := Add(buildItem); err != nil {
slog.Info("unable to add package to queue", "error", err)
}
2024-08-04 19:01:27 +02:00
}
2024-08-10 15:06:31 +02:00
func createBuildItem(v domain.SourcePackage, state domain.BuildState, isI386 bool) domain.BuildQueueItem {
2024-08-04 19:01:27 +02:00
buildVersion := v.NewVersion
if buildVersion == "" {
buildVersion = v.Version
}
2024-08-10 15:06:31 +02:00
var typ domain.BuildType
if isI386 {
typ = domain.BuildTypeI386
} else {
typ = domain.BuildTypeLTO
if state.BuildNumber == 1 {
typ = domain.BuildTypeNormal
} else if state.BuildNumber == 2 {
typ = domain.BuildTypeAmd64
}
2024-08-04 19:01:27 +02:00
}
return domain.BuildQueueItem{
Source: v,
Status: domain.Queued,
Type: typ,
Patch: false,
Rebuild: false,
BuildNumber: state.BuildNumber,
BuildVersion: buildVersion,
}
}
2024-08-10 15:06:31 +02:00
func shouldBuild(v domain.SourcePackage) bool {
return v.Status == domain.Missing || v.Status == domain.Stale || v.Status == domain.Error
}
2024-08-04 19:01:27 +02:00
2024-07-30 02:30:26 +02:00
func processQueueAndStatus(ctx context.Context) {
2024-07-29 23:40:26 +02:00
for {
select {
case <-ctx.Done():
return
default:
2024-08-04 19:01:27 +02:00
processQueue()
time.Sleep(10 * time.Second)
}
}
}
2024-07-30 02:30:26 +02:00
2024-08-04 19:01:27 +02:00
func processQueue() {
q := GetQueue()
2024-08-10 15:06:31 +02:00
itemsToRemove := []struct {
name string
buildType domain.BuildType
}{}
2024-08-04 19:01:27 +02:00
buildingFound := false
2024-07-30 02:30:26 +02:00
2024-08-04 19:01:27 +02:00
q.ForEach(func(k string, item domain.BuildQueueItem) bool {
if item.Status == domain.Building {
buildingFound = true
complete, err := checkIfBuildComplete(item)
if complete {
handleCompletedBuild(item, err)
2024-08-10 15:06:31 +02:00
itemsToRemove = append(itemsToRemove, struct {
name string
buildType domain.BuildType
}{
name: item.Source.Name,
buildType: item.Type,
})
2024-07-30 02:30:26 +02:00
}
2024-08-04 19:01:27 +02:00
}
return true
})
2024-07-30 02:30:26 +02:00
2024-08-04 19:01:27 +02:00
for _, item := range itemsToRemove {
2024-08-10 15:06:31 +02:00
Remove(item.name, item.buildType)
2024-08-04 19:01:27 +02:00
}
if !buildingFound {
if err := ProcessNext(); err != nil {
slog.Error("unable to process queue", "error", err)
2024-07-29 23:40:26 +02:00
}
}
}
2024-08-04 19:01:27 +02:00
func handleCompletedBuild(item domain.BuildQueueItem, err error) {
status := domain.Built
if err != nil {
status = domain.Error
}
if item.Type != domain.BuildTypeI386 {
if err != nil {
item.Source.BuildAttempts++
} else {
item.Source.BuildAttempts = 0
item.Source.Version = item.BuildVersion
}
item.Source.Status = status
item.Source.LastBuildStatus = status
updatePackageStatus(&item, status)
packages.UpdateSourcePackage(item.Source)
}
2024-08-04 19:01:27 +02:00
updateBuildState(item, status)
}
func updatePackageStatus(item *domain.BuildQueueItem, status domain.PackageStatus) {
2024-07-30 02:30:26 +02:00
item.Source.Packages.ForEach(func(k string, v domain.PackageInfo) bool {
v.Status = status
if status == domain.Current {
v.Version = item.BuildVersion
v.NewVersion = ""
2024-07-29 23:40:26 +02:00
}
2024-07-30 02:30:26 +02:00
item.Source.Packages.Set(k, v)
return true
})
}
func updateBuildState(item domain.BuildQueueItem, buildStatus domain.PackageStatus) {
state, err := helpers.DBInst.GetBuildState(item.Source.Name)
if err != nil {
state = domain.BuildState{
Name: item.Source.Name,
Status: string(domain.Queued),
BuildVersion: item.BuildVersion,
2024-08-10 15:06:31 +02:00
Type: string(item.Type),
BuildNumber: 0,
}
}
state.Status = string(buildStatus)
2024-08-10 15:06:31 +02:00
state.Type = string(item.Type)
state.BuildVersion = item.BuildVersion
if state.Status == string(domain.Built) {
state.BuildNumber = 0
} else {
state.BuildNumber++
}
helpers.DBInst.UpdateBuildState(state)
2024-07-30 02:30:26 +02:00
}
2024-08-04 19:01:27 +02:00
func checkIfBuildComplete(item domain.BuildQueueItem) (bool, error) {
builds, err := fetchBuilds(item.Source.Name + "=" + item.BuildVersion)
if err != nil {
return false, err
}
if len(builds) == 0 {
slog.Info("No matching builds found", "buildName", item.Source.Name+"="+item.BuildVersion)
return false, nil
}
mostRecentBuild := builds[0]
switch mostRecentBuild.status {
case "Success":
return true, nil
case "Failure":
return true, fmt.Errorf("build failed")
case "Running", "Queued", "Waiting":
return false, nil
default:
slog.Warn("Unknown build status", "status", mostRecentBuild.status)
return false, fmt.Errorf("unknown build status: %s", mostRecentBuild.status)
}
2024-07-29 23:40:26 +02:00
}
2024-08-04 19:01:27 +02:00
func fetchBuilds(buildName string) ([]build, error) {
2024-07-29 23:40:26 +02:00
resp, err := http.Get(config.Configs.ActionsUrl)
if err != nil {
2024-08-04 19:01:27 +02:00
return nil, err
2024-07-29 23:40:26 +02:00
}
defer resp.Body.Close()
doc, err := html.Parse(resp.Body)
if err != nil {
2024-08-04 19:01:27 +02:00
return nil, err
2024-07-29 23:40:26 +02:00
}
2024-08-04 19:01:27 +02:00
var builds []build
var f func(*html.Node)
f = func(n *html.Node) {
2024-07-29 23:40:26 +02:00
if n.Type == html.ElementNode && n.Data == "div" {
for _, a := range n.Attr {
2024-07-30 02:24:18 +02:00
if a.Key == "class" && a.Val == "flex-item tw-items-center" {
2024-08-04 19:01:27 +02:00
if b := parseBuildBlock(n, buildName); b != nil {
builds = append(builds, *b)
2024-07-29 23:40:26 +02:00
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
2024-08-04 19:01:27 +02:00
f(c)
2024-07-29 23:40:26 +02:00
}
2024-07-30 02:24:18 +02:00
}
2024-08-04 19:01:27 +02:00
f(doc)
2024-07-30 02:24:18 +02:00
sort.Slice(builds, func(i, j int) bool {
return builds[i].time.After(builds[j].time)
})
2024-08-04 19:01:27 +02:00
return builds, nil
}
2024-07-30 02:24:18 +02:00
2024-08-04 19:01:27 +02:00
type build struct {
status string
time time.Time
2024-07-29 23:40:26 +02:00
}
2024-08-04 19:01:27 +02:00
func parseBuildBlock(n *html.Node, buildName string) *build {
var title, status string
2024-07-30 02:24:18 +02:00
var buildTime time.Time
2024-07-29 23:40:26 +02:00
var f func(*html.Node)
f = func(n *html.Node) {
if n.Type == html.ElementNode {
switch n.Data {
case "a":
for _, a := range n.Attr {
if a.Key == "class" && a.Val == "flex-item-title" {
for _, attr := range n.Attr {
if attr.Key == "title" {
title = attr.Val
}
}
}
}
2024-07-30 02:24:18 +02:00
case "div":
2024-07-29 23:40:26 +02:00
for _, a := range n.Attr {
2024-07-30 02:24:18 +02:00
if a.Key == "class" && a.Val == "flex-item-leading" {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && c.Data == "span" {
for _, attr := range c.Attr {
if attr.Key == "data-tooltip-content" {
status = attr.Val
}
}
}
2024-07-29 23:40:26 +02:00
}
}
}
2024-07-30 02:24:18 +02:00
case "relative-time":
for _, a := range n.Attr {
if a.Key == "datetime" {
buildTime, _ = time.Parse(time.RFC3339, a.Val)
}
}
2024-07-29 23:40:26 +02:00
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
f(c)
}
}
f(n)
2024-08-04 19:01:27 +02:00
if title == buildName {
return &build{status: status, time: buildTime}
}
return nil
2024-07-29 23:40:26 +02:00
}