package buildqueue import ( "brunel/config" "brunel/domain" "brunel/helpers" "brunel/packages" "context" "fmt" "log/slog" "net/http" "sort" "time" "golang.org/x/net/html" ) func StartPackageQueueWorker(ctx context.Context) { go func() { for { select { case <-ctx.Done(): return default: processPackages() time.Sleep(1 * time.Hour) } } }() } func StartQueueAndStatusWorker(ctx context.Context) { go processQueueAndStatus(ctx) } func processPackages() { if err := packages.ProcessPackages(); err != nil { slog.Error("unable to process packages", "error", err) return } packages.GetPackages().ForEach(func(k string, v domain.SourcePackage) bool { if !shouldBuild(v) { return true } state, _ := helpers.DBInst.GetBuildState(v.Name) if state.BuildNumber > 3 { return true } if state.BuildNumber > 1 && state.Type == string(domain.BuildTypeI386) { return true } queueBuildItem(v, state, false) if v.Has32bit { queueBuildItem(v, state, true) } return true }) } func queueBuildItem(v domain.SourcePackage, state domain.BuildState, isI386 bool) { buildItem := createBuildItem(v, state, isI386) if err := Add(buildItem); err != nil { slog.Info("unable to add package to queue", "error", err) } } func createBuildItem(v domain.SourcePackage, state domain.BuildState, isI386 bool) domain.BuildQueueItem { buildVersion := v.NewVersion if buildVersion == "" { buildVersion = v.Version } var typ domain.BuildType if isI386 { typ = domain.BuildTypeI386 } else { typ = domain.BuildTypeLTO if state.BuildNumber == 1 { typ = domain.BuildTypeNormal } else if state.BuildNumber == 2 { typ = domain.BuildTypeAmd64 } } return domain.BuildQueueItem{ Source: v, Status: domain.Queued, Type: typ, Patch: false, Rebuild: false, BuildNumber: state.BuildNumber, BuildVersion: buildVersion, } } func shouldBuild(v domain.SourcePackage) bool { return v.Status == domain.Missing || v.Status == domain.Stale || v.Status == domain.Error } func processQueueAndStatus(ctx context.Context) { for { select { case <-ctx.Done(): return default: processQueue() time.Sleep(10 * time.Second) } } } func processQueue() { q := GetQueue() itemsToRemove := []struct { name string buildType domain.BuildType }{} buildingFound := false q.ForEach(func(k string, item domain.BuildQueueItem) bool { if item.Status == domain.Building { buildingFound = true complete, err := checkIfBuildComplete(item) if complete { handleCompletedBuild(item, err) itemsToRemove = append(itemsToRemove, struct { name string buildType domain.BuildType }{ name: item.Source.Name, buildType: item.Type, }) } } return true }) for _, item := range itemsToRemove { Remove(item.name, item.buildType) } if !buildingFound { if err := ProcessNext(); err != nil { slog.Error("unable to process queue", "error", err) } } } func handleCompletedBuild(item domain.BuildQueueItem, err error) { status := domain.Built if err != nil { status = domain.Error } if item.Type != domain.BuildTypeI386 { if err != nil { item.Source.BuildAttempts++ } else { item.Source.BuildAttempts = 0 item.Source.Version = item.BuildVersion } item.Source.Status = status item.Source.LastBuildStatus = status updatePackageStatus(&item, status) packages.UpdateSourcePackage(item.Source) } updateBuildState(item, status) } func updatePackageStatus(item *domain.BuildQueueItem, status domain.PackageStatus) { item.Source.Packages.ForEach(func(k string, v domain.PackageInfo) bool { v.Status = status if status == domain.Current { v.Version = item.BuildVersion v.NewVersion = "" } item.Source.Packages.Set(k, v) return true }) } func updateBuildState(item domain.BuildQueueItem, buildStatus domain.PackageStatus) { state, err := helpers.DBInst.GetBuildState(item.Source.Name) if err != nil { state = domain.BuildState{ Name: item.Source.Name, Status: string(domain.Queued), BuildVersion: item.BuildVersion, Type: string(item.Type), BuildNumber: 0, } } state.Status = string(buildStatus) state.Type = string(item.Type) state.BuildVersion = item.BuildVersion if state.Status == string(domain.Built) { state.BuildNumber = 0 } else { state.BuildNumber++ } helpers.DBInst.UpdateBuildState(state) } func checkIfBuildComplete(item domain.BuildQueueItem) (bool, error) { builds, err := fetchBuilds(item.Source.Name + "=" + item.BuildVersion) if err != nil { return false, err } if len(builds) == 0 { slog.Info("No matching builds found", "buildName", item.Source.Name+"="+item.BuildVersion) return false, nil } mostRecentBuild := builds[0] switch mostRecentBuild.status { case "Success": return true, nil case "Failure": return true, fmt.Errorf("build failed") case "Running", "Queued", "Waiting": return false, nil default: slog.Warn("Unknown build status", "status", mostRecentBuild.status) return false, fmt.Errorf("unknown build status: %s", mostRecentBuild.status) } } func fetchBuilds(buildName string) ([]build, error) { resp, err := http.Get(config.Configs.ActionsUrl) if err != nil { return nil, err } defer resp.Body.Close() doc, err := html.Parse(resp.Body) if err != nil { return nil, err } var builds []build var f func(*html.Node) f = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "div" { for _, a := range n.Attr { if a.Key == "class" && a.Val == "flex-item tw-items-center" { if b := parseBuildBlock(n, buildName); b != nil { builds = append(builds, *b) } } } } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } f(doc) sort.Slice(builds, func(i, j int) bool { return builds[i].time.After(builds[j].time) }) return builds, nil } type build struct { status string time time.Time } func parseBuildBlock(n *html.Node, buildName string) *build { var title, status string var buildTime time.Time var f func(*html.Node) f = func(n *html.Node) { if n.Type == html.ElementNode { switch n.Data { case "a": for _, a := range n.Attr { if a.Key == "class" && a.Val == "flex-item-title" { for _, attr := range n.Attr { if attr.Key == "title" { title = attr.Val } } } } case "div": for _, a := range n.Attr { if a.Key == "class" && a.Val == "flex-item-leading" { for c := n.FirstChild; c != nil; c = c.NextSibling { if c.Type == html.ElementNode && c.Data == "span" { for _, attr := range c.Attr { if attr.Key == "data-tooltip-content" { status = attr.Val } } } } } } case "relative-time": for _, a := range n.Attr { if a.Key == "datetime" { buildTime, _ = time.Parse(time.RFC3339, a.Val) } } } } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } f(n) if title == buildName { return &build{status: status, time: buildTime} } return nil }