From e1c49febd045f42b5363f03f6b9f4875554e898f Mon Sep 17 00:00:00 2001 From: ferreo Date: Sun, 28 Jul 2024 19:59:50 +0100 Subject: [PATCH] Initial Commit --- .gitignore | 5 + README | 5 + auth/auth.go | 126 ++++++++++ config/config.go | 101 ++++++++ db/db.go | 42 ++++ db/repository.go | 141 +++++++++++ deb/format.go | 307 +++++++++++++++++++++++ domain/packages.go | 74 ++++++ domain/user.go | 14 ++ example_config_json | 77 ++++++ example_users_json | 6 + fastmap/fastmap.go | 140 +++++++++++ go.mod | 34 +++ go.sum | 63 +++++ handlers/auth/login.go | 43 ++++ handlers/auth/register.go | 36 +++ handlers/auth/updatePassword.go | 35 +++ handlers/packages/counts.go | 11 + handlers/packages/packages.go | 70 ++++++ helpers/db.go | 5 + main.go | 47 ++++ middleware/auth.go | 34 +++ packages/packages.go | 430 ++++++++++++++++++++++++++++++++ server.go | 93 +++++++ 24 files changed, 1939 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100644 auth/auth.go create mode 100644 config/config.go create mode 100644 db/db.go create mode 100644 db/repository.go create mode 100644 deb/format.go create mode 100644 domain/packages.go create mode 100644 domain/user.go create mode 100644 example_config_json create mode 100644 example_users_json create mode 100644 fastmap/fastmap.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 handlers/auth/login.go create mode 100644 handlers/auth/register.go create mode 100644 handlers/auth/updatePassword.go create mode 100644 handlers/packages/counts.go create mode 100644 handlers/packages/packages.go create mode 100644 helpers/db.go create mode 100644 main.go create mode 100644 middleware/auth.go create mode 100644 packages/packages.go create mode 100644 server.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3c23ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +users.json +pikabldr.db +config.json +apicalls/ +brunel \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..e8aa600 --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +Copy the example config and user json files and name them config.json and user.json +Fill in what you need + +go build -o brunel +./brunel \ No newline at end of file diff --git a/auth/auth.go b/auth/auth.go new file mode 100644 index 0000000..30d4d04 --- /dev/null +++ b/auth/auth.go @@ -0,0 +1,126 @@ +package auth + +import ( + "brunel/config" + "brunel/domain" + "brunel/helpers" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "golang.org/x/crypto/argon2" +) + +type Params struct { + Memory uint32 + Iterations uint32 + Parallelism uint8 + KeyLength uint32 +} + +func NewPasswordHash(password string) string { + params := &Params{ + Memory: 64 * 1024, + Iterations: 3, + Parallelism: 2, + KeyLength: 32, + } + salt := []byte(config.Configs.Salt) + hash := argon2.IDKey([]byte(password), salt, params.Iterations, params.Memory, params.Parallelism, params.KeyLength) + return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$%s$%s", params.Memory, params.Iterations, params.Parallelism, base64.RawURLEncoding.EncodeToString(salt), base64.RawURLEncoding.EncodeToString(hash)) +} + +func VerifyPassword(username, password string) (bool, error) { + var user domain.User + user, err := helpers.DBInst.GetUser(username) + if err != nil { + usr, ok := config.Users[username] + if !ok { + return false, errors.New("user not found") + } + user = usr + } + + encodedHash := user.PasswordHash + _, _, _, _, salt, hash, err := decodeHash(encodedHash) + if err != nil { + slog.Error(err.Error()) + return false, err + } + params := &Params{ + Memory: 64 * 1024, + Iterations: 3, + Parallelism: 2, + KeyLength: 32, + } + + newHash := argon2.IDKey([]byte(password), salt, params.Iterations, params.Memory, params.Parallelism, params.KeyLength) + if subtle.ConstantTimeCompare(hash, newHash) == 1 { + return true, nil + } + return false, nil +} + +func GenerateAndStoreSessionToken(username string) (string, error) { + token := make([]byte, 32) + _, err := rand.Read(token) + if err != nil { + return "", err + } + + endcodedToken := base64.RawURLEncoding.EncodeToString(token) + helpers.DBInst.CreateSession(domain.Session{ + Token: endcodedToken, + Username: username, + Expiry: time.Now().Add(24 * time.Hour), + }) + return endcodedToken, nil +} + +func CheckSessionToken(token string) (bool, string) { + session, err := helpers.DBInst.GetSession(token) + if err != nil { + return false, "" + } + return true, session.Username +} + +func decodeHash(encodedHash string) (version int, memory, iterations, parallelism uint32, salt, hash []byte, err error) { + parts := strings.Split(encodedHash, "$") + if len(parts) != 6 { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("invalid hash format") + } + + if parts[1] != "argon2id" { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("unsupported hash algorithm") + } + + var v int + _, err = fmt.Sscanf(parts[2], "v=%d", &v) + if err != nil { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("invalid version: %v", err) + } + version = v + + _, err = fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &iterations, ¶llelism) + if err != nil { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("invalid parameters: %v", err) + } + + salt, err = base64.RawURLEncoding.DecodeString(parts[4]) + if err != nil { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("invalid salt: %v", err) + } + + hash, err = base64.RawURLEncoding.DecodeString(parts[5]) + if err != nil { + return 0, 0, 0, 0, nil, nil, fmt.Errorf("invalid hash: %v", err) + } + + return version, memory, iterations, parallelism, salt, hash, nil +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..e19886e --- /dev/null +++ b/config/config.go @@ -0,0 +1,101 @@ +package config + +import ( + "brunel/domain" + "fmt" + "io" + "os" + + "github.com/goccy/go-json" +) + +var Users map[string]domain.User +var Configs Config + +// Struct representing an individual package file entries +type PackageFile struct { + Name string `json:"name"` + Url string `json:"url"` + Subrepos []string `json:"subrepos"` + Priority int `json:"priority"` + UseWhitelist bool `json:"usewhitelist"` + Whitelist []string `json:"whitelist"` + Blacklist []string `json:"blacklist"` + Packagepath string `json:"packagepath"` + Compression string `json:"compression"` +} + +// Struct for the overall configuration +type Config struct { + Hostname string `json:"hostname"` + Port int `json:"port"` + UpstreamFallback bool `json:"upstreamFallback"` + LocalPackageFiles []PackageFile `json:"localPackageFiles"` + ExternalPackageFiles []PackageFile `json:"externalPackageFiles"` + LTOBlocklist []string `json:"ltoBlocklist"` + DeboutputDir string `json:"deboutputDir"` + Salt string `json:"salt"` +} + +func Init() error { + err := loadUsers() + if err != nil { + return err + } + + err = loadConfig() + if err != nil { + return err + } + + return nil +} + +func loadUsers() error { + jsonFile, err := os.Open("users.json") + if err != nil { + fmt.Println(err) + return err + } + defer jsonFile.Close() + + byteValue, _ := io.ReadAll(jsonFile) + + var users []domain.User + err = json.Unmarshal(byteValue, &users) + if err != nil { + fmt.Println(err) + return err + } + + var usersMap = make(map[string]domain.User, len(users)) + for _, user := range users { + usersMap[user.Username] = user + } + + Users = usersMap + + return nil +} + +func loadConfig() error { + jsonFile, err := os.Open("config.json") + if err != nil { + fmt.Println(err) + return err + } + defer jsonFile.Close() + + byteValue, _ := io.ReadAll(jsonFile) + + var config Config + err = json.Unmarshal(byteValue, &config) + if err != nil { + fmt.Println(err) + return err + } + + Configs = config + + return nil +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..97a67bc --- /dev/null +++ b/db/db.go @@ -0,0 +1,42 @@ +package db + +import ( + "brunel/domain" + "log" + "os" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func New() (*gorm.DB, error) { + newLogger := logger.New( + log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer + logger.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + LogLevel: logger.Silent, // Log level + IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger + ParameterizedQueries: true, // Don't include params in the SQL log + Colorful: true, // Disable color + }, + ) + + db, err := gorm.Open(sqlite.Open("pikabldr.db"), &gorm.Config{ + SkipDefaultTransaction: true, + CreateBatchSize: 1999, + Logger: newLogger, + }) + if err != nil { + panic("failed to connect database") + } + + db.AutoMigrate(&domain.User{}) + db.AutoMigrate(&domain.Session{}) + db.AutoMigrate(&domain.PackageInfo{}) + db.AutoMigrate(&domain.TimeContainer{}) + db.AutoMigrate(&domain.SourcePackageDTO{}) + + return db, nil +} diff --git a/db/repository.go b/db/repository.go new file mode 100644 index 0000000..b99f20a --- /dev/null +++ b/db/repository.go @@ -0,0 +1,141 @@ +package db + +import ( + "brunel/domain" + "brunel/fastmap" + "time" + + "github.com/jinzhu/now" + "gorm.io/gorm" +) + +const MaxBatchSize = 1999 + +type Repository struct { + db *gorm.DB +} + +func NewRepository(db *gorm.DB) *Repository { + return &Repository{db: db} +} + +func (r *Repository) GetUser(username string) (domain.User, error) { + var user domain.User + tx := r.db.Where("username = ?", username).First(&user) + return user, tx.Error +} + +func (r *Repository) CreateUser(user domain.User) error { + tx := r.db.Create(&user) + return tx.Error +} + +func (r *Repository) UpdateUser(user domain.User) error { + tx := r.db.Save(&user) + return tx.Error +} + +func (r *Repository) CreateSession(session domain.Session) error { + tx := r.db.Create(&session) + return tx.Error +} + +func (r *Repository) GetSession(token string) (domain.Session, error) { + now := now.BeginningOfMinute() + var session domain.Session + tx := r.db.Where("token = ? AND expiry > ?", token, now).First(&session) + return session, tx.Error +} + +func (r *Repository) DeleteSession(token string) error { + tx := r.db.Delete(&domain.Session{}, "token = ?", token) + return tx.Error +} + +func (r *Repository) DeleteExpiredSessions() error { + now := now.BeginningOfMinute() + tx := r.db.Where("expiry < ?", now).Delete(&domain.Session{}) + return tx.Error +} + +func (r *Repository) UpdatePackage(pkg domain.SourcePackage) error { + dto := sourcePackageToDto(pkg) + tx := r.db.Save(&dto) + return tx.Error +} + +func (r *Repository) GetPackages() ([]domain.SourcePackage, error) { + var packages []domain.SourcePackageDTO + tx := r.db.Find(&packages) + if tx.Error != nil { + return nil, tx.Error + } + var output []domain.SourcePackage + for _, v := range packages { + output = append(output, sourcePackageDtoToDomain(v)) + } + return output, nil +} + +func (r *Repository) GetPackage(name string) (domain.SourcePackage, error) { + var pkg domain.SourcePackageDTO + tx := r.db.Where("name = ?", name).First(&pkg) + if tx.Error != nil { + return domain.SourcePackage{}, tx.Error + } + return sourcePackageDtoToDomain(pkg), nil +} + +func (r *Repository) SavePackages(pkgs *fastmap.Fastmap[string, domain.SourcePackage]) error { + packs := make([]domain.SourcePackageDTO, 0) + pkgs.Iter(func(k string, v domain.SourcePackage) bool { + packs = append(packs, sourcePackageToDto(v)) + return true + }) + + for i := 0; i < len(packs); i += MaxBatchSize { + end := i + MaxBatchSize + length := len(packs) + if end > length { + end = length + } + tx := r.db.Save(packs[i:end]) + if tx.Error != nil { + return tx.Error + } + } + + return nil +} + +func (r *Repository) UpdateLastUpdateTime(time time.Time) error { + val := &domain.TimeContainer{ + Time: time, + ID: "lastupdatetime", + } + tx := r.db.Save(val) + return tx.Error +} + +func sourcePackageToDto(pkg domain.SourcePackage) domain.SourcePackageDTO { + dto := domain.SourcePackageDTO{ + Name: pkg.Name, + Packages: make([]domain.PackageInfo, 0), + } + pkg.Packages.Iter(func(k string, v domain.PackageInfo) bool { + dto.Packages = append(dto.Packages, v) + return true + }) + return dto +} + +func sourcePackageDtoToDomain(dto domain.SourcePackageDTO) domain.SourcePackage { + pkg := domain.SourcePackage{ + Name: dto.Name, + Packages: fastmap.New[string, domain.PackageInfo](), + } + for _, v := range dto.Packages { + pkg.Packages.Set(v.PackageName, v) + } + return pkg +} diff --git a/deb/format.go b/deb/format.go new file mode 100644 index 0000000..95febe3 --- /dev/null +++ b/deb/format.go @@ -0,0 +1,307 @@ +package deb + +import ( + "bufio" + "errors" + "io" + "sort" + "strings" + "unicode" +) + +// Stanza or paragraph of Debian control file +type Stanza map[string]string + +// MaxFieldSize is maximum stanza field size in bytes +const MaxFieldSize = 2 * 1024 * 1024 + +// Canonical order of fields in stanza +// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504 +var ( + canonicalOrderRelease = []string{ + "Origin", + "Label", + "Archive", + "Suite", + "Version", + "Codename", + "Date", + "NotAutomatic", + "ButAutomaticUpgrades", + "Architectures", + "Architecture", + "Components", + "Component", + "Description", + "MD5Sum", + "SHA1", + "SHA256", + "SHA512", + } + + canonicalOrderBinary = []string{ + "Package", + "Essential", + "Status", + "Priority", + "Section", + "Installed-Size", + "Maintainer", + "Original-Maintainer", + "Architecture", + "Source", + "Version", + "Replaces", + "Provides", + "Depends", + "Pre-Depends", + "Recommends", + "Suggests", + "Conflicts", + "Breaks", + "Conffiles", + "Filename", + "Size", + "MD5Sum", + "MD5sum", + "SHA1", + "SHA256", + "SHA512", + "Description", + } + + canonicalOrderSource = []string{ + "Package", + "Source", + "Binary", + "Version", + "Priority", + "Section", + "Maintainer", + "Original-Maintainer", + "Build-Depends", + "Build-Depends-Indep", + "Build-Conflicts", + "Build-Conflicts-Indep", + "Architecture", + "Standards-Version", + "Format", + "Directory", + "Files", + } + canonicalOrderInstaller = []string{ + "", + } +) + +// Copy returns copy of Stanza +func (s Stanza) Copy() (result Stanza) { + result = make(Stanza, len(s)) + for k, v := range s { + result[k] = v + } + return +} + +func isMultilineField(field string, isRelease bool) bool { + switch field { + // file without a section + case "": + return true + case "Description": + return true + case "Files": + return true + case "Changes": + return true + case "Checksums-Sha1": + return true + case "Checksums-Sha256": + return true + case "Checksums-Sha512": + return true + case "Package-List": + return true + case "MD5Sum": + return isRelease + case "SHA1": + return isRelease + case "SHA256": + return isRelease + case "SHA512": + return isRelease + } + return false +} + +// Write single field from Stanza to writer. +// +// nolint: interfacer +func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) { + if !isMultilineField(field, isRelease) { + _, err = w.WriteString(field + ": " + value + "\n") + } else { + if field != "" && !strings.HasSuffix(value, "\n") { + value = value + "\n" + } + + if field != "Description" && field != "" { + value = "\n" + value + } + + if field != "" { + _, err = w.WriteString(field + ":" + value) + } else { + _, err = w.WriteString(value) + } + } + + return +} + +// WriteTo saves stanza back to stream, modifying itself on the fly +func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error { + canonicalOrder := canonicalOrderBinary + if isSource { + canonicalOrder = canonicalOrderSource + } + if isRelease { + canonicalOrder = canonicalOrderRelease + } + if isInstaller { + canonicalOrder = canonicalOrderInstaller + } + + for _, field := range canonicalOrder { + value, ok := s[field] + if ok { + delete(s, field) + err := writeField(w, field, value, isRelease) + if err != nil { + return err + } + } + } + + // no extra fields in installer + if !isInstaller { + // Print extra fields in deterministic order (alphabetical) + keys := make([]string, len(s)) + i := 0 + for field := range s { + keys[i] = field + i++ + } + sort.Strings(keys) + for _, field := range keys { + err := writeField(w, field, s[field], isRelease) + if err != nil { + return err + } + } + } + + return nil +} + +// Parsing errors +var ( + ErrMalformedStanza = errors.New("malformed stanza syntax") +) + +func canonicalCase(field string) string { + upper := strings.ToUpper(field) + switch upper { + case "SHA1", "SHA256", "SHA512": + return upper + case "MD5SUM": + return "MD5Sum" + case "NOTAUTOMATIC": + return "NotAutomatic" + case "BUTAUTOMATICUPGRADES": + return "ButAutomaticUpgrades" + } + + startOfWord := true + + return strings.Map(func(r rune) rune { + if startOfWord { + startOfWord = false + return unicode.ToUpper(r) + } + + if r == '-' { + startOfWord = true + } + + return unicode.ToLower(r) + }, field) +} + +// ControlFileReader implements reading of control files stanza by stanza +type ControlFileReader struct { + scanner *bufio.Scanner + isRelease bool + isInstaller bool +} + +// NewControlFileReader creates ControlFileReader, it wraps with buffering +func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader { + scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768)) + scnr.Buffer(nil, MaxFieldSize) + + return &ControlFileReader{ + scanner: scnr, + isRelease: isRelease, + isInstaller: isInstaller, + } +} + +// ReadStanza reeads one stanza from control file +func (c *ControlFileReader) ReadStanza() (Stanza, error) { + stanza := make(Stanza, 32) + lastField := "" + lastFieldMultiline := c.isInstaller + + for c.scanner.Scan() { + line := c.scanner.Text() + + // Current stanza ends with empty line + if line == "" { + if len(stanza) > 0 { + return stanza, nil + } + continue + } + + if line[0] == ' ' || line[0] == '\t' || c.isInstaller { + if lastFieldMultiline { + stanza[lastField] += line + "\n" + } else { + stanza[lastField] += " " + strings.TrimSpace(line) + } + } else { + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + return nil, ErrMalformedStanza + } + lastField = canonicalCase(parts[0]) + lastFieldMultiline = isMultilineField(lastField, c.isRelease) + if lastFieldMultiline { + stanza[lastField] = parts[1] + if parts[1] != "" { + stanza[lastField] += "\n" + } + } else { + stanza[lastField] = strings.TrimSpace(parts[1]) + } + } + } + if err := c.scanner.Err(); err != nil { + return nil, err + } + if len(stanza) > 0 { + return stanza, nil + } + return nil, nil +} diff --git a/domain/packages.go b/domain/packages.go new file mode 100644 index 0000000..939eb11 --- /dev/null +++ b/domain/packages.go @@ -0,0 +1,74 @@ +package domain + +import ( + "brunel/fastmap" + "time" +) + +type PackagesCount struct { + Stale int `json:"stale"` + Missing int `json:"missing"` + Current int `json:"built"` + Error int `json:"error"` + Queued int `json:"queued"` + Building int `json:"building"` +} + +type BuildQueue *fastmap.Fastmap[string, BuildQueueItem] + +type BuildQueueItem struct { + Source SourcePackage + Status BuildStatus + Patch bool + LTO bool + Rebuild bool + BuildNumber int + BuildVersion string + I386 bool +} + +type SourcePackage struct { + Name string `gorm:"primarykey"` + Packages *fastmap.Fastmap[string, PackageInfo] `gorm:"foreignKey:PackageInfo;references:PackageName"` +} + +type SourcePackageDTO struct { + Name string `gorm:"primarykey"` + Packages []PackageInfo `gorm:"foreignKey:PackageName"` +} + +type PackageInfo struct { + PackageName string `gorm:"primarykey"` + Version string + Source string + Architecture string + Description string + Status PackageStatus + NewVersion string + LastBuildStatus PackageStatus +} + +type PackageStatus string +type BuildStatus string + +const ( + // Package is built + Built PackageStatus = "Built" + // Package is stale + Stale PackageStatus = "Stale" + // Package build errored out + Error PackageStatus = "Error" + // Package is being missing + Missing PackageStatus = "Missing" + // Package is upto date + Current PackageStatus = "Current" + // Package is queued for building + Queued BuildStatus = "Queued" + // Package is being built + Building BuildStatus = "Building" +) + +type TimeContainer struct { + ID string + Time time.Time +} diff --git a/domain/user.go b/domain/user.go new file mode 100644 index 0000000..1b6ff18 --- /dev/null +++ b/domain/user.go @@ -0,0 +1,14 @@ +package domain + +import "time" + +type User struct { + Username string `json:"username"` + PasswordHash string `json:"passwordHash"` +} + +type Session struct { + Token string `json:"token"` + Username string `json:"user"` + Expiry time.Time `json:"expiry"` +} diff --git a/example_config_json b/example_config_json new file mode 100644 index 0000000..965a87d --- /dev/null +++ b/example_config_json @@ -0,0 +1,77 @@ +{ + "hostname": "127.0.0.1", + "port": 7555, + "deboutputDir": "/srv/www/canary-incoming/", + "upstreamFallback": true, + "ltoBlocklist": [ + "biber", + "publican", + "xserver-xorg-video-intel", + "debian-package-book-de", + "debian-package-scripts", + "libcoq-iris", + "libproc-pid-file-perl" + ], + "localPackageFiles": [ + { + "name": "PikaOS", + "url": "https://ppa.pika-os.com/dists/pika/", + "subrepos": [ + "nest", "canary", "parrot", "pigeon", "raven", "cockatiel" + ], + "priority": 500, + "blacklist": [], + "usewhitelist": false, + "whitelist": [], + "packagepath": "/binary-amd64/Packages", + "compression": "bz2" + } + ], + "externalPackageFiles": [ + { + "name": "Debian sid", + "url": "http://ftp.debian.org/debian/dists/sid/", + "subrepos": [ + "main", "contrib", "non-free", "non-free-firmware" + ], + "priority": 350, + "blacklist": ["acl2"], + "usewhitelist": false, + "whitelist": [], + "packagepath": "/binary-amd64/Packages", + "compression": "xz" + }, + { + "name": "Debian experimental", + "url": "http://ftp.debian.org/debian/dists/experimental/", + "subrepos": [ + "main", "contrib", "non-free", "non-free-firmware" + ], + "priority": 400, + "usewhitelist": true, + "whitelist": [ + "plasma-workspace-data", + "plasma-workspace-dev" + ], + "blacklist": [ + "libwebrtc-audio-processing", "libhsa-runtime64", "hipcc", "rocm" + ], + "packagepath": "/binary-amd64/Packages", + "compression": "xz" + }, + { + "name": "Debian multimedia", + "url": "https://debian-mirrors.sdinet.de/deb-multimedia/dists/sid/", + "subrepos": [ + "main", "non-free" + ], + "usewhitelist": false, + "whitelist": [], + "priority": 450, + "blacklist": [], + "packagepath": "/binary-amd64/Packages", + "compression": "xz" + } + ], + "salt": "sdfwefwfwfwefwef" +} diff --git a/example_users_json b/example_users_json new file mode 100644 index 0000000..53e3c90 --- /dev/null +++ b/example_users_json @@ -0,0 +1,6 @@ +[ + { + "username": "testuser", + "passwordHash": "$argon2id$v=19$m=65536,t=3,p=2$MHVsRnHuN8hUcTE5$lC3bWNpyjYAcBrhSNx31GW_k-P6gd4GcOM4WJIbYii4" + } +] \ No newline at end of file diff --git a/fastmap/fastmap.go b/fastmap/fastmap.go new file mode 100644 index 0000000..aaee166 --- /dev/null +++ b/fastmap/fastmap.go @@ -0,0 +1,140 @@ +package fastmap + +import ( + "fmt" + "strings" + + "github.com/goccy/go-json" +) + +type Fastmap[K comparable, V any] struct { + idx map[K]int + store []fastmapValue[K, V] +} + +type fastmapValue[K comparable, V any] struct { + Key K + Value V +} + +func New[K comparable, V any]() *Fastmap[K, V] { + return &Fastmap[K, V]{ + idx: make(map[K]int), + store: make([]fastmapValue[K, V], 0), + } +} + +func (m *Fastmap[K, V]) Set(key K, value V) { + if _, ok := m.idx[key]; ok { + m.store[m.idx[key]].Value = value + return + } + m.idx[key] = len(m.store) + m.store = append(m.store, fastmapValue[K, V]{Key: key, Value: value}) +} + +func (m *Fastmap[K, V]) Get(key K) (value V, ok bool) { + idx, ok := m.idx[key] + if !ok { + return + } + return m.store[idx].Value, true +} + +func (m *Fastmap[K, V]) Delete(key K) { + idx, ok := m.idx[key] + if !ok { + return + } + delete(m.idx, key) + m.store[idx] = m.store[len(m.store)-1] + m.store = m.store[:len(m.store)-1] +} + +func (m *Fastmap[K, V]) Has(key K) bool { + _, ok := m.idx[key] + return ok +} + +func (m *Fastmap[K, V]) Len() int { + return len(m.idx) +} + +func (m *Fastmap[K, V]) GetPage(pageNum int, pageSize int) *Fastmap[K, V] { + start := pageSize * pageNum + end := start + pageSize + if end > len(m.store) { + end = len(m.store) + } + + returnVal := New[K, V]() + for i := start; i < end; i++ { + returnVal.Set(m.store[i].Key, m.store[i].Value) + } + return returnVal +} + +func (m *Fastmap[K, V]) Clear() { + m.idx = make(map[K]int) + m.store = make([]fastmapValue[K, V], 0) +} + +func (m *Fastmap[K, V]) Iter(fn func(key K, value V) bool) { + for _, v := range m.store { + if !fn(v.Key, v.Value) { + break + } + } +} + +func (m *Fastmap[K, V]) MarshalText() ([]byte, error) { + var builder strings.Builder + for _, v := range m.store { + builder.WriteString(fmt.Sprintf("%v:%v\n", v.Key, v.Value)) + } + return []byte(builder.String()), nil +} + +func (m *Fastmap[K, V]) UnmarshalText(text []byte) error { + m.Clear() + lines := strings.Split(string(text), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid format: %s", line) + } + var key K + var value V + if _, err := fmt.Sscan(parts[0], &key); err != nil { + return fmt.Errorf("error parsing key: %v", err) + } + if _, err := fmt.Sscan(parts[1], &value); err != nil { + return fmt.Errorf("error parsing value: %v", err) + } + m.Set(key, value) + } + return nil +} + +func (m *Fastmap[K, V]) MarshalJSON() ([]byte, error) { + temp := make(map[K]V) + for _, v := range m.store { + temp[v.Key] = v.Value + } + return json.Marshal(temp) +} + +func (m *Fastmap[K, V]) UnmarshalJSON(data []byte) error { + temp := make(map[K]V) + if err := json.Unmarshal(data, &temp); err != nil { + return err + } + m.Clear() + for k, v := range temp { + m.Set(k, v) + } + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..885e920 --- /dev/null +++ b/go.mod @@ -0,0 +1,34 @@ +module brunel + +go 1.22.5 + +require ( + github.com/goccy/go-json v0.10.3 + github.com/gofiber/fiber/v2 v2.52.5 + github.com/jinzhu/now v1.1.5 + github.com/klauspost/compress v1.17.9 + github.com/samber/slog-fiber v1.16.0 + github.com/ulikunitz/xz v0.5.12 + golang.org/x/crypto v0.14.0 + gorm.io/driver/sqlite v1.5.6 + gorm.io/gorm v1.25.11 + pault.ag/go/debian v0.16.0 +) + +require ( + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f0ac750 --- /dev/null +++ b/go.sum @@ -0,0 +1,63 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= +github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/samber/slog-fiber v1.16.0 h1:7HUfFC1c4OhoU9pGhrUyIX+BK+4OVPJduuQiWM6fzzU= +github.com/samber/slog-fiber v1.16.0/go.mod h1:RQr46XiBUwVNgWTiAizSGBxV9IbOpGbMMEEsth05iXg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +pault.ag/go/debian v0.16.0 h1:fivXn/IO9rn2nzTGndflDhOkNU703Axs/StWihOeU2g= +pault.ag/go/debian v0.16.0/go.mod h1:JFl0XWRCv9hWBrB5MDDZjA5GSEs1X3zcFK/9kCNIUmE= diff --git a/handlers/auth/login.go b/handlers/auth/login.go new file mode 100644 index 0000000..6d9eb44 --- /dev/null +++ b/handlers/auth/login.go @@ -0,0 +1,43 @@ +package handlers_auth + +import ( + "brunel/auth" + "brunel/config" + "time" + + "github.com/gofiber/fiber/v2" +) + +func Login(c *fiber.Ctx) error { + + username := c.FormValue("username") + password := c.FormValue("password") + + ok, err := auth.VerifyPassword(username, password) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + if !ok { + return c.Status(fiber.StatusUnauthorized).SendString("Unauthorized") + } + + token, err := auth.GenerateAndStoreSessionToken(username) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + tokenCookie := fiber.Cookie{ + Name: "pt", + Value: token + ":" + username, + Domain: config.Configs.Hostname, + Expires: time.Now().Add(24 * time.Hour), + Secure: true, + HTTPOnly: true, + SameSite: "lax", + } + + c.Cookie(&tokenCookie) + + return c.Status(fiber.StatusOK).SendString("Login") +} diff --git a/handlers/auth/register.go b/handlers/auth/register.go new file mode 100644 index 0000000..8343c63 --- /dev/null +++ b/handlers/auth/register.go @@ -0,0 +1,36 @@ +package handlers_auth + +import ( + "brunel/auth" + "brunel/domain" + "brunel/helpers" + + "github.com/gofiber/fiber/v2" +) + +func Register(c *fiber.Ctx) error { + + username := c.FormValue("username") + password := c.FormValue("password") + passwordConfirm := c.FormValue("passwordConfirm") + + _, err := helpers.DBInst.GetUser(username) + if err == nil { + return c.Status(fiber.StatusBadRequest).SendString("Username already taken") + } + + if password != passwordConfirm { + return c.Status(fiber.StatusBadRequest).SendString("Passwords do not match") + } + + pw := auth.NewPasswordHash(password) + + user := domain.User{ + Username: username, + PasswordHash: pw, + } + + helpers.DBInst.CreateUser(user) + + return c.Status(fiber.StatusOK).SendString("User created") +} diff --git a/handlers/auth/updatePassword.go b/handlers/auth/updatePassword.go new file mode 100644 index 0000000..a30d4a2 --- /dev/null +++ b/handlers/auth/updatePassword.go @@ -0,0 +1,35 @@ +package handlers_auth + +import ( + "brunel/auth" + "brunel/helpers" + + "github.com/gofiber/fiber/v2" +) + +func UpdatePassword(c *fiber.Ctx) error { + + username := c.FormValue("username") + password := c.FormValue("password") + passwordConfirm := c.FormValue("passwordConfirm") + + user, err := helpers.DBInst.GetUser(username) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + if password != passwordConfirm { + return c.Status(fiber.StatusBadRequest).SendString("Passwords do not match") + } + + pw := auth.NewPasswordHash(password) + + user.PasswordHash = pw + + err = helpers.DBInst.UpdateUser(user) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + return c.Status(fiber.StatusOK).SendString("Password updated") +} diff --git a/handlers/packages/counts.go b/handlers/packages/counts.go new file mode 100644 index 0000000..51d34ab --- /dev/null +++ b/handlers/packages/counts.go @@ -0,0 +1,11 @@ +package handlers_packages + +import ( + "brunel/packages" + + "github.com/gofiber/fiber/v2" +) + +func Counts(c *fiber.Ctx) error { + return c.Status(fiber.StatusOK).JSON(packages.GetPackagesCount()) +} diff --git a/handlers/packages/packages.go b/handlers/packages/packages.go new file mode 100644 index 0000000..9f1efb3 --- /dev/null +++ b/handlers/packages/packages.go @@ -0,0 +1,70 @@ +package handlers_packages + +import ( + "brunel/domain" + "brunel/fastmap" + "brunel/packages" + "strings" + + "github.com/gofiber/fiber/v2" +) + +func Packages(c *fiber.Ctx) error { + pageNum := c.QueryInt("page") + pageSize := c.QueryInt("pageSize") + search := c.Query("search") + filter := c.Query("filter") + if pageNum == 0 { + pageNum = 1 + } + if pageSize == 0 { + pageSize = 250 + } + packs := packages.GetPackages() + + if filter != "" { + finalReturn := fastmap.New[string, domain.SourcePackage]() + filteredPacks := fastmap.New[string, domain.SourcePackage]() + packs.Iter(func(k string, source domain.SourcePackage) bool { + source.Packages.Iter(func(key string, value domain.PackageInfo) bool { + if value.Status == domain.PackageStatus(filter) { + filteredPacks.Set(k, source) + return false + } + return true + }) + return true + }) + if search != "" { + filteredPacks.Iter(func(k string, source domain.SourcePackage) bool { + source.Packages.Iter(func(key string, value domain.PackageInfo) bool { + if strings.Contains(key, search) { + finalReturn.Set(k, source) + return false + } + return true + }) + return true + }) + } else { + finalReturn = filteredPacks + } + return c.Status(fiber.StatusOK).JSON(finalReturn.GetPage(pageNum, pageSize)) + } + if search != "" { + finalReturn := fastmap.New[string, domain.SourcePackage]() + packs.Iter(func(k string, source domain.SourcePackage) bool { + source.Packages.Iter(func(key string, value domain.PackageInfo) bool { + if value.Status == domain.PackageStatus(filter) { + finalReturn.Set(k, source) + return false + } + return true + }) + return true + }) + return c.Status(fiber.StatusOK).JSON(finalReturn.GetPage(pageNum, pageSize)) + } + + return c.Status(fiber.StatusOK).JSON(packs.GetPage(pageNum, pageSize)) +} diff --git a/helpers/db.go b/helpers/db.go new file mode 100644 index 0000000..7e10fe2 --- /dev/null +++ b/helpers/db.go @@ -0,0 +1,5 @@ +package helpers + +import "brunel/db" + +var DBInst *db.Repository diff --git a/main.go b/main.go new file mode 100644 index 0000000..b049168 --- /dev/null +++ b/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "brunel/db" + "brunel/helpers" + "context" + "log/slog" + "os" + "os/signal" +) + +func main() { + datab, err := db.New() + if err != nil { + panic("failed to connect database") + } + + repo := db.NewRepository(datab) + helpers.DBInst = repo + + // Run your server. + ctx := context.Background() + + // trap Ctrl+C and call cancel on the context + ctx, cancel := context.WithCancel(ctx) + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + defer func() { + signal.Stop(c) + cancel() + }() + go func() { + select { + case <-c: + cancel() + case <-ctx.Done(): + } + }() + + go func() { + if err := runServer(ctx); err != nil { + slog.Error("Failed to start server!", "details", err.Error()) + os.Exit(1) + } + }() + <-ctx.Done() +} diff --git a/middleware/auth.go b/middleware/auth.go new file mode 100644 index 0000000..e9410c4 --- /dev/null +++ b/middleware/auth.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "brunel/auth" + "fmt" + "strings" + + "github.com/gofiber/fiber/v2" +) + +func NewAuth() fiber.Handler { + return func(c *fiber.Ctx) error { + tokenPlusUsername := c.Cookies("pt") + fmt.Println("cookie", tokenPlusUsername) + if tokenPlusUsername == "" { + return c.Status(fiber.StatusUnauthorized).SendString("Unauthorized") + } + + split := strings.Split(tokenPlusUsername, ":") + token := split[0] + username := split[1] + ok, suser := auth.CheckSessionToken(token) + if !ok { + fmt.Println("not ok") + return c.Status(fiber.StatusUnauthorized).SendString("Unauthorized") + } + if suser != username { + fmt.Println("not suser") + return c.Status(fiber.StatusUnauthorized).SendString("Unauthorized") + } + + return c.Next() + } +} diff --git a/packages/packages.go b/packages/packages.go new file mode 100644 index 0000000..fd5b411 --- /dev/null +++ b/packages/packages.go @@ -0,0 +1,430 @@ +package packages + +import ( + "brunel/config" + "brunel/deb" + "brunel/domain" + "brunel/fastmap" + "brunel/helpers" + "compress/bzip2" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strings" + "time" + + "pault.ag/go/debian/version" + + "github.com/klauspost/compress/gzip" + "github.com/ulikunitz/xz" +) + +var LastUpdateTime time.Time +var currentPackagesFastMap = fastmap.New[string, domain.SourcePackage]() +var buildQueue domain.BuildQueue = fastmap.New[string, domain.BuildQueueItem]() + +func ProcessPackages() error { + var internalPackages = fastmap.New[string, domain.SourcePackage]() + var externalPackages = fastmap.New[string, domain.SourcePackage]() + err := LoadInternalPackages(internalPackages) + if err != nil { + return err + } + err = LoadExternalPackages(externalPackages) + if err != nil { + return err + } + ProcessStalePackages(internalPackages, externalPackages) + ProcessMissingPackages(internalPackages, externalPackages) + + currentPackagesFastMap.Clear() + internalPackages.Iter(func(k string, v domain.SourcePackage) bool { + currentPackagesFastMap.Set(k, v) + return true + }) + + LastUpdateTime = time.Now() + err = SaveToDb() + if err != nil { + return err + } + + return nil +} + +func GetBuildQueue() domain.BuildQueue { + return buildQueue +} + +func GetPackages() *fastmap.Fastmap[string, domain.SourcePackage] { + return currentPackagesFastMap +} + +func UpdatePackage(pkg domain.PackageInfo) error { + curr, ok := currentPackagesFastMap.Get(pkg.Source) + if !ok { + return fmt.Errorf("package %s not found", pkg.Source) + } + curr.Packages.Set(pkg.PackageName, pkg) + currentPackagesFastMap.Set(pkg.Source, curr) + return saveSingleToDb(curr) +} + +func IsBuilt(pkg domain.PackageInfo) bool { + curr, ok := currentPackagesFastMap.Get(pkg.Source) + if !ok { + return false + } + pk, ok := curr.Packages.Get(pkg.PackageName) + if !ok { + return false + } + return pk.Status == domain.Built || pk.Status == domain.Current +} + +func saveSingleToDb(pkg domain.SourcePackage) error { + err := helpers.DBInst.UpdatePackage(pkg) + if err != nil { + return err + } + LastUpdateTime = time.Now() + err = helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) + if err != nil { + return err + } + return nil +} + +func SaveToDb() error { + err := helpers.DBInst.SavePackages(currentPackagesFastMap) + if err != nil { + slog.Error(err.Error()) + return err + } + LastUpdateTime = time.Now() + return helpers.DBInst.UpdateLastUpdateTime(LastUpdateTime) +} + +func LoadFromDb() error { + packages, err := helpers.DBInst.GetPackages() + if err != nil { + slog.Error(err.Error()) + return nil + } + slices.SortStableFunc(packages, func(a, b domain.SourcePackage) int { + if a.Name == b.Name { + return 0 + } + if a.Name > b.Name { + return 1 + } + return -1 + }) + currentPackagesFastMap.Clear() + for _, pkg := range packages { + currentPackagesFastMap.Set(pkg.Name, pkg) + } + return nil +} + +func LoadInternalPackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage]) error { + localPackageFile := config.Configs.LocalPackageFiles + slices.SortStableFunc(localPackageFile, func(a, b config.PackageFile) int { + if a.Priority == b.Priority { + return 0 + } + if a.Priority < b.Priority { + return 1 + } + return -1 + }) + + for _, pkg := range config.Configs.LocalPackageFiles { + for _, repo := range pkg.Subrepos { + packages, err := fetchPackageFile(pkg, repo) + if err != nil { + return err + } + packages.Iter(func(newKey string, newPkg domain.PackageInfo) bool { + pk, ok := internalPackages.Get(newPkg.Source) + if !ok { + newMap := fastmap.New[string, domain.PackageInfo]() + newMap.Set(newKey, newPkg) + internalPackages.Set(newPkg.Source, domain.SourcePackage{ + Name: newPkg.Source, + Packages: newMap, + }) + return true + } + pkg, ok := pk.Packages.Get(newKey) + if !ok { + pk.Packages.Set(newKey, newPkg) + return true + } + mVer, _ := version.Parse(pkg.Version) + extVer, _ := version.Parse(newPkg.Version) + cmpVal := version.Compare(extVer, mVer) + if cmpVal >= 0 { + pk.Packages.Set(newKey, newPkg) + return true + } + return true + }) + } + } + + return nil +} + +func LoadExternalPackages(externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) error { + externalPackageFile := config.Configs.ExternalPackageFiles + slices.SortStableFunc(externalPackageFile, func(a, b config.PackageFile) int { + if a.Priority == b.Priority { + return 0 + } + if a.Priority < b.Priority { + return -1 + } + return 1 + }) + + for _, pkg := range config.Configs.ExternalPackageFiles { + for _, repo := range pkg.Subrepos { + packages, err := fetchPackageFile(pkg, repo) + if err != nil { + return err + } + packages.Iter(func(k string, v domain.PackageInfo) bool { + pk, ok := externalPackages.Get(v.Source) + if !ok { + newMap := fastmap.New[string, domain.PackageInfo]() + newMap.Set(k, v) + externalPackages.Set(v.Source, domain.SourcePackage{ + Name: v.Source, + Packages: newMap, + }) + return true + } + pkg, ok := pk.Packages.Get(k) + if !ok { + pk.Packages.Set(k, v) + return true + } + mVer, _ := version.Parse(pkg.Version) + extVer, _ := version.Parse(v.Version) + cmpVal := version.Compare(extVer, mVer) + if cmpVal >= 0 { + pk.Packages.Set(k, v) + return true + } + return true + }) + } + } + + return nil +} + +func ProcessMissingPackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage], externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) { + externalPackages.Iter(func(k string, src domain.SourcePackage) bool { + _, ok := internalPackages.Get(k) + if !ok && src.Packages.Len() > 0 { + newStatus := domain.Missing + src.Packages.Iter(func(k string, v domain.PackageInfo) bool { + v.Status = newStatus + src.Packages.Set(k, v) + return true + }) + internalPackages.Set(k, src) + } + return true + }) +} + +func ProcessStalePackages(internalPackages *fastmap.Fastmap[string, domain.SourcePackage], externalPackages *fastmap.Fastmap[string, domain.SourcePackage]) { + externalPackages.Iter(func(newPackage string, newSource domain.SourcePackage) bool { + matchedPackage, ok := internalPackages.Get(newPackage) + if !ok || matchedPackage.Packages.Len() == 0 { + return true + } + matchedPackage.Packages.Iter(func(currentKey string, currentPackage domain.PackageInfo) bool { + if currentPackage.Status == domain.Missing { + return true + } + newSource.Packages.Iter(func(newKey string, newPackage domain.PackageInfo) bool { + if currentKey != newKey { + return true + } + newVersion := strings.Split(newPackage.Version, "+b")[0] + mVer, _ := version.Parse(currentPackage.Version) + extVer, _ := version.Parse(newVersion) + cmpVal := version.Compare(mVer, extVer) + if cmpVal < 0 { + currentPackage.Status = domain.Stale + currentPackage.NewVersion = extVer.String() + matchedPackage.Packages.Set(currentKey, currentPackage) + } + return false + }) + return true + }) + wasMissing := false + newSource.Packages.Iter(func(newKey string, newPackage domain.PackageInfo) bool { + found := false + matchedPackage.Packages.Iter(func(currentKey string, currentPackage domain.PackageInfo) bool { + if currentKey != newKey { + return true + } + found = true + return false + }) + if !found { + wasMissing = true + newPackage.Status = domain.Missing + newPackage.NewVersion = newPackage.Version + matchedPackage.Packages.Set(newKey, newPackage) + } + return true + }) + if wasMissing { + matchedPackage.Packages.Iter(func(k string, v domain.PackageInfo) bool { + if v.Status == domain.Missing { + return true + } + v.Status = domain.Missing + return true + }) + } + return true + }) +} + +func fetchPackageFile(pkg config.PackageFile, selectedRepo string) (*fastmap.Fastmap[string, domain.PackageInfo], error) { + resp, err := http.Get(pkg.Url + selectedRepo + "/" + pkg.Packagepath + "." + pkg.Compression) + if err != nil { + return nil, err + } + defer resp.Body.Close() + rdr := io.Reader(resp.Body) + if pkg.Compression == "bz2" { + r := bzip2.NewReader(resp.Body) + rdr = r + } + if pkg.Compression == "xz" { + r, err := xz.NewReader(resp.Body) + if err != nil { + return nil, err + } + rdr = r + } + if pkg.Compression == "gz" { + r, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + rdr = r + } + + packages := fastmap.New[string, domain.PackageInfo]() + sreader := deb.NewControlFileReader(rdr, false, false) + for { + stanza, err := sreader.ReadStanza() + if err != nil || stanza == nil { + break + } + + if stanza["Section"] == "debian-installer" { + continue + } + + name := stanza["Package"] + + useWhitelist := pkg.UseWhitelist && len(pkg.Whitelist) > 0 + if useWhitelist { + contained := nameContains(name, pkg.Whitelist) + if !contained { + continue + } + } + + broken := nameContains(name, pkg.Blacklist) + if broken { + continue + } + + ver, err := version.Parse(stanza["Version"]) + if err != nil { + return nil, err + } + + pk, ok := packages.Get(name) + if ok { + matchedVer, _ := version.Parse(pk.Version) + cmpVal := version.Compare(ver, matchedVer) + if cmpVal < 0 { + continue + } + } + + sourceSplit := strings.Split(stanza["Source"], " ") + source := sourceSplit[0] + if source == "" { + source = name + } + + packages.Set(name, domain.PackageInfo{ + PackageName: name, + Version: ver.String(), + Source: source, + Architecture: stanza["Architecture"], + Description: stanza["Description"], + Status: domain.Current, + }) + } + + return packages, nil +} + +func nameContains(name string, match []string) bool { + for _, m := range match { + if strings.Contains(name, m) { + return true + } + } + return false +} + +func GetPackagesCount() domain.PackagesCount { + count := domain.PackagesCount{ + Stale: 0, + Missing: 0, + Current: 0, + Error: 0, + Queued: 0, + Building: 0, + } + currentPackagesFastMap.Iter(func(k string, v domain.SourcePackage) bool { + v.Packages.Iter(func(k string, pkg domain.PackageInfo) bool { + switch pkg.Status { + case domain.Stale: + count.Stale++ + case domain.Missing: + count.Missing++ + case domain.Built: + count.Current++ + case domain.Current: + count.Current++ + case domain.Error: + count.Error++ + } + if pkg.LastBuildStatus == domain.Error { + count.Error++ + } + return false + }) + return true + }) + return count +} diff --git a/server.go b/server.go new file mode 100644 index 0000000..be25638 --- /dev/null +++ b/server.go @@ -0,0 +1,93 @@ +package main + +import ( + "brunel/config" + handlers_auth "brunel/handlers/auth" + handlers_packages "brunel/handlers/packages" + "brunel/helpers" + "brunel/middleware" + "brunel/packages" + "context" + "fmt" + "log/slog" + "os" + "time" + + "github.com/goccy/go-json" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/etag" + "github.com/gofiber/fiber/v2/middleware/recover" + slogfiber "github.com/samber/slog-fiber" +) + +// runServer runs a new HTTP server with the loaded environment variables. +func runServer(ctx context.Context) error { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + err := config.Init() + if err != nil { + slog.Error("unable to load configuration: " + err.Error()) + return err + } + + go func() { + for { + select { + case <-ctx.Done(): + return + default: + err := helpers.DBInst.DeleteExpiredSessions() + if err != nil { + slog.Error("unable to delete expired sessions: " + err.Error()) + } + time.Sleep(time.Minute) + } + } + }() + + start := time.Now() + err = packages.LoadFromDb() + if err != nil { + slog.Error("unable to load packages from db: " + err.Error()) + return err + } + slog.Info("packages loaded from db in " + time.Since(start).String()) + + start = time.Now() + err = packages.ProcessPackages() + if err != nil { + slog.Error("unable to process packages: " + err.Error()) + return err + } + slog.Info("packages processed in " + time.Since(start).String()) + + cfg := fiber.Config{ + JSONEncoder: json.Marshal, + JSONDecoder: json.Unmarshal, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + AppName: "brunel", + Prefork: false, + } + + server := fiber.New(cfg) + + server.Use(slogfiber.NewWithConfig(logger, slogfiber.Config{ + WithSpanID: true, + WithTraceID: true, + })) + server.Use(recover.New()) + server.Use(etag.New()) + + adminRoutes := server.Group("api/admin") + adminRoutes.Use(middleware.NewAuth()) + + server.Get("/api/counts", handlers_packages.Counts) + server.Get("/api/packages", handlers_packages.Packages) + + server.Post("/api/login", handlers_auth.Login) + adminRoutes.Post("/register", handlers_auth.Register) + adminRoutes.Post("/updatePassword", handlers_auth.UpdatePassword) + + return server.Listen(fmt.Sprintf(":%d", config.Configs.Port)) +}