From 4c1aa120aa5b35afbb4aba67faaf52bb541a2be0 Mon Sep 17 00:00:00 2001 From: Erikas Date: Thu, 11 Jul 2024 19:33:15 +0300 Subject: [PATCH] adsf --- .gitignore | 2 +- benchmark_data.go | 312 +++++++++++++++++++++++++++++++++++++++ benchmarks.go | 15 +- cmd/fsmig1/main.go | 11 ++ csv.go | 256 -------------------------------- models.go | 12 +- static/js/benchmark.js | 0 templates/benchmark.tmpl | 119 ++++++++------- templates/header.tmpl | 1 + 9 files changed, 398 insertions(+), 330 deletions(-) create mode 100644 benchmark_data.go create mode 100644 cmd/fsmig1/main.go delete mode 100644 csv.go create mode 100644 static/js/benchmark.js diff --git a/.gitignore b/.gitignore index 159583b..3885e54 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,4 @@ go.work go.work.sum # Custom -/data +/data* diff --git a/benchmark_data.go b/benchmark_data.go new file mode 100644 index 0000000..af8a27f --- /dev/null +++ b/benchmark_data.go @@ -0,0 +1,312 @@ +package flightlesssomething + +import ( + "bufio" + "bytes" + "encoding/gob" + "errors" + "fmt" + "log" + "math/big" + "mime/multipart" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/dustin/go-humanize" + "github.com/klauspost/compress/zstd" +) + +type BenchmarkData struct { + Label string + + // Specs + SpecOS string + SpecGPU string + SpecCPU string + SpecRAM string + SpecLinuxKernel string + SpecLinuxScheduler string + + // Data + DataFPS []float64 + DataFrameTime []float64 + DataCPULoad []float64 + DataGPULoad []float64 + DataCPUTemp []float64 + DataGPUTemp []float64 + DataGPUCoreClock []float64 + DataGPUMemClock []float64 + DataGPUVRAMUsed []float64 + DataGPUPower []float64 + DataRAMUsed []float64 + DataSwapUsed []float64 +} + +// readBenchmarkFiles reads the uploaded benchmark files and returns a slice of BenchmarkData. +func readBenchmarkFiles(files []*multipart.FileHeader) ([]*BenchmarkData, error) { + csvFiles := make([]*BenchmarkData, 0) + linesCount := 0 + + for _, fileHeader := range files { + csvFile := BenchmarkData{} + + file, err := fileHeader.Open() + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + // Label is filename without extension + csvFile.Label = strings.TrimSuffix(fileHeader.Filename, ".csv") + csvFile.Label = strings.TrimSuffix(csvFile.Label, ".htm") + + // First line should contain this: os,cpu,gpu,ram,kernel,driver,cpuscheduler + if !scanner.Scan() { + return nil, errors.New("invalid CSV file (err 1)") + } + record := strings.Split(strings.TrimRight(scanner.Text(), ","), ",") + if len(record) != 7 { + return nil, errors.New("invalid CSV file (err 2)") + } + + // Second line should contain values + if !scanner.Scan() { + return nil, errors.New("invalid CSV file (err 3)") + } + record = strings.Split(scanner.Text(), ",") + + for i, v := range record { + switch i { + case 0: + csvFile.SpecOS = truncateString(strings.TrimSpace(v)) + case 1: + csvFile.SpecCPU = truncateString(strings.TrimSpace(v)) + case 2: + csvFile.SpecGPU = truncateString(strings.TrimSpace(v)) + case 3: + kilobytes := new(big.Int) + _, ok := kilobytes.SetString(strings.TrimSpace(v), 10) + if !ok { + return nil, errors.New("failed to convert RAM to big.Int") + } + bytes := new(big.Int).Mul(kilobytes, big.NewInt(1024)) + csvFile.SpecRAM = humanize.Bytes(bytes.Uint64()) + case 4: + csvFile.SpecLinuxKernel = truncateString(strings.TrimSpace(v)) + case 6: + csvFile.SpecLinuxScheduler = truncateString(strings.TrimSpace(v)) + } + } + + // 3rd line contain headers for benchmark data: fps,frametime,cpu_load,gpu_load,cpu_temp,gpu_temp,gpu_core_clock,gpu_mem_clock,gpu_vram_used,gpu_power,ram_used,swap_used,process_rss,elapsed + if !scanner.Scan() { + return nil, errors.New("invalid CSV file (err 5)") + } + record = strings.Split(strings.TrimRight(scanner.Text(), ","), ",") + if len(record) != 14 { + return nil, errors.New("invalid CSV file (err 6)") + } + + // Preallocate slices. First file will be inefficient, but later files will contain + // value of linesCount that would help to optimize preallocation. + csvFile.DataFPS = make([]float64, 0, linesCount) + csvFile.DataFrameTime = make([]float64, 0, linesCount) + csvFile.DataCPULoad = make([]float64, 0, linesCount) + csvFile.DataGPULoad = make([]float64, 0, linesCount) + csvFile.DataCPUTemp = make([]float64, 0, linesCount) + csvFile.DataGPUTemp = make([]float64, 0, linesCount) + csvFile.DataGPUCoreClock = make([]float64, 0, linesCount) + csvFile.DataGPUMemClock = make([]float64, 0, linesCount) + csvFile.DataGPUVRAMUsed = make([]float64, 0, linesCount) + csvFile.DataGPUPower = make([]float64, 0, linesCount) + csvFile.DataRAMUsed = make([]float64, 0, linesCount) + csvFile.DataSwapUsed = make([]float64, 0, linesCount) + + var counter uint + + for scanner.Scan() { + record = strings.Split(scanner.Text(), ",") + if len(record) != 14 { + return nil, errors.New("invalid CSV file (err 7)") + } + + val, err := strconv.ParseFloat(record[0], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse FPS value '%s': %v", record[0], err) + } + csvFile.DataFPS = append(csvFile.DataFPS, val) + + val, err = strconv.ParseFloat(record[1], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse frametime value '%s': %v", record[1], err) + } + csvFile.DataFrameTime = append(csvFile.DataFrameTime, val) + + val, err = strconv.ParseFloat(record[2], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse CPU load value '%s': %v", record[2], err) + } + csvFile.DataCPULoad = append(csvFile.DataCPULoad, val) + + val, err = strconv.ParseFloat(record[3], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU load value '%s': %v", record[3], err) + } + csvFile.DataGPULoad = append(csvFile.DataGPULoad, val) + + val, err = strconv.ParseFloat(record[4], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse CPU temp value '%s': %v", record[4], err) + } + csvFile.DataCPUTemp = append(csvFile.DataCPUTemp, val) + + val, err = strconv.ParseFloat(record[5], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU temp value '%s': %v", record[5], err) + } + csvFile.DataGPUTemp = append(csvFile.DataGPUTemp, val) + + val, err = strconv.ParseFloat(record[6], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU core clock value '%s': %v", record[6], err) + } + csvFile.DataGPUCoreClock = append(csvFile.DataGPUCoreClock, val) + + val, err = strconv.ParseFloat(record[7], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU mem clock value '%s': %v", record[7], err) + } + csvFile.DataGPUMemClock = append(csvFile.DataGPUMemClock, val) + + val, err = strconv.ParseFloat(record[8], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU VRAM used value '%s': %v", record[8], err) + } + csvFile.DataGPUVRAMUsed = append(csvFile.DataGPUVRAMUsed, val) + + val, err = strconv.ParseFloat(record[9], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse GPU power value '%s': %v", record[9], err) + } + csvFile.DataGPUPower = append(csvFile.DataGPUPower, val) + + val, err = strconv.ParseFloat(record[10], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse RAM used value '%s': %v", record[10], err) + } + csvFile.DataRAMUsed = append(csvFile.DataRAMUsed, val) + + val, err = strconv.ParseFloat(record[11], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse SWAP used value '%s': %v", record[11], err) + } + csvFile.DataSwapUsed = append(csvFile.DataSwapUsed, val) + + counter++ + if counter == 100000 { + return nil, errors.New("CSV file cannot have more than 100000 data lines") + } + } + + // Next file would be more efficient to preallocate slices + if linesCount < len(csvFile.DataFPS) { + linesCount = len(csvFile.DataFPS) + } + + if err := scanner.Err(); err != nil { + log.Println("error (4) parsing CSV:", err) + return nil, err + } + + if len(csvFile.DataFPS) == 0 && + len(csvFile.DataFrameTime) == 0 && + len(csvFile.DataCPULoad) == 0 && + len(csvFile.DataGPULoad) == 0 && + len(csvFile.DataCPUTemp) == 0 && + len(csvFile.DataGPUTemp) == 0 && + len(csvFile.DataGPUCoreClock) == 0 && + len(csvFile.DataGPUMemClock) == 0 && + len(csvFile.DataGPUVRAMUsed) == 0 && + len(csvFile.DataGPUPower) == 0 && + len(csvFile.DataRAMUsed) == 0 && + len(csvFile.DataSwapUsed) == 0 { + return nil, errors.New("empty CSV file (err 8)") + } + + csvFiles = append(csvFiles, &csvFile) + } + + return csvFiles, nil +} + +// truncateString truncates the input string to a maximum of 100 characters and appends "..." if it exceeds that length. +func truncateString(s string) string { + const maxLength = 100 + if len(s) > maxLength { + return s[:maxLength] + "..." + } + return s +} + +func storeBenchmarkData(csvFiles []*BenchmarkData, benchmarkID uint) error { + // Store to disk + filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + // Convert to []byte + var buffer bytes.Buffer + gobEncoder := gob.NewEncoder(&buffer) + err = gobEncoder.Encode(csvFiles) + if err != nil { + return err + } + + // Compress and write to file + zstdEncoder, err := zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest)) + if err != nil { + return err + } + defer zstdEncoder.Close() + _, err = zstdEncoder.Write(buffer.Bytes()) + return err +} + +func retrieveBenchmarkData(benchmarkID uint) (csvFiles []*BenchmarkData, err error) { + filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + // Decompress and read from file + zstdDecoder, err := zstd.NewReader(file) + if err != nil { + return nil, err + } + defer zstdDecoder.Close() + + var buffer bytes.Buffer + _, err = buffer.ReadFrom(zstdDecoder) + if err != nil { + return nil, err + } + + // Decode + gobDecoder := gob.NewDecoder(&buffer) + err = gobDecoder.Decode(&csvFiles) + return csvFiles, err +} + +func deleteBenchmarkData(benchmarkID uint) error { + filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) + return os.Remove(filePath) +} diff --git a/benchmarks.go b/benchmarks.go index e371329..15717c1 100644 --- a/benchmarks.go +++ b/benchmarks.go @@ -181,7 +181,7 @@ func postBenchmarkCreate(c *gin.Context) { // Read CSV files // Store to disk only when DB record is created successfully - csvFiles, csvSpecs, err := readCSVFiles(files) + csvFiles, err := readBenchmarkFiles(files) if err != nil { c.HTML(http.StatusUnauthorized, "error.tmpl", gin.H{ "activePage": "error", @@ -197,13 +197,6 @@ func postBenchmarkCreate(c *gin.Context) { UserID: session.Get("ID").(uint), Title: title, Description: description, - - SpecDistro: csvSpecs.Distro, - SpecCPU: csvSpecs.CPU, - SpecGPU: csvSpecs.GPU, - SpecRAM: csvSpecs.RAM, - SpecKernel: csvSpecs.Kernel, - SpecScheduler: csvSpecs.Scheduler, } result := db.Create(&benchmark) @@ -331,7 +324,7 @@ func getBenchmark(c *gin.Context) { var benchmark Benchmark benchmark.ID = uint(intID) - var csvFiles []*CSVFile + var benchmarkDatas []*BenchmarkData var errCSV, errDB error errHTTPStatus := http.StatusInternalServerError @@ -340,7 +333,7 @@ func getBenchmark(c *gin.Context) { go func() { defer wg.Done() - csvFiles, errCSV = retrieveBenchmarkData(benchmark.ID) + benchmarkDatas, errCSV = retrieveBenchmarkData(benchmark.ID) }() go func() { @@ -379,6 +372,6 @@ func getBenchmark(c *gin.Context) { "userID": session.Get("ID"), "benchmark": benchmark, - "benchmarkData": csvFiles, + "benchmarkData": benchmarkDatas, }) } diff --git a/cmd/fsmig1/main.go b/cmd/fsmig1/main.go new file mode 100644 index 0000000..4f2f945 --- /dev/null +++ b/cmd/fsmig1/main.go @@ -0,0 +1,11 @@ +package main + +import "fmt" + +/* + This app is a migration tool from v0.0.7 to v0.0.8 +*/ + +func main() { + fmt.Println("Hello, World!") +} diff --git a/csv.go b/csv.go deleted file mode 100644 index 33ee177..0000000 --- a/csv.go +++ /dev/null @@ -1,256 +0,0 @@ -package flightlesssomething - -import ( - "bufio" - "bytes" - "encoding/gob" - "errors" - "fmt" - "log" - "math/big" - "mime/multipart" - "os" - "path/filepath" - "strings" - - "github.com/dustin/go-humanize" - "github.com/klauspost/compress/zstd" -) - -type CSVFile struct { - Filename string - - FPSPointsArray string - FrameTimeArray string - CPULoadArray string - GPULoadArray string - CPUTempArray string - GPUTempArray string - GPUCoreClockArray string - GPUMemClockArray string - GPUVRAMUsedArray string - GPUPowerArray string - RAMUsedArray string - SwapUsedArray string -} - -type CSVSpecs struct { - MaxPoints int - - Distro string - Kernel string - GPU string - CPU string - RAM string - Scheduler string -} - -// readCSVFiles reads multiple CSV files and returns a slice of CSVFile pointers and the maximum number of FPS records found in any file -func readCSVFiles(files []*multipart.FileHeader) ([]*CSVFile, *CSVSpecs, error) { - csvFiles := make([]*CSVFile, 0) - csvSpecs := &CSVSpecs{} - - var linesCount int - - for _, fileHeader := range files { - csvFile := CSVFile{} - - file, err := fileHeader.Open() - if err != nil { - return nil, nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - // Set file name (without extension) - csvFile.Filename = strings.TrimSuffix(fileHeader.Filename, ".csv") - - // First line should contain this: os,cpu,gpu,ram,kernel,driver,cpuscheduler - if !scanner.Scan() { - return nil, nil, errors.New("invalid CSV file (err 1)") - } - record := strings.Split(strings.TrimRight(scanner.Text(), ","), ",") - if len(record) != 7 { - return nil, nil, errors.New("invalid CSV file (err 2)") - } - - // Second line should contain values - if !scanner.Scan() { - return nil, nil, errors.New("invalid CSV file (err 3)") - } - record = strings.Split(scanner.Text(), ",") - - for i, v := range record { - switch i { - case 0: - csvSpecs.Distro = truncateString(strings.TrimSpace(v)) - case 1: - csvSpecs.CPU = truncateString(strings.TrimSpace(v)) - case 2: - csvSpecs.GPU = truncateString(strings.TrimSpace(v)) - case 3: - kilobytes := new(big.Int) - _, ok := kilobytes.SetString(strings.TrimSpace(v), 10) - if !ok { - return nil, nil, errors.New("failed to convert RAM to big.Int") - } - bytes := new(big.Int).Mul(kilobytes, big.NewInt(1024)) - csvSpecs.RAM = humanize.Bytes(bytes.Uint64()) - case 4: - csvSpecs.Kernel = truncateString(strings.TrimSpace(v)) - case 6: - csvSpecs.Scheduler = truncateString(strings.TrimSpace(v)) - } - } - - // 3rd line contain headers for benchmark data: fps,frametime,cpu_load,gpu_load,cpu_temp,gpu_temp,gpu_core_clock,gpu_mem_clock,gpu_vram_used,gpu_power,ram_used,swap_used,process_rss,elapsed - if !scanner.Scan() { - return nil, nil, errors.New("invalid CSV file (err 5)") - } - record = strings.Split(strings.TrimRight(scanner.Text(), ","), ",") - if len(record) != 14 { - return nil, nil, errors.New("invalid CSV file (err 6)") - } - - fpsPoints := make([]string, 0, linesCount) - frametimePoints := make([]string, 0, linesCount) - cpuLoadPoints := make([]string, 0, linesCount) - gpuLoadPoints := make([]string, 0, linesCount) - cpuTempPoints := make([]string, 0, linesCount) - gpuTempPoints := make([]string, 0, linesCount) - gpuCoreClockPoints := make([]string, 0, linesCount) - gpuMemClockPoints := make([]string, 0, linesCount) - gpuVRAMUsedPoints := make([]string, 0, linesCount) - gpuPowerPoints := make([]string, 0, linesCount) - RAMUsedPoints := make([]string, 0, linesCount) - SWAPUsedPoints := make([]string, 0, linesCount) - - var counter uint - - for scanner.Scan() { - record = strings.Split(scanner.Text(), ",") - if len(record) != 14 { - return nil, nil, errors.New("invalid CSV file (err 7)") - } - fpsPoints = append(fpsPoints, record[0]) - frametimePoints = append(frametimePoints, record[1]) - cpuLoadPoints = append(cpuLoadPoints, record[2]) - gpuLoadPoints = append(gpuLoadPoints, record[3]) - cpuTempPoints = append(cpuTempPoints, record[4]) - gpuTempPoints = append(gpuTempPoints, record[5]) - gpuCoreClockPoints = append(gpuCoreClockPoints, record[6]) - gpuMemClockPoints = append(gpuMemClockPoints, record[7]) - gpuVRAMUsedPoints = append(gpuVRAMUsedPoints, record[8]) - gpuPowerPoints = append(gpuPowerPoints, record[9]) - RAMUsedPoints = append(RAMUsedPoints, record[10]) - SWAPUsedPoints = append(SWAPUsedPoints, record[11]) - - counter++ - if counter == 100000 { - return nil, nil, errors.New("too large CSV file") - } - } - - // More efficient buffer allocation - linesCount = len(fpsPoints) - - if err := scanner.Err(); err != nil { - log.Println("error (4) parsing CSV:", err) - return nil, nil, err - } - - if len(fpsPoints) == 0 { - return nil, nil, errors.New("invalid CSV file (err 8)") - } - - if len(fpsPoints) > csvSpecs.MaxPoints { - csvSpecs.MaxPoints = len(fpsPoints) - } - - csvFile.FPSPointsArray = strings.Join(fpsPoints, ",") - csvFile.FrameTimeArray = strings.Join(frametimePoints, ",") - csvFile.CPULoadArray = strings.Join(cpuLoadPoints, ",") - csvFile.GPULoadArray = strings.Join(gpuLoadPoints, ",") - csvFile.CPUTempArray = strings.Join(cpuTempPoints, ",") - csvFile.GPUTempArray = strings.Join(gpuTempPoints, ",") - csvFile.GPUCoreClockArray = strings.Join(gpuCoreClockPoints, ",") - csvFile.GPUMemClockArray = strings.Join(gpuMemClockPoints, ",") - csvFile.GPUVRAMUsedArray = strings.Join(gpuVRAMUsedPoints, ",") - csvFile.GPUPowerArray = strings.Join(gpuPowerPoints, ",") - csvFile.RAMUsedArray = strings.Join(RAMUsedPoints, ",") - csvFile.SwapUsedArray = strings.Join(SWAPUsedPoints, ",") - - csvFiles = append(csvFiles, &csvFile) - } - - return csvFiles, csvSpecs, nil -} - -// truncateString truncates the input string to a maximum of 100 characters and appends "..." if it exceeds that length. -func truncateString(s string) string { - const maxLength = 100 - if len(s) > maxLength { - return s[:maxLength] + "..." - } - return s -} - -func storeBenchmarkData(csvFiles []*CSVFile, benchmarkID uint) error { - // Store to disk - filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) - file, err := os.Create(filePath) - if err != nil { - return err - } - defer file.Close() - - // Convert to []byte - var buffer bytes.Buffer - gobEncoder := gob.NewEncoder(&buffer) - err = gobEncoder.Encode(csvFiles) - if err != nil { - return err - } - - // Compress and write to file - zstdEncoder, err := zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest)) - if err != nil { - return err - } - defer zstdEncoder.Close() - _, err = zstdEncoder.Write(buffer.Bytes()) - return err -} - -func retrieveBenchmarkData(benchmarkID uint) (csvFiles []*CSVFile, err error) { - filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - // Decompress and read from file - zstdDecoder, err := zstd.NewReader(file) - if err != nil { - return nil, err - } - defer zstdDecoder.Close() - - var buffer bytes.Buffer - _, err = buffer.ReadFrom(zstdDecoder) - if err != nil { - return nil, err - } - - // Decode - gobDecoder := gob.NewDecoder(&buffer) - err = gobDecoder.Decode(&csvFiles) - return csvFiles, err -} - -func deleteBenchmarkData(benchmarkID uint) error { - filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID)) - return os.Remove(filePath) -} diff --git a/models.go b/models.go index d40f579..d80454f 100644 --- a/models.go +++ b/models.go @@ -15,15 +15,9 @@ type User struct { type Benchmark struct { gorm.Model - UserID uint - Title string - Description string - SpecDistro string - SpecCPU string - SpecGPU string - SpecRAM string - SpecKernel string - SpecScheduler string + UserID uint + Title string + Description string CreatedAtHumanized string `gorm:"-"` // Human readable "X h/m/s ago" version of CreatedAt (filled automatically) diff --git a/static/js/benchmark.js b/static/js/benchmark.js new file mode 100644 index 0000000..e69de29 diff --git a/templates/benchmark.tmpl b/templates/benchmark.tmpl index d4a00bd..b6c13e4 100644 --- a/templates/benchmark.tmpl +++ b/templates/benchmark.tmpl @@ -1,29 +1,21 @@ {{template "header.tmpl" .}} -
-
-
-
-
{{ .benchmark.Title }}
-

{{ .benchmark.Description }}

-

Submitted {{ .benchmark.CreatedAtHumanized }} by {{ .benchmark.User.Username }}.

-
-
-
-
-
    -
  • Distro: {{ .benchmark.SpecDistro }}
    • -
  • Kernel: {{ .benchmark.SpecKernel }}
    • -
  • GPU: {{ .benchmark.SpecGPU }}
    • -
  • CPU: {{ .benchmark.SpecCPU }}
    • -
  • RAM: {{ .benchmark.SpecRAM }}
    • -
  • Scheduler: {{ .benchmark.SpecScheduler }}
    • -
-
+
+

Benchmark #{{ .benchmark.ID }}

+
+ +
+
{{ .benchmark.Title }}
+

{{ .benchmark.Description }}

+

Submitted {{ .benchmark.CreatedAtHumanized }} by {{ .benchmark.User.Username }}.

{{if eq .benchmark.UserID .userID }} -Delete benchmark +
+
+ Delete benchmark +
+