313 lines
9.1 KiB
Go
313 lines
9.1 KiB
Go
package flightlesssomething
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/gob"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"math/big"
|
|
"mime/multipart"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/dustin/go-humanize"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
type BenchmarkData struct {
|
|
Label string
|
|
|
|
// Specs
|
|
SpecOS string
|
|
SpecGPU string
|
|
SpecCPU string
|
|
SpecRAM string
|
|
SpecLinuxKernel string
|
|
SpecLinuxScheduler string
|
|
|
|
// Data
|
|
DataFPS []float64
|
|
DataFrameTime []float64
|
|
DataCPULoad []float64
|
|
DataGPULoad []float64
|
|
DataCPUTemp []float64
|
|
DataGPUTemp []float64
|
|
DataGPUCoreClock []float64
|
|
DataGPUMemClock []float64
|
|
DataGPUVRAMUsed []float64
|
|
DataGPUPower []float64
|
|
DataRAMUsed []float64
|
|
DataSwapUsed []float64
|
|
}
|
|
|
|
// readBenchmarkFiles reads the uploaded benchmark files and returns a slice of BenchmarkData.
|
|
func readBenchmarkFiles(files []*multipart.FileHeader) ([]*BenchmarkData, error) {
|
|
csvFiles := make([]*BenchmarkData, 0)
|
|
linesCount := 0
|
|
|
|
for _, fileHeader := range files {
|
|
csvFile := BenchmarkData{}
|
|
|
|
file, err := fileHeader.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
|
|
scanner := bufio.NewScanner(file)
|
|
|
|
// Label is filename without extension
|
|
csvFile.Label = strings.TrimSuffix(fileHeader.Filename, ".csv")
|
|
csvFile.Label = strings.TrimSuffix(csvFile.Label, ".htm")
|
|
|
|
// First line should contain this: os,cpu,gpu,ram,kernel,driver,cpuscheduler
|
|
if !scanner.Scan() {
|
|
return nil, errors.New("invalid CSV file (err 1)")
|
|
}
|
|
record := strings.Split(strings.TrimRight(scanner.Text(), ","), ",")
|
|
if len(record) != 7 {
|
|
return nil, errors.New("invalid CSV file (err 2)")
|
|
}
|
|
|
|
// Second line should contain values
|
|
if !scanner.Scan() {
|
|
return nil, errors.New("invalid CSV file (err 3)")
|
|
}
|
|
record = strings.Split(scanner.Text(), ",")
|
|
|
|
for i, v := range record {
|
|
switch i {
|
|
case 0:
|
|
csvFile.SpecOS = truncateString(strings.TrimSpace(v))
|
|
case 1:
|
|
csvFile.SpecCPU = truncateString(strings.TrimSpace(v))
|
|
case 2:
|
|
csvFile.SpecGPU = truncateString(strings.TrimSpace(v))
|
|
case 3:
|
|
kilobytes := new(big.Int)
|
|
_, ok := kilobytes.SetString(strings.TrimSpace(v), 10)
|
|
if !ok {
|
|
return nil, errors.New("failed to convert RAM to big.Int")
|
|
}
|
|
bytes := new(big.Int).Mul(kilobytes, big.NewInt(1024))
|
|
csvFile.SpecRAM = humanize.Bytes(bytes.Uint64())
|
|
case 4:
|
|
csvFile.SpecLinuxKernel = truncateString(strings.TrimSpace(v))
|
|
case 6:
|
|
csvFile.SpecLinuxScheduler = truncateString(strings.TrimSpace(v))
|
|
}
|
|
}
|
|
|
|
// 3rd line contain headers for benchmark data: fps,frametime,cpu_load,gpu_load,cpu_temp,gpu_temp,gpu_core_clock,gpu_mem_clock,gpu_vram_used,gpu_power,ram_used,swap_used,process_rss,elapsed
|
|
if !scanner.Scan() {
|
|
return nil, errors.New("invalid CSV file (err 5)")
|
|
}
|
|
record = strings.Split(strings.TrimRight(scanner.Text(), ","), ",")
|
|
if len(record) != 14 {
|
|
return nil, errors.New("invalid CSV file (err 6)")
|
|
}
|
|
|
|
// Preallocate slices. First file will be inefficient, but later files will contain
|
|
// value of linesCount that would help to optimize preallocation.
|
|
csvFile.DataFPS = make([]float64, 0, linesCount)
|
|
csvFile.DataFrameTime = make([]float64, 0, linesCount)
|
|
csvFile.DataCPULoad = make([]float64, 0, linesCount)
|
|
csvFile.DataGPULoad = make([]float64, 0, linesCount)
|
|
csvFile.DataCPUTemp = make([]float64, 0, linesCount)
|
|
csvFile.DataGPUTemp = make([]float64, 0, linesCount)
|
|
csvFile.DataGPUCoreClock = make([]float64, 0, linesCount)
|
|
csvFile.DataGPUMemClock = make([]float64, 0, linesCount)
|
|
csvFile.DataGPUVRAMUsed = make([]float64, 0, linesCount)
|
|
csvFile.DataGPUPower = make([]float64, 0, linesCount)
|
|
csvFile.DataRAMUsed = make([]float64, 0, linesCount)
|
|
csvFile.DataSwapUsed = make([]float64, 0, linesCount)
|
|
|
|
var counter uint
|
|
|
|
for scanner.Scan() {
|
|
record = strings.Split(scanner.Text(), ",")
|
|
if len(record) != 14 {
|
|
return nil, errors.New("invalid CSV file (err 7)")
|
|
}
|
|
|
|
val, err := strconv.ParseFloat(record[0], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse FPS value '%s': %v", record[0], err)
|
|
}
|
|
csvFile.DataFPS = append(csvFile.DataFPS, val)
|
|
|
|
val, err = strconv.ParseFloat(record[1], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse frametime value '%s': %v", record[1], err)
|
|
}
|
|
csvFile.DataFrameTime = append(csvFile.DataFrameTime, val)
|
|
|
|
val, err = strconv.ParseFloat(record[2], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse CPU load value '%s': %v", record[2], err)
|
|
}
|
|
csvFile.DataCPULoad = append(csvFile.DataCPULoad, val)
|
|
|
|
val, err = strconv.ParseFloat(record[3], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU load value '%s': %v", record[3], err)
|
|
}
|
|
csvFile.DataGPULoad = append(csvFile.DataGPULoad, val)
|
|
|
|
val, err = strconv.ParseFloat(record[4], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse CPU temp value '%s': %v", record[4], err)
|
|
}
|
|
csvFile.DataCPUTemp = append(csvFile.DataCPUTemp, val)
|
|
|
|
val, err = strconv.ParseFloat(record[5], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU temp value '%s': %v", record[5], err)
|
|
}
|
|
csvFile.DataGPUTemp = append(csvFile.DataGPUTemp, val)
|
|
|
|
val, err = strconv.ParseFloat(record[6], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU core clock value '%s': %v", record[6], err)
|
|
}
|
|
csvFile.DataGPUCoreClock = append(csvFile.DataGPUCoreClock, val)
|
|
|
|
val, err = strconv.ParseFloat(record[7], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU mem clock value '%s': %v", record[7], err)
|
|
}
|
|
csvFile.DataGPUMemClock = append(csvFile.DataGPUMemClock, val)
|
|
|
|
val, err = strconv.ParseFloat(record[8], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU VRAM used value '%s': %v", record[8], err)
|
|
}
|
|
csvFile.DataGPUVRAMUsed = append(csvFile.DataGPUVRAMUsed, val)
|
|
|
|
val, err = strconv.ParseFloat(record[9], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse GPU power value '%s': %v", record[9], err)
|
|
}
|
|
csvFile.DataGPUPower = append(csvFile.DataGPUPower, val)
|
|
|
|
val, err = strconv.ParseFloat(record[10], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse RAM used value '%s': %v", record[10], err)
|
|
}
|
|
csvFile.DataRAMUsed = append(csvFile.DataRAMUsed, val)
|
|
|
|
val, err = strconv.ParseFloat(record[11], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse SWAP used value '%s': %v", record[11], err)
|
|
}
|
|
csvFile.DataSwapUsed = append(csvFile.DataSwapUsed, val)
|
|
|
|
counter++
|
|
if counter == 100000 {
|
|
return nil, errors.New("CSV file cannot have more than 100000 data lines")
|
|
}
|
|
}
|
|
|
|
// Next file would be more efficient to preallocate slices
|
|
if linesCount < len(csvFile.DataFPS) {
|
|
linesCount = len(csvFile.DataFPS)
|
|
}
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
log.Println("error (4) parsing CSV:", err)
|
|
return nil, err
|
|
}
|
|
|
|
if len(csvFile.DataFPS) == 0 &&
|
|
len(csvFile.DataFrameTime) == 0 &&
|
|
len(csvFile.DataCPULoad) == 0 &&
|
|
len(csvFile.DataGPULoad) == 0 &&
|
|
len(csvFile.DataCPUTemp) == 0 &&
|
|
len(csvFile.DataGPUTemp) == 0 &&
|
|
len(csvFile.DataGPUCoreClock) == 0 &&
|
|
len(csvFile.DataGPUMemClock) == 0 &&
|
|
len(csvFile.DataGPUVRAMUsed) == 0 &&
|
|
len(csvFile.DataGPUPower) == 0 &&
|
|
len(csvFile.DataRAMUsed) == 0 &&
|
|
len(csvFile.DataSwapUsed) == 0 {
|
|
return nil, errors.New("empty CSV file (err 8)")
|
|
}
|
|
|
|
csvFiles = append(csvFiles, &csvFile)
|
|
}
|
|
|
|
return csvFiles, nil
|
|
}
|
|
|
|
// truncateString truncates the input string to a maximum of 100 characters and appends "..." if it exceeds that length.
|
|
func truncateString(s string) string {
|
|
const maxLength = 100
|
|
if len(s) > maxLength {
|
|
return s[:maxLength] + "..."
|
|
}
|
|
return s
|
|
}
|
|
|
|
func storeBenchmarkData(csvFiles []*BenchmarkData, benchmarkID uint) error {
|
|
// Store to disk
|
|
filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID))
|
|
file, err := os.Create(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
// Convert to []byte
|
|
var buffer bytes.Buffer
|
|
gobEncoder := gob.NewEncoder(&buffer)
|
|
err = gobEncoder.Encode(csvFiles)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Compress and write to file
|
|
zstdEncoder, err := zstd.NewWriter(file, zstd.WithEncoderLevel(zstd.SpeedFastest))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer zstdEncoder.Close()
|
|
_, err = zstdEncoder.Write(buffer.Bytes())
|
|
return err
|
|
}
|
|
|
|
func retrieveBenchmarkData(benchmarkID uint) (csvFiles []*BenchmarkData, err error) {
|
|
filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID))
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
|
|
// Decompress and read from file
|
|
zstdDecoder, err := zstd.NewReader(file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer zstdDecoder.Close()
|
|
|
|
var buffer bytes.Buffer
|
|
_, err = buffer.ReadFrom(zstdDecoder)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Decode
|
|
gobDecoder := gob.NewDecoder(&buffer)
|
|
err = gobDecoder.Decode(&csvFiles)
|
|
return csvFiles, err
|
|
}
|
|
|
|
func deleteBenchmarkData(benchmarkID uint) error {
|
|
filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID))
|
|
return os.Remove(filePath)
|
|
}
|