Add download function, prepare for RivaTuner format

This commit is contained in:
Erikas 2024-07-12 10:45:48 +03:00
parent af26d221ff
commit fb2642aad2
5 changed files with 297 additions and 181 deletions

View File

@ -1,12 +1,13 @@
package flightlesssomething
import (
"archive/zip"
"bufio"
"bytes"
"encoding/csv"
"encoding/gob"
"errors"
"fmt"
"log"
"math/big"
"mime/multipart"
"os"
@ -46,201 +47,206 @@ type BenchmarkData struct {
// readBenchmarkFiles reads the uploaded benchmark files and returns a slice of BenchmarkData.
func readBenchmarkFiles(files []*multipart.FileHeader) ([]*BenchmarkData, error) {
csvFiles := make([]*BenchmarkData, 0)
linesCount := 0
benchmarkDatas := make([]*BenchmarkData, 0)
for _, fileHeader := range files {
csvFile := BenchmarkData{}
file, err := fileHeader.Open()
if err != nil {
return nil, err
}
defer file.Close()
defer file.Close()
scanner := bufio.NewScanner(file)
// Label is filename without extension
csvFile.Label = strings.TrimSuffix(fileHeader.Filename, ".csv")
csvFile.Label = strings.TrimSuffix(csvFile.Label, ".htm")
// First line should contain this: os,cpu,gpu,ram,kernel,driver,cpuscheduler
// FirstLine identifies file format
if !scanner.Scan() {
return nil, errors.New("invalid CSV file (err 1)")
return nil, errors.New("failed to read file (err 1)")
}
record := strings.Split(strings.TrimRight(scanner.Text(), ","), ",")
if len(record) != 7 {
return nil, errors.New("invalid CSV file (err 2)")
firstLine := scanner.Text()
var benchmarkData *BenchmarkData
var suffix string
switch firstLine {
case "os,cpu,gpu,ram,kernel,driver,cpuscheduler": // MangoHud
benchmarkData, err = readMangoHudFile(scanner)
suffix = ".csv"
case "PLACEHOLDER": // RivaTuner
benchmarkData, err = readMangoHudFile(scanner)
suffix = ".htm"
default:
return nil, errors.New("unsupported file format")
}
// Second line should contain values
if !scanner.Scan() {
return nil, errors.New("invalid CSV file (err 3)")
}
record = strings.Split(scanner.Text(), ",")
for i, v := range record {
switch i {
case 0:
csvFile.SpecOS = truncateString(strings.TrimSpace(v))
case 1:
csvFile.SpecCPU = truncateString(strings.TrimSpace(v))
case 2:
csvFile.SpecGPU = truncateString(strings.TrimSpace(v))
case 3:
kilobytes := new(big.Int)
_, ok := kilobytes.SetString(strings.TrimSpace(v), 10)
if !ok {
return nil, errors.New("failed to convert RAM to big.Int")
}
bytes := new(big.Int).Mul(kilobytes, big.NewInt(1024))
csvFile.SpecRAM = humanize.Bytes(bytes.Uint64())
case 4:
csvFile.SpecLinuxKernel = truncateString(strings.TrimSpace(v))
case 6:
csvFile.SpecLinuxScheduler = truncateString(strings.TrimSpace(v))
}
}
// 3rd line contain headers for benchmark data: fps,frametime,cpu_load,gpu_load,cpu_temp,gpu_temp,gpu_core_clock,gpu_mem_clock,gpu_vram_used,gpu_power,ram_used,swap_used,process_rss,elapsed
if !scanner.Scan() {
return nil, errors.New("invalid CSV file (err 5)")
}
record = strings.Split(strings.TrimRight(scanner.Text(), ","), ",")
if len(record) != 14 {
return nil, errors.New("invalid CSV file (err 6)")
}
// Preallocate slices. First file will be inefficient, but later files will contain
// value of linesCount that would help to optimize preallocation.
csvFile.DataFPS = make([]float64, 0, linesCount)
csvFile.DataFrameTime = make([]float64, 0, linesCount)
csvFile.DataCPULoad = make([]float64, 0, linesCount)
csvFile.DataGPULoad = make([]float64, 0, linesCount)
csvFile.DataCPUTemp = make([]float64, 0, linesCount)
csvFile.DataGPUTemp = make([]float64, 0, linesCount)
csvFile.DataGPUCoreClock = make([]float64, 0, linesCount)
csvFile.DataGPUMemClock = make([]float64, 0, linesCount)
csvFile.DataGPUVRAMUsed = make([]float64, 0, linesCount)
csvFile.DataGPUPower = make([]float64, 0, linesCount)
csvFile.DataRAMUsed = make([]float64, 0, linesCount)
csvFile.DataSwapUsed = make([]float64, 0, linesCount)
var counter uint
for scanner.Scan() {
record = strings.Split(scanner.Text(), ",")
if len(record) != 14 {
return nil, errors.New("invalid CSV file (err 7)")
}
val, err := strconv.ParseFloat(record[0], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse FPS value '%s': %v", record[0], err)
}
csvFile.DataFPS = append(csvFile.DataFPS, val)
val, err = strconv.ParseFloat(record[1], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse frametime value '%s': %v", record[1], err)
}
csvFile.DataFrameTime = append(csvFile.DataFrameTime, val)
val, err = strconv.ParseFloat(record[2], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse CPU load value '%s': %v", record[2], err)
}
csvFile.DataCPULoad = append(csvFile.DataCPULoad, val)
val, err = strconv.ParseFloat(record[3], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU load value '%s': %v", record[3], err)
}
csvFile.DataGPULoad = append(csvFile.DataGPULoad, val)
val, err = strconv.ParseFloat(record[4], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse CPU temp value '%s': %v", record[4], err)
}
csvFile.DataCPUTemp = append(csvFile.DataCPUTemp, val)
val, err = strconv.ParseFloat(record[5], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU temp value '%s': %v", record[5], err)
}
csvFile.DataGPUTemp = append(csvFile.DataGPUTemp, val)
val, err = strconv.ParseFloat(record[6], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU core clock value '%s': %v", record[6], err)
}
csvFile.DataGPUCoreClock = append(csvFile.DataGPUCoreClock, val)
val, err = strconv.ParseFloat(record[7], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU mem clock value '%s': %v", record[7], err)
}
csvFile.DataGPUMemClock = append(csvFile.DataGPUMemClock, val)
val, err = strconv.ParseFloat(record[8], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU VRAM used value '%s': %v", record[8], err)
}
csvFile.DataGPUVRAMUsed = append(csvFile.DataGPUVRAMUsed, val)
val, err = strconv.ParseFloat(record[9], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU power value '%s': %v", record[9], err)
}
csvFile.DataGPUPower = append(csvFile.DataGPUPower, val)
val, err = strconv.ParseFloat(record[10], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse RAM used value '%s': %v", record[10], err)
}
csvFile.DataRAMUsed = append(csvFile.DataRAMUsed, val)
val, err = strconv.ParseFloat(record[11], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse SWAP used value '%s': %v", record[11], err)
}
csvFile.DataSwapUsed = append(csvFile.DataSwapUsed, val)
counter++
if counter == 100000 {
return nil, errors.New("CSV file cannot have more than 100000 data lines")
}
}
// Next file would be more efficient to preallocate slices
if linesCount < len(csvFile.DataFPS) {
linesCount = len(csvFile.DataFPS)
}
if err := scanner.Err(); err != nil {
log.Println("error (4) parsing CSV:", err)
if err != nil {
return nil, err
}
if len(csvFile.DataFPS) == 0 &&
len(csvFile.DataFrameTime) == 0 &&
len(csvFile.DataCPULoad) == 0 &&
len(csvFile.DataGPULoad) == 0 &&
len(csvFile.DataCPUTemp) == 0 &&
len(csvFile.DataGPUTemp) == 0 &&
len(csvFile.DataGPUCoreClock) == 0 &&
len(csvFile.DataGPUMemClock) == 0 &&
len(csvFile.DataGPUVRAMUsed) == 0 &&
len(csvFile.DataGPUPower) == 0 &&
len(csvFile.DataRAMUsed) == 0 &&
len(csvFile.DataSwapUsed) == 0 {
return nil, errors.New("empty CSV file (err 8)")
}
csvFiles = append(csvFiles, &csvFile)
benchmarkData.Label = strings.TrimSuffix(fileHeader.Filename, suffix)
benchmarkDatas = append(benchmarkDatas, benchmarkData)
}
return csvFiles, nil
return benchmarkDatas, nil
}
func readMangoHudFile(scanner *bufio.Scanner) (*BenchmarkData, error) {
benchmarkData := &BenchmarkData{}
// Second line should contain values
if !scanner.Scan() {
return nil, errors.New("failed to read file (err mh1)")
}
record := strings.Split(scanner.Text(), ",")
for i, v := range record {
switch i {
case 0:
benchmarkData.SpecOS = truncateString(strings.TrimSpace(v))
case 1:
benchmarkData.SpecCPU = truncateString(strings.TrimSpace(v))
case 2:
benchmarkData.SpecGPU = truncateString(strings.TrimSpace(v))
case 3:
kilobytes := new(big.Int)
_, ok := kilobytes.SetString(strings.TrimSpace(v), 10)
if !ok {
bytes := new(big.Int).Mul(kilobytes, big.NewInt(1024))
benchmarkData.SpecRAM = humanize.Bytes(bytes.Uint64())
} else {
benchmarkData.SpecRAM = truncateString(strings.TrimSpace(v))
}
case 4:
benchmarkData.SpecLinuxKernel = truncateString(strings.TrimSpace(v))
case 6:
benchmarkData.SpecLinuxScheduler = truncateString(strings.TrimSpace(v))
}
}
// 3rd line contain headers for benchmark data
if !scanner.Scan() {
return nil, errors.New("failed to read file (err mh2)")
}
record = strings.Split(strings.TrimRight(scanner.Text(), ","), ",")
if len(record) == 0 {
return nil, errors.New("failed to read file (err mh3)")
}
benchmarkData.DataFPS = make([]float64, 0)
benchmarkData.DataFrameTime = make([]float64, 0)
benchmarkData.DataCPULoad = make([]float64, 0)
benchmarkData.DataGPULoad = make([]float64, 0)
benchmarkData.DataCPUTemp = make([]float64, 0)
benchmarkData.DataGPUTemp = make([]float64, 0)
benchmarkData.DataGPUCoreClock = make([]float64, 0)
benchmarkData.DataGPUMemClock = make([]float64, 0)
benchmarkData.DataGPUVRAMUsed = make([]float64, 0)
benchmarkData.DataGPUPower = make([]float64, 0)
benchmarkData.DataRAMUsed = make([]float64, 0)
benchmarkData.DataSwapUsed = make([]float64, 0)
var counter uint
for scanner.Scan() {
record = strings.Split(scanner.Text(), ",")
if len(record) < 12 { // Ignore last 2 columns as they are not needed
return nil, errors.New("failed to read file (err mh4)")
}
val, err := strconv.ParseFloat(record[0], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse FPS value '%s': %v", record[0], err)
}
benchmarkData.DataFPS = append(benchmarkData.DataFPS, val)
val, err = strconv.ParseFloat(record[1], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse frametime value '%s': %v", record[1], err)
}
benchmarkData.DataFrameTime = append(benchmarkData.DataFrameTime, val)
val, err = strconv.ParseFloat(record[2], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse CPU load value '%s': %v", record[2], err)
}
benchmarkData.DataCPULoad = append(benchmarkData.DataCPULoad, val)
val, err = strconv.ParseFloat(record[3], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU load value '%s': %v", record[3], err)
}
benchmarkData.DataGPULoad = append(benchmarkData.DataGPULoad, val)
val, err = strconv.ParseFloat(record[4], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse CPU temp value '%s': %v", record[4], err)
}
benchmarkData.DataCPUTemp = append(benchmarkData.DataCPUTemp, val)
val, err = strconv.ParseFloat(record[5], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU temp value '%s': %v", record[5], err)
}
benchmarkData.DataGPUTemp = append(benchmarkData.DataGPUTemp, val)
val, err = strconv.ParseFloat(record[6], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU core clock value '%s': %v", record[6], err)
}
benchmarkData.DataGPUCoreClock = append(benchmarkData.DataGPUCoreClock, val)
val, err = strconv.ParseFloat(record[7], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU mem clock value '%s': %v", record[7], err)
}
benchmarkData.DataGPUMemClock = append(benchmarkData.DataGPUMemClock, val)
val, err = strconv.ParseFloat(record[8], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU VRAM used value '%s': %v", record[8], err)
}
benchmarkData.DataGPUVRAMUsed = append(benchmarkData.DataGPUVRAMUsed, val)
val, err = strconv.ParseFloat(record[9], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse GPU power value '%s': %v", record[9], err)
}
benchmarkData.DataGPUPower = append(benchmarkData.DataGPUPower, val)
val, err = strconv.ParseFloat(record[10], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse RAM used value '%s': %v", record[10], err)
}
benchmarkData.DataRAMUsed = append(benchmarkData.DataRAMUsed, val)
val, err = strconv.ParseFloat(record[11], 64)
if err != nil {
return nil, fmt.Errorf("failed to parse SWAP used value '%s': %v", record[11], err)
}
benchmarkData.DataSwapUsed = append(benchmarkData.DataSwapUsed, val)
counter++
if counter == 100000 {
return nil, errors.New("CSV file cannot have more than 100000 data lines")
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
if len(benchmarkData.DataFPS) == 0 &&
len(benchmarkData.DataFrameTime) == 0 &&
len(benchmarkData.DataCPULoad) == 0 &&
len(benchmarkData.DataGPULoad) == 0 &&
len(benchmarkData.DataCPUTemp) == 0 &&
len(benchmarkData.DataGPUTemp) == 0 &&
len(benchmarkData.DataGPUCoreClock) == 0 &&
len(benchmarkData.DataGPUMemClock) == 0 &&
len(benchmarkData.DataGPUVRAMUsed) == 0 &&
len(benchmarkData.DataGPUPower) == 0 &&
len(benchmarkData.DataRAMUsed) == 0 &&
len(benchmarkData.DataSwapUsed) == 0 {
return nil, errors.New("empty CSV file")
}
return benchmarkData, nil
}
// truncateString truncates the input string to a maximum of 100 characters and appends "..." if it exceeds that length.
@ -310,3 +316,63 @@ func deleteBenchmarkData(benchmarkID uint) error {
filePath := filepath.Join(benchmarksDir, fmt.Sprintf("%d.bin", benchmarkID))
return os.Remove(filePath)
}
func createZipFromBenchmarkData(benchmarkData []*BenchmarkData) (*bytes.Buffer, error) {
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
zipWriter := zip.NewWriter(buf)
for _, data := range benchmarkData {
// Create a new CSV file in the zip archive.
fileName := fmt.Sprintf("%s.csv", data.Label)
fileWriter, err := zipWriter.Create(fileName)
if err != nil {
return nil, fmt.Errorf("could not create file in zip: %v", err)
}
// Create a CSV writer.
csvWriter := csv.NewWriter(fileWriter)
// Write the header.
header := []string{"os", "cpu", "gpu", "ram", "kernel", "driver", "cpuscheduler"}
csvWriter.Write(header)
specs := []string{data.SpecOS, data.SpecCPU, data.SpecGPU, data.SpecRAM, data.SpecLinuxKernel, "", data.SpecLinuxScheduler}
csvWriter.Write(specs)
// Write the data header.
dataHeader := []string{"fps", "frametime", "cpu_load", "gpu_load", "cpu_temp", "gpu_temp", "gpu_core_clock", "gpu_mem_clock", "gpu_vram_used", "gpu_power", "ram_used", "swap_used"}
csvWriter.Write(dataHeader)
// Write the data rows.
for i := range data.DataFPS {
row := []string{
strconv.FormatFloat(data.DataFPS[i], 'f', 4, 64),
strconv.FormatFloat(data.DataFrameTime[i], 'f', 4, 64),
strconv.FormatFloat(data.DataCPULoad[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPULoad[i], 'f', 4, 64),
strconv.FormatFloat(data.DataCPUTemp[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPUTemp[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPUCoreClock[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPUMemClock[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPUVRAMUsed[i], 'f', 4, 64),
strconv.FormatFloat(data.DataGPUPower[i], 'f', 4, 64),
strconv.FormatFloat(data.DataRAMUsed[i], 'f', 4, 64),
strconv.FormatFloat(data.DataSwapUsed[i], 'f', 4, 64),
}
csvWriter.Write(row)
}
// Make sure to flush the writer.
csvWriter.Flush()
if err := csvWriter.Error(); err != nil {
return nil, fmt.Errorf("could not write CSV: %v", err)
}
}
// Close the zip writer to flush the buffer.
if err := zipWriter.Close(); err != nil {
return nil, fmt.Errorf("could not close zip writer: %v", err)
}
return buf, nil
}

View File

@ -375,3 +375,53 @@ func getBenchmark(c *gin.Context) {
"benchmarkData": benchmarkDatas,
})
}
func getBenchmarkDownload(c *gin.Context) {
session := sessions.Default(c)
// Get benchmark ID from the path
id := c.Param("id")
// Get benchmark details
intID, err := strconv.Atoi(id)
if err != nil {
c.HTML(http.StatusInternalServerError, "error.tmpl", gin.H{
"activePage": "error",
"username": session.Get("Username"),
"userID": session.Get("ID"),
"errorMessage": "Internal server error occurred: " + err.Error(),
})
return
}
var benchmark Benchmark
benchmark.ID = uint(intID)
benchmarkDatas, err := retrieveBenchmarkData(benchmark.ID)
if err != nil {
c.HTML(http.StatusInternalServerError, "error.tmpl", gin.H{
"activePage": "error",
"username": session.Get("Username"),
"userID": session.Get("ID"),
"errorMessage": "Error occurred: " + err.Error(),
})
return
}
content, err := createZipFromBenchmarkData(benchmarkDatas)
if err != nil {
c.HTML(http.StatusInternalServerError, "error.tmpl", gin.H{
"activePage": "error",
"username": session.Get("Username"),
"userID": session.Get("ID"),
"errorMessage": "Error occurred: " + err.Error(),
})
return
}
fileName := "benchmark_" + id + ".zip"
c.Header("Content-Type", "application/zip")
c.Header("Content-Disposition", "attachment; filename="+fileName)
c.Data(http.StatusOK, "application/zip", content.Bytes())
}

View File

@ -94,6 +94,7 @@ func Start(c *Config) {
r.POST("/benchmark", postBenchmarkCreate)
r.GET("/benchmark/:id", getBenchmark)
r.DELETE("/benchmark/:id", deleteBenchmark)
r.GET("/benchmark/:id/download", getBenchmarkDownload)
r.GET("/user/:id", getUser)

View File

@ -6,7 +6,7 @@
{{if eq .benchmark.UserID .userID }}
<a class="btn btn-danger me-2" data-bs-toggle="modal" data-bs-target="#exampleModal"><i class="fa-solid fa-trash"></i> Delete</a>
{{end}}
<a class="btn btn-secondary"><i class="fa-solid fa-download"></i> Download</a>
<a class="btn btn-secondary" href="/benchmark/{{ .benchmark.ID }}/download" target="_blank" title="Reconstruct MangoHud-like CSV files"><i class="fa-solid fa-download"></i> Download</a>
</div>
</div>

View File

@ -1,5 +1,4 @@
</div>
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.6/dist/umd/popper.min.js" integrity="sha384-oBqDVmMz9ATKxIep9tiCxS/Z9fNfEXiDAYTujMAeBAsjFuCZSmKbSSUnQlmh/jp3" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css" integrity="sha512-SnH5WK+bZxgPHs44uWIX+LLJAJ9/2PkPKZ5QiAj6Ta86w+fsb2TkcmfRyVX3pBnMFcV7oQPJkl9QevSCWr3W6A==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<script src="https://unpkg.com/htmx.org@2.0.0"></script>