From 2b5a73509195f38c42fc14facdf2a3d2c6faff5b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 6 Sep 2016 20:15:18 +0200 Subject: [PATCH] Report historical performance --- cmd/aggregate/main.go | 61 ++++++++++++++++++++-- cmd/ursrv/main.go | 47 +++++++++++++++++ static/index.html | 40 +++++++++++++- static/performance.html | 112 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 static/performance.html diff --git a/cmd/aggregate/main.go b/cmd/aggregate/main.go index 4fe8bf508..d783f7518 100644 --- a/cmd/aggregate/main.go +++ b/cmd/aggregate/main.go @@ -53,6 +53,14 @@ func runAggregation(db *sql.DB) { log.Fatalln("aggregate:", err) } log.Println("Inserted", rows, "rows") + + log.Println("Aggregating Performance data") + since = maxIndexedDay(db, "Performance") + rows, err = aggregatePerformance(db, since) + if err != nil { + log.Fatalln("aggregate:", err) + } + log.Println("Inserted", rows, "rows") } func sleepUntilNext(intv, margin time.Duration) { @@ -82,21 +90,40 @@ func setupDB(db *sql.DB) error { return err } + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance ( + Day TIMESTAMP NOT NULL, + TotFiles INTEGER NOT NULL, + TotMiB INTEGER NOT NULL, + SHA256Perf DOUBLE PRECISION NOT NULL, + MemorySize INTEGER NOT NULL, + MemoryUsageMiB INTEGER NOT NULL + )`) + if err != nil { + return err + } + + var t string + row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`) - if err := row.Scan(nil); err != nil { + if err := row.Scan(&t); err != nil { _, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`) } - row = db.QueryRow(`SELECT 'DayIndex'::regclass`) - if err := row.Scan(nil); err != nil { - _, err = db.Exec(`CREATE INDEX DayIndex ON VerionSummary (Day)`) + row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`) + if err := row.Scan(&t); err != nil { + _, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`) } row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`) - if err := row.Scan(nil); err != nil { + if err := row.Scan(&t); err != nil { _, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`) } + row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`) + if err := row.Scan(&t); err != nil { + _, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`) + } + return err } @@ -209,3 +236,27 @@ func aggregateUserMovement(db *sql.DB) (int64, error) { return int64(len(sumRows)), tx.Commit() } + +func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) { + res, err := db.Exec(`INSERT INTO Performance ( + SELECT + DATE_TRUNC('day', Received) AS Day, + AVG(TotFiles) As TotFiles, + AVG(TotMiB) As TotMiB, + AVG(SHA256Perf) As SHA256Perf, + AVG(MemorySize) As MemorySize, + AVG(MemoryUsageMiB) As MemoryUsageMiB + FROM Reports + WHERE + DATE_TRUNC('day', Received) > $1 + AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW()) + AND Version like 'v0.%' + GROUP BY Day + ); + `, since) + if err != nil { + return 0, err + } + + return res.RowsAffected() +} diff --git a/cmd/ursrv/main.go b/cmd/ursrv/main.go index 73ce7349f..e2fb5c431 100644 --- a/cmd/ursrv/main.go +++ b/cmd/ursrv/main.go @@ -350,6 +350,7 @@ func main() { http.HandleFunc("/newdata", withDB(db, newDataHandler)) http.HandleFunc("/summary.json", withDB(db, summaryHandler)) http.HandleFunc("/movement.json", withDB(db, movementHandler)) + http.HandleFunc("/performance.json", withDB(db, performanceHandler)) http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static")))) err = srv.Serve(listener) @@ -458,6 +459,25 @@ func movementHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) { w.Write(bs) } +func performanceHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) { + s, err := getPerformance(db) + if err != nil { + log.Println("performanceHandler:", err) + http.Error(w, "Database Error", http.StatusInternalServerError) + return + } + + bs, err := json.Marshal(s) + if err != nil { + log.Println("performanceHandler:", err) + http.Error(w, "JSON Encode Error", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(bs) +} + type category struct { Values [4]float64 Key string @@ -919,6 +939,33 @@ func getMovement(db *sql.DB) ([][]interface{}, error) { return res, nil } +func getPerformance(db *sql.DB) ([][]interface{}, error) { + rows, err := db.Query(`SELECT Day, TotFiles, TotMiB, SHA256Perf, MemorySize, MemoryUsageMiB FROM Performance WHERE Day > '2014-06-20'::TIMESTAMP ORDER BY Day`) + if err != nil { + return nil, err + } + defer rows.Close() + + res := [][]interface{}{ + {"Day", "TotFiles", "TotMiB", "SHA256Perf", "MemorySize", "MemoryUsageMiB"}, + } + + for rows.Next() { + var day time.Time + var sha256Perf float64 + var totFiles, totMiB, memorySize, memoryUsage int + err := rows.Scan(&day, &totFiles, &totMiB, &sha256Perf, &memorySize, &memoryUsage) + if err != nil { + return nil, err + } + + row := []interface{}{day.Format("2006-01-02"), totFiles, totMiB, float64(int(sha256Perf*10)) / 10, memorySize, memoryUsage} + res = append(res, row) + } + + return res, nil +} + type sortableFeatureList []feature func (l sortableFeatureList) Len() int { diff --git a/static/index.html b/static/index.html index d4ec153c0..f555ee8f3 100644 --- a/static/index.html +++ b/static/index.html @@ -41,6 +41,7 @@ found in the LICENSE file. @@ -119,9 +155,11 @@ found in the LICENSE file.

Reappearance of users cause the "left" data to shrink retroactively. - Spikes in December 2014 were due to the unique ID format changing and have been partly removed to avoid skewing the graph scale.

+
+ +

Usage Metrics

This is the aggregated usage report data for the last 24 hours. Data based on {{.nodes}} devices that have reported in. diff --git a/static/performance.html b/static/performance.html new file mode 100644 index 000000000..5ae940f3a --- /dev/null +++ b/static/performance.html @@ -0,0 +1,112 @@ + + + + + + + + + + + + Historical Performance Data + + + + + + + + + + +

+
+
+

Historical Performance Data

+

These charts are all the average of the corresponding metric, for the entire population of a given day.

+ +

Hash Performance (MiB/s)

+
+ +

Memory Usage (MiB)

+
+ +

Total Number of Files

+
+ +

Total Folder Size (GiB)

+
+ +

System RAM Size (GiB)

+
+ +
+
+ +