Report historical performance

cmd/aggregate/main.go

@@ -53,6 +53,14 @@ func runAggregation(db *sql.DB) {
 		log.Fatalln("aggregate:", err)
 	}
 	log.Println("Inserted", rows, "rows")
+
+	log.Println("Aggregating Performance data")
+	since = maxIndexedDay(db, "Performance")
+	rows, err = aggregatePerformance(db, since)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
 }
 
 func sleepUntilNext(intv, margin time.Duration) {
@@ -82,21 +90,40 @@ func setupDB(db *sql.DB) error {
 		return err
 	}
 
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
+		Day TIMESTAMP NOT NULL,
+		TotFiles INTEGER NOT NULL,
+		TotMiB INTEGER NOT NULL,
+		SHA256Perf DOUBLE PRECISION NOT NULL,
+		MemorySize INTEGER NOT NULL,
+		MemoryUsageMiB INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	var t string
+
 	row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err := row.Scan(&t); err != nil {
 		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
 	}
 
-	row = db.QueryRow(`SELECT 'DayIndex'::regclass`)
+	row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE INDEX DayIndex ON VerionSummary (Day)`)
+		_, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
 	}
 
 	row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err := row.Scan(&t); err != nil {
 		_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
 	}
 
+	row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
+	}
+
 	return err
 }
 
@@ -209,3 +236,27 @@ func aggregateUserMovement(db *sql.DB) (int64, error) {
 
 	return int64(len(sumRows)), tx.Commit()
 }
+
+func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
+	res, err := db.Exec(`INSERT INTO Performance (
+	SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		AVG(TotFiles) As TotFiles,
+		AVG(TotMiB) As TotMiB,
+		AVG(SHA256Perf) As SHA256Perf,
+		AVG(MemorySize) As MemorySize,
+		AVG(MemoryUsageMiB) As MemoryUsageMiB
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) > $1
+			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Version like 'v0.%'
+		GROUP BY Day
+		);
+	`, since)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}

cmd/ursrv/main.go

@@ -350,6 +350,7 @@ func main() {
 	http.HandleFunc("/newdata", withDB(db, newDataHandler))
 	http.HandleFunc("/summary.json", withDB(db, summaryHandler))
 	http.HandleFunc("/movement.json", withDB(db, movementHandler))
+	http.HandleFunc("/performance.json", withDB(db, performanceHandler))
 	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
 
 	err = srv.Serve(listener)
@@ -458,6 +459,25 @@ func movementHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 	w.Write(bs)
 }
 
+func performanceHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
+	s, err := getPerformance(db)
+	if err != nil {
+		log.Println("performanceHandler:", err)
+		http.Error(w, "Database Error", http.StatusInternalServerError)
+		return
+	}
+
+	bs, err := json.Marshal(s)
+	if err != nil {
+		log.Println("performanceHandler:", err)
+		http.Error(w, "JSON Encode Error", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
 type category struct {
 	Values [4]float64
 	Key    string
@@ -919,6 +939,33 @@ func getMovement(db *sql.DB) ([][]interface{}, error) {
 	return res, nil
 }
 
+func getPerformance(db *sql.DB) ([][]interface{}, error) {
+	rows, err := db.Query(`SELECT Day, TotFiles, TotMiB, SHA256Perf, MemorySize, MemoryUsageMiB FROM Performance WHERE Day > '2014-06-20'::TIMESTAMP ORDER BY Day`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	res := [][]interface{}{
+		{"Day", "TotFiles", "TotMiB", "SHA256Perf", "MemorySize", "MemoryUsageMiB"},
+	}
+
+	for rows.Next() {
+		var day time.Time
+		var sha256Perf float64
+		var totFiles, totMiB, memorySize, memoryUsage int
+		err := rows.Scan(&day, &totFiles, &totMiB, &sha256Perf, &memorySize, &memoryUsage)
+		if err != nil {
+			return nil, err
+		}
+
+		row := []interface{}{day.Format("2006-01-02"), totFiles, totMiB, float64(int(sha256Perf*10)) / 10, memorySize, memoryUsage}
+		res = append(res, row)
+	}
+
+	return res, nil
+}
+
 type sortableFeatureList []feature
 
 func (l sortableFeatureList) Len() int {

static/index.html

@@ -41,6 +41,7 @@ found in the LICENSE file.
   <script type="text/javascript">
     google.setOnLoadCallback(drawVersionChart);
     google.setOnLoadCallback(drawMovementChart);
+    google.setOnLoadCallback(drawPerformanceChart);
 
     function drawVersionChart() {
       var jsonData = $.ajax({url: "summary.json", dataType:"json", async: false}).responseText;
@@ -97,6 +98,41 @@ found in the LICENSE file.
       var chart = new google.visualization.AreaChart(document.getElementById('movementChart'));
       chart.draw(data, options);
     }
+
+    function drawPerformanceChart() {
+      var jsonData = $.ajax({url: "performance.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      data.addColumn('number', 'Hash Performance (MiB/s)');
+      data.addColumn('number', 'Memory Usage (MiB)');
+
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+        if (rows[i][1] > 500) {
+          rows[i][1] = null;
+        }
+        data.addRow(rows[i]);
+      };
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '940', height: '300'},
+        series: {
+          0: {targetAxisIndex: 0},
+          1: {targetAxisIndex: 1},
+        },
+        vAxes: {
+          0: {title: 'MiB/s'},
+          1: {title: 'MiB'},
+        }
+      };
+
+      var chart = new google.visualization.LineChart(document.getElementById('performanceChart'));
+      chart.draw(data, options);
+    }
   </script>
 </head>
 
@@ -119,9 +155,11 @@ found in the LICENSE file.
         <div class="img-thumbnail" id="movementChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
         <p class="text-muted">
          Reappearance of users cause the "left" data to shrink retroactively.
-         Spikes in December 2014 were due to the unique ID format changing and have been partly removed to avoid skewing the graph scale.
         </p>
 
+        <div class="img-thumbnail" id="performanceChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+
         <h4 id="metrics">Usage Metrics</h4>
         <p>
           This is the aggregated usage report data for the last 24 hours. Data based on <b>{{.nodes}}</b> devices that have reported in.

static/performance.html

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<!--
+Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
+Use of this source code is governed by an MIT-style license that can be
+found in the LICENSE file.
+-->
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="shortcut icon" href="assets/img/favicon.png">
+
+  <title>Historical Performance Data</title>
+  <link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
+  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+  <script type="text/javascript" src="bootstrap/js/bootstrap.min.js"></script>
+  <style type="text/css">
+    body {
+      margin: 40px;
+      font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    }
+    tr.main td {
+      font-weight: bold;
+    }
+    tr.child td.first {
+      padding-left: 2em;
+    }
+  </style>
+  <script type="text/javascript"
+          src="https://www.google.com/jsapi?autoload={
+            'modules':[{
+              'name':'visualization',
+              'version':'1',
+              'packages':['corechart']
+            }]
+          }"></script>
+
+  <script type="text/javascript">
+    google.setOnLoadCallback(drawPerformanceCharts);
+
+    function drawPerformanceCharts() {
+      var jsonData = $.ajax({url: "/performance.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+      }
+
+      drawChart(rows, 1, 'Total Number of Files', 'totFilesChart', 1e6, 1);
+      drawChart(rows, 2, 'Total Folder Size (GiB)', 'totMiBChart', 1e6, 1024);
+      drawChart(rows, 3, 'Hash Performance (MiB/s)', 'hashPerfChart', 1000, 1);
+      drawChart(rows, 4, 'System RAM Size (GiB)', 'memSizeChart', 1e6, 1024);
+      drawChart(rows, 5, 'Memory Usage (MiB)', 'memUsageChart', 250, 1);
+    }
+
+    // 		{"Day", "TotFiles", "TotMiB", "SHA256Perf", "MemorySize", "MemoryUsageMiB"},
+
+    function drawChart(rows, index, title, id, cutoff, divisor) {
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      data.addColumn('number', title);
+
+      var row;
+      for (var i = 1; i < rows.length; i++){
+          row = [rows[i][0], rows[i][index] / divisor];
+        if (row[1] > cutoff) {
+          row[1] = null;
+        }
+        data.addRow(row);
+      }
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+        vAxes: {0: {minValue: 0}},
+      };
+
+      var chart = new google.visualization.LineChart(document.getElementById(id));
+      chart.draw(data, options);
+    }
+  </script>
+</head>
+
+<body>
+  <div class="container">
+    <div class="row">
+      <div class="col-md-12">
+        <h1>Historical Performance Data</h1>
+        <p>These charts are all the average of the corresponding metric, for the entire population of a given day.</p>
+
+        <h4 id="active-users">Hash Performance (MiB/s)</h4>
+        <div class="img-thumbnail" id="hashPerfChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="active-users">Memory Usage (MiB)</h4>
+        <div class="img-thumbnail" id="memUsageChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="active-users">Total Number of Files</h4>
+        <div class="img-thumbnail" id="totFilesChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="active-users">Total Folder Size (GiB)</h4>
+        <div class="img-thumbnail" id="totMiBChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="active-users">System RAM Size (GiB)</h4>
+        <div class="img-thumbnail" id="memSizeChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+      </div>
+  </div>
+</body>
+</html>