Add user movement graph

cmd/aggregate/main.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"database/sql"
-	_ "github.com/lib/pq"
 	"log"
 	"os"
 	"time"
+
+	_ "github.com/lib/pq"
 )
 
 var dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
@@ -38,9 +39,16 @@ func main() {
 }
 
 func runAggregation(db *sql.DB) {
-	since := maxIndexedDay(db)
+	since := maxIndexedDay(db, "VersionSummary")
-	log.Println("Aggregating data since", since)
+	log.Println("Aggregating VersionSummary data since", since)
-	rows, err := aggregate(db, since)
+	rows, err := aggregateVersionSummary(db, since)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
+
+	log.Println("Aggregating UserMovement data")
+	rows, err = aggregateUserMovement(db)
 	if err != nil {
 		log.Fatalln("aggregate:", err)
 	}
@@ -64,6 +72,15 @@ func setupDB(db *sql.DB) error {
 		return err
 	}
 
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS UserMovement (
+		Day TIMESTAMP NOT NULL,
+		Added INTEGER NOT NULL,
+		Removed INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
 	row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
 	if err := row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
@@ -74,12 +91,17 @@ func setupDB(db *sql.DB) error {
 		_, err = db.Exec(`CREATE INDEX DayIndex ON VerionSummary (Day)`)
 	}
 
+	row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
+	if err := row.Scan(nil); err != nil {
+		_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
+	}
+
 	return err
 }
 
-func maxIndexedDay(db *sql.DB) time.Time {
+func maxIndexedDay(db *sql.DB, table string) time.Time {
 	var t time.Time
-	row := db.QueryRow("SELECT MAX(Day) FROM VersionSummary")
+	row := db.QueryRow("SELECT MAX(Day) FROM " + table)
 	err := row.Scan(&t)
 	if err != nil {
 		return time.Time{}
@@ -87,7 +109,7 @@ func maxIndexedDay(db *sql.DB) time.Time {
 	return t
 }
 
-func aggregate(db *sql.DB, since time.Time) (int64, error) {
+func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
 	res, err := db.Exec(`INSERT INTO VersionSummary (
 	SELECT
 		DATE_TRUNC('day', Received) AS Day,
@@ -107,3 +129,76 @@ func aggregate(db *sql.DB, since time.Time) (int64, error) {
 
 	return res.RowsAffected()
 }
+
+func aggregateUserMovement(db *sql.DB) (int64, error) {
+	rows, err := db.Query(`SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		UniqueID
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Version like 'v0.%'
+		ORDER BY Day
+	`)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	firstSeen := make(map[string]time.Time)
+	lastSeen := make(map[string]time.Time)
+	var minTs time.Time
+
+	for rows.Next() {
+		var ts time.Time
+		var id string
+		if err := rows.Scan(&ts, &id); err != nil {
+			return 0, err
+		}
+
+		if minTs.IsZero() {
+			minTs = ts
+		}
+		if _, ok := firstSeen[id]; !ok {
+			firstSeen[id] = ts
+		}
+		lastSeen[id] = ts
+	}
+
+	type sumRow struct {
+		day     time.Time
+		added   int
+		removed int
+	}
+	var sumRows []sumRow
+	for t := minTs; t.Before(time.Now().Truncate(24 * time.Hour)); t = t.AddDate(0, 0, 1) {
+		var added, removed int
+		for id, first := range firstSeen {
+			last := lastSeen[id]
+			if first.Equal(t) {
+				added++
+			}
+			if last == t && t.Before(time.Now().AddDate(0, 0, -14)) {
+				removed++
+			}
+		}
+		sumRows = append(sumRows, sumRow{t, added, removed})
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec("DELETE FROM UserMovement"); err != nil {
+		tx.Rollback()
+		return 0, err
+	}
+	for _, r := range sumRows {
+		if _, err := tx.Exec("INSERT INTO UserMovement (Day, Added, Removed) VALUES ($1, $2, $3)", r.day, r.added, r.removed); err != nil {
+			tx.Rollback()
+			return 0, err
+		}
+	}
+
+	return int64(len(sumRows)), tx.Commit()
+}

cmd/ursrv/main.go

@@ -180,6 +180,7 @@ func main() {
 	http.HandleFunc("/", withDB(db, rootHandler))
 	http.HandleFunc("/newdata", withDB(db, newDataHandler))
 	http.HandleFunc("/summary.json", withDB(db, summaryHandler))
+	http.HandleFunc("/movement.json", withDB(db, movementHandler))
 	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
 
 	err = srv.Serve(listener)
@@ -269,6 +270,25 @@ func summaryHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 	w.Write(bs)
 }
 
+func movementHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
+	s, err := getMovement(db)
+	if err != nil {
+		log.Println("movementHandler:", err)
+		http.Error(w, "Database Error", http.StatusInternalServerError)
+		return
+	}
+
+	bs, err := json.Marshal(s)
+	if err != nil {
+		log.Println("movementHandler:", err)
+		http.Error(w, "JSON Encode Error", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
 type category struct {
 	Values [4]float64
 	Key    string
@@ -554,3 +574,33 @@ func getSummary(db *sql.DB) (summary, error) {
 
 	return s, nil
 }
+
+func getMovement(db *sql.DB) ([][]interface{}, error) {
+	rows, err := db.Query(`SELECT Day, Added, Removed FROM UserMovement WHERE Day > now() - '1 year'::INTERVAL ORDER BY Day`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	res := [][]interface{}{
+		{"Day", "Joined", "Left"},
+	}
+
+	for rows.Next() {
+		var day time.Time
+		var added, removed int
+		err := rows.Scan(&day, &added, &removed)
+		if err != nil {
+			return nil, err
+		}
+
+		row := []interface{}{day.Format("2006-01-02"), added, -removed}
+		if removed == 0 {
+			row[2] = nil
+		}
+
+		res = append(res, row)
+	}
+
+	return res, nil
+}

static/index.html

@@ -15,14 +15,14 @@ found in the LICENSE file.
 
   <title>Syncthing Usage Reports</title>
   <link href="static/bootstrap/css/bootstrap.min.css" rel="stylesheet">
-  <script src="static/bootstrap/js/bootstrap.min.js"></script>
+  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+  <script type="text/javascript" src="static/bootstrap/js/bootstrap.min.js"></script>
   <style type="text/css">
     body {
       margin: 40px;
       font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
     }
   </style>
-  <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
   <script type="text/javascript"
           src="https://www.google.com/jsapi?autoload={
             'modules':[{
@@ -33,9 +33,10 @@ found in the LICENSE file.
           }"></script>
 
   <script type="text/javascript">
-    google.setOnLoadCallback(drawChart);
+    google.setOnLoadCallback(drawVersionChart);
+    google.setOnLoadCallback(drawMovementChart);
 
-    function drawChart() {
+    function drawVersionChart() {
       var jsonData = $.ajax({url: "summary.json", dataType:"json", async: false}).responseText;
       var rows = JSON.parse(jsonData);
 
@@ -56,7 +57,37 @@ found in the LICENSE file.
         chartArea: {left: 80, top: 20, width: '1020', height: '300'},
       };
 
-      var chart = new google.visualization.AreaChart(document.getElementById('curve_chart'));
+      var chart = new google.visualization.AreaChart(document.getElementById('versionChart'));
+      chart.draw(data, options);
+    }
+
+    function drawMovementChart() {
+      var jsonData = $.ajax({url: "movement.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      for (var i = 1; i < rows[0].length; i++){
+        data.addColumn('number', rows[0][i]);
+      }
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+        if (rows[i][1] > 500) {
+          rows[i][1] = null;
+        }
+        if (rows[i][2] < -500) {
+          rows[i][2] = null;
+        }
+        data.addRow(rows[i]);
+      };
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+      };
+
+      var chart = new google.visualization.AreaChart(document.getElementById('movementChart'));
       chart.draw(data, options);
     }
   </script>
@@ -72,7 +103,17 @@ found in the LICENSE file.
         <p>
           This is the total number of unique users with reporting enabled, per day. Area color represents the major version.
         </p>
-        <div class="img-thumbnail" id="curve_chart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+        <div class="img-thumbnail" id="versionChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4>Users Joining and Leaving per Day</h4>
+        <p>
+          This is the total number of unique users joining and leaving per day. A user is counted as "joined" on first the day their unique ID is seen, and as "left" on the last day the unique ID was seen before a two weeks or longer absence.
+        </p>
+        <div class="img-thumbnail" id="movementChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+        <p class="text-muted">
+         Reappearance of users cause the "left" data to shrink retroactively.
+         Spikes in December 2014 were due to the unique ID format changing and have been partly removed to avoid skewing the graph scale.
+        </p>
 
         <h4>Usage Metrics</h4>
         <p>