mirror of
https://github.com/octoleo/syncthing.git
synced 2025-01-22 14:48:30 +00:00
Add user movement graph
This commit is contained in:
Jakob Borg
parent
ca96b13233
commit
0890b49e78
@ -2,10 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
_ "github.com/lib/pq"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
var dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
|
||||
@ -38,9 +39,16 @@ func main() {
|
||||
}
|
||||
|
||||
func runAggregation(db *sql.DB) {
|
||||
since := maxIndexedDay(db)
|
||||
log.Println("Aggregating data since", since)
|
||||
rows, err := aggregate(db, since)
|
||||
since := maxIndexedDay(db, "VersionSummary")
|
||||
log.Println("Aggregating VersionSummary data since", since)
|
||||
rows, err := aggregateVersionSummary(db, since)
|
||||
if err != nil {
|
||||
log.Fatalln("aggregate:", err)
|
||||
}
|
||||
log.Println("Inserted", rows, "rows")
|
||||
|
||||
log.Println("Aggregating UserMovement data")
|
||||
rows, err = aggregateUserMovement(db)
|
||||
if err != nil {
|
||||
log.Fatalln("aggregate:", err)
|
||||
}
|
||||
@ -64,6 +72,15 @@ func setupDB(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS UserMovement (
|
||||
Day TIMESTAMP NOT NULL,
|
||||
Added INTEGER NOT NULL,
|
||||
Removed INTEGER NOT NULL
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
|
||||
if err := row.Scan(nil); err != nil {
|
||||
_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
|
||||
@ -74,12 +91,17 @@ func setupDB(db *sql.DB) error {
|
||||
_, err = db.Exec(`CREATE INDEX DayIndex ON VerionSummary (Day)`)
|
||||
}
|
||||
|
||||
row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
|
||||
if err := row.Scan(nil); err != nil {
|
||||
_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func maxIndexedDay(db *sql.DB) time.Time {
|
||||
func maxIndexedDay(db *sql.DB, table string) time.Time {
|
||||
var t time.Time
|
||||
row := db.QueryRow("SELECT MAX(Day) FROM VersionSummary")
|
||||
row := db.QueryRow("SELECT MAX(Day) FROM " + table)
|
||||
err := row.Scan(&t)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
@ -87,7 +109,7 @@ func maxIndexedDay(db *sql.DB) time.Time {
|
||||
return t
|
||||
}
|
||||
|
||||
func aggregate(db *sql.DB, since time.Time) (int64, error) {
|
||||
func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
|
||||
res, err := db.Exec(`INSERT INTO VersionSummary (
|
||||
SELECT
|
||||
DATE_TRUNC('day', Received) AS Day,
|
||||
@ -107,3 +129,76 @@ func aggregate(db *sql.DB, since time.Time) (int64, error) {
|
||||
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func aggregateUserMovement(db *sql.DB) (int64, error) {
|
||||
rows, err := db.Query(`SELECT
|
||||
DATE_TRUNC('day', Received) AS Day,
|
||||
UniqueID
|
||||
FROM Reports
|
||||
WHERE
|
||||
DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
|
||||
AND Version like 'v0.%'
|
||||
ORDER BY Day
|
||||
`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
firstSeen := make(map[string]time.Time)
|
||||
lastSeen := make(map[string]time.Time)
|
||||
var minTs time.Time
|
||||
|
||||
for rows.Next() {
|
||||
var ts time.Time
|
||||
var id string
|
||||
if err := rows.Scan(&ts, &id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if minTs.IsZero() {
|
||||
minTs = ts
|
||||
}
|
||||
if _, ok := firstSeen[id]; !ok {
|
||||
firstSeen[id] = ts
|
||||
}
|
||||
lastSeen[id] = ts
|
||||
}
|
||||
|
||||
type sumRow struct {
|
||||
day time.Time
|
||||
added int
|
||||
removed int
|
||||
}
|
||||
var sumRows []sumRow
|
||||
for t := minTs; t.Before(time.Now().Truncate(24 * time.Hour)); t = t.AddDate(0, 0, 1) {
|
||||
var added, removed int
|
||||
for id, first := range firstSeen {
|
||||
last := lastSeen[id]
|
||||
if first.Equal(t) {
|
||||
added++
|
||||
}
|
||||
if last == t && t.Before(time.Now().AddDate(0, 0, -14)) {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
sumRows = append(sumRows, sumRow{t, added, removed})
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.Exec("DELETE FROM UserMovement"); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
for _, r := range sumRows {
|
||||
if _, err := tx.Exec("INSERT INTO UserMovement (Day, Added, Removed) VALUES ($1, $2, $3)", r.day, r.added, r.removed); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return int64(len(sumRows)), tx.Commit()
|
||||
}
|
||||
|
||||
@ -180,6 +180,7 @@ func main() {
|
||||
http.HandleFunc("/", withDB(db, rootHandler))
|
||||
http.HandleFunc("/newdata", withDB(db, newDataHandler))
|
||||
http.HandleFunc("/summary.json", withDB(db, summaryHandler))
|
||||
http.HandleFunc("/movement.json", withDB(db, movementHandler))
|
||||
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
|
||||
|
||||
err = srv.Serve(listener)
|
||||
@ -269,6 +270,25 @@ func summaryHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(bs)
|
||||
}
|
||||
|
||||
func movementHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
|
||||
s, err := getMovement(db)
|
||||
if err != nil {
|
||||
log.Println("movementHandler:", err)
|
||||
http.Error(w, "Database Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
bs, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
log.Println("movementHandler:", err)
|
||||
http.Error(w, "JSON Encode Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(bs)
|
||||
}
|
||||
|
||||
type category struct {
|
||||
Values [4]float64
|
||||
Key string
|
||||
@ -554,3 +574,33 @@ func getSummary(db *sql.DB) (summary, error) {
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func getMovement(db *sql.DB) ([][]interface{}, error) {
|
||||
rows, err := db.Query(`SELECT Day, Added, Removed FROM UserMovement WHERE Day > now() - '1 year'::INTERVAL ORDER BY Day`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
res := [][]interface{}{
|
||||
{"Day", "Joined", "Left"},
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var day time.Time
|
||||
var added, removed int
|
||||
err := rows.Scan(&day, &added, &removed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
row := []interface{}{day.Format("2006-01-02"), added, -removed}
|
||||
if removed == 0 {
|
||||
row[2] = nil
|
||||
}
|
||||
|
||||
res = append(res, row)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@ -15,14 +15,14 @@ found in the LICENSE file.
|
||||
|
||||
<title>Syncthing Usage Reports</title>
|
||||
<link href="static/bootstrap/css/bootstrap.min.css" rel="stylesheet">
|
||||
<script src="static/bootstrap/js/bootstrap.min.js"></script>
|
||||
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script type="text/javascript" src="static/bootstrap/js/bootstrap.min.js"></script>
|
||||
<style type="text/css">
|
||||
body {
|
||||
margin: 40px;
|
||||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
}
|
||||
</style>
|
||||
<script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="https://www.google.com/jsapi?autoload={
|
||||
'modules':[{
|
||||
@ -33,9 +33,10 @@ found in the LICENSE file.
|
||||
}"></script>
|
||||
|
||||
<script type="text/javascript">
|
||||
google.setOnLoadCallback(drawChart);
|
||||
google.setOnLoadCallback(drawVersionChart);
|
||||
google.setOnLoadCallback(drawMovementChart);
|
||||
|
||||
function drawChart() {
|
||||
function drawVersionChart() {
|
||||
var jsonData = $.ajax({url: "summary.json", dataType:"json", async: false}).responseText;
|
||||
var rows = JSON.parse(jsonData);
|
||||
|
||||
@ -56,7 +57,37 @@ found in the LICENSE file.
|
||||
chartArea: {left: 80, top: 20, width: '1020', height: '300'},
|
||||
};
|
||||
|
||||
var chart = new google.visualization.AreaChart(document.getElementById('curve_chart'));
|
||||
var chart = new google.visualization.AreaChart(document.getElementById('versionChart'));
|
||||
chart.draw(data, options);
|
||||
}
|
||||
|
||||
function drawMovementChart() {
|
||||
var jsonData = $.ajax({url: "movement.json", dataType:"json", async: false}).responseText;
|
||||
var rows = JSON.parse(jsonData);
|
||||
|
||||
var data = new google.visualization.DataTable();
|
||||
data.addColumn('date', 'Day');
|
||||
for (var i = 1; i < rows[0].length; i++){
|
||||
data.addColumn('number', rows[0][i]);
|
||||
}
|
||||
for (var i = 1; i < rows.length; i++){
|
||||
rows[i][0] = new Date(rows[i][0]);
|
||||
if (rows[i][1] > 500) {
|
||||
rows[i][1] = null;
|
||||
}
|
||||
if (rows[i][2] < -500) {
|
||||
rows[i][2] = null;
|
||||
}
|
||||
data.addRow(rows[i]);
|
||||
};
|
||||
|
||||
var options = {
|
||||
legend: { position: 'bottom', alignment: 'center' },
|
||||
colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
|
||||
chartArea: {left: 80, top: 20, width: '1020', height: '300'},
|
||||
};
|
||||
|
||||
var chart = new google.visualization.AreaChart(document.getElementById('movementChart'));
|
||||
chart.draw(data, options);
|
||||
}
|
||||
</script>
|
||||
@ -72,7 +103,17 @@ found in the LICENSE file.
|
||||
<p>
|
||||
This is the total number of unique users with reporting enabled, per day. Area color represents the major version.
|
||||
</p>
|
||||
<div class="img-thumbnail" id="curve_chart" style="width: 1130px; height: 400px; padding: 10px;"></div>
|
||||
<div class="img-thumbnail" id="versionChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
|
||||
|
||||
<h4>Users Joining and Leaving per Day</h4>
|
||||
<p>
|
||||
This is the total number of unique users joining and leaving per day. A user is counted as "joined" on first the day their unique ID is seen, and as "left" on the last day the unique ID was seen before a two weeks or longer absence.
|
||||
</p>
|
||||
<div class="img-thumbnail" id="movementChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
|
||||
<p class="text-muted">
|
||||
Reappearance of users cause the "left" data to shrink retroactively.
|
||||
Spikes in December 2014 were due to the unique ID format changing and have been partly removed to avoid skewing the graph scale.
|
||||
</p>
|
||||
|
||||
<h4>Usage Metrics</h4>
|
||||
<p>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user