From 9afbca300147d476088c7778b5452a83c873dcb9 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 25 Apr 2015 22:53:44 +0100 Subject: [PATCH] Add pagination to Out of sync item list (fixes #1509) --- cmd/syncthing/gui.go | 26 +++- gui/index.html | 43 ++++--- gui/scripts/syncthing/app.js | 1 + .../core/controllers/syncthingController.js | 82 ++++++++++--- internal/model/model.go | 116 +++++++++++------- 5 files changed, 184 insertions(+), 84 deletions(-) diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 6bcb6d173..e7a5526b6 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -111,7 +111,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro getRestMux.HandleFunc("/rest/db/completion", withModel(m, restGetDBCompletion)) // device folder getRestMux.HandleFunc("/rest/db/file", withModel(m, restGetDBFile)) // folder file getRestMux.HandleFunc("/rest/db/ignores", withModel(m, restGetDBIgnores)) // folder - getRestMux.HandleFunc("/rest/db/need", withModel(m, restGetDBNeed)) // folder + getRestMux.HandleFunc("/rest/db/need", withModel(m, restGetDBNeed)) // folder [perpage] [page] getRestMux.HandleFunc("/rest/db/status", withModel(m, restGetDBStatus)) // folder getRestMux.HandleFunc("/rest/db/browse", withModel(m, restGetDBBrowse)) // folder [prefix] [dirsonly] [levels] getRestMux.HandleFunc("/rest/events", restGetEvents) // since [limit] @@ -133,7 +133,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro // The POST handlers postRestMux := http.NewServeMux() - postRestMux.HandleFunc("/rest/db/prio", withModel(m, restPostDBPrio)) // folder file + postRestMux.HandleFunc("/rest/db/prio", withModel(m, restPostDBPrio)) // folder file [perpage] [page] postRestMux.HandleFunc("/rest/db/ignores", withModel(m, restPostDBIgnores)) // folder postRestMux.HandleFunc("/rest/db/override", withModel(m, restPostDBOverride)) // folder postRestMux.HandleFunc("/rest/db/scan", withModel(m, restPostDBScan)) // folder [sub...] @@ -379,15 +379,29 @@ func restPostDBOverride(m *model.Model, w http.ResponseWriter, r *http.Request) } func restGetDBNeed(m *model.Model, w http.ResponseWriter, r *http.Request) { - var qs = r.URL.Query() - var folder = qs.Get("folder") + qs := r.URL.Query() + + folder := qs.Get("folder") + + page, err := strconv.Atoi(qs.Get("page")) + if err != nil || page < 1 { + page = 1 + } + perpage, err := strconv.Atoi(qs.Get("perpage")) + if err != nil || perpage < 1 { + perpage = 1 << 16 + } + + progress, queued, rest, total := m.NeedFolderFiles(folder, page, perpage) - progress, queued, rest := m.NeedFolderFiles(folder, 100) // Convert the struct to a more loose structure, and inject the size. - output := map[string][]jsonDBFileInfo{ + output := map[string]interface{}{ "progress": toNeedSlice(progress), "queued": toNeedSlice(queued), "rest": toNeedSlice(rest), + "total": total, + "page": page, + "perpage": perpage, } w.Header().Set("Content-Type", "application/json; charset=utf-8") diff --git a/gui/index.html b/gui/index.html index d16d7764c..2fef953d0 100644 --- a/gui/index.html +++ b/gui/index.html @@ -967,10 +967,22 @@
- - - - + + + + + + + + + - - - - - - - - - - - +
{{needActions[a]}}{{f.name | basename}} + +
{{needActions[f.action]}}{{f.name | basename}} + + + +  {{f.name | basename}} +
@@ -982,24 +994,20 @@
+ {{f.size | binary}}B
{{needActions[a]}} {{f.name | basename}} - {{f.size | binary}}B -
{{needActions[a]}}{{f.name | basename}}{{f.size | binary}}B
+ + +
@@ -1087,6 +1095,7 @@ + diff --git a/gui/scripts/syncthing/app.js b/gui/scripts/syncthing/app.js index d53bb93e3..7f04e3d45 100644 --- a/gui/scripts/syncthing/app.js +++ b/gui/scripts/syncthing/app.js @@ -9,6 +9,7 @@ /*global $: false, angular: false, console: false, validLangs: false */ var syncthing = angular.module('syncthing', [ + 'angularUtils.directives.dirPagination', 'pascalprecht.translate', 'syncthing.core' diff --git a/gui/scripts/syncthing/core/controllers/syncthingController.js b/gui/scripts/syncthing/core/controllers/syncthingController.js index 38b68a7a1..84fd71029 100644 --- a/gui/scripts/syncthing/core/controllers/syncthingController.js +++ b/gui/scripts/syncthing/core/controllers/syncthingController.js @@ -40,6 +40,10 @@ angular.module('syncthing.core') $scope.folderStats = {}; $scope.progress = {}; $scope.version = {}; + $scope.needed = []; + $scope.neededTotal = 0; + $scope.neededCurrentPage = 1; + $scope.neededPageSize = 10; $(window).bind('beforeunload', function () { navigatingAway = true; @@ -415,14 +419,63 @@ angular.module('syncthing.core') } function refreshNeed(folder) { - $http.get(urlbase + "/db/need?folder=" + encodeURIComponent(folder)).success(function (data) { + var url = urlbase + "/db/need?folder=" + encodeURIComponent(folder); + url += "&page=" + $scope.neededCurrentPage; + url += "&perpage=" + $scope.neededPageSize; + $http.get(url).success(function (data) { if ($scope.neededFolder == folder) { console.log("refreshNeed", folder, data); - $scope.needed = data; + parseNeeded(data); } }).error($scope.emitHTTPError); } + function needAction(file) { + var fDelete = 4096; + var fDirectory = 16384; + + if ((file.flags & (fDelete + fDirectory)) === fDelete + fDirectory) { + return 'rmdir'; + } else if ((file.flags & fDelete) === fDelete) { + return 'rm'; + } else if ((file.flags & fDirectory) === fDirectory) { + return 'touch'; + } else { + return 'sync'; + } + }; + + function parseNeeded(data) { + var merged = []; + data.progress.forEach(function (item) { + item.type = "progress"; + item.action = needAction(item); + merged.push(item); + }); + data.queued.forEach(function (item) { + item.type = "queued"; + item.action = needAction(item); + merged.push(item); + }); + data.rest.forEach(function (item) { + item.type = "rest"; + item.action = needAction(item); + merged.push(item); + }); + $scope.needed = merged; + $scope.neededTotal = data.total; + } + + $scope.neededPageChanged = function (page) { + $scope.neededCurrentPage = page; + refreshNeed($scope.neededFolder); + }; + + $scope.neededChangePageSize = function (perpage) { + $scope.neededPageSize = perpage; + refreshNeed($scope.neededFolder); + } + var refreshDeviceStats = debounce(function () { $http.get(urlbase + "/stats/device").success(function (data) { $scope.deviceStats = data; @@ -1181,24 +1234,11 @@ angular.module('syncthing.core') $('#needed').modal().on('hidden.bs.modal', function () { $scope.neededFolder = undefined; $scope.needed = undefined; + $scope.neededTotal = 0; + $scope.neededCurrentPage = 1; }); }; - $scope.needAction = function (file) { - var fDelete = 4096; - var fDirectory = 16384; - - if ((file.flags & (fDelete + fDirectory)) === fDelete + fDirectory) { - return 'rmdir'; - } else if ((file.flags & fDelete) === fDelete) { - return 'rm'; - } else if ((file.flags & fDirectory) === fDirectory) { - return 'touch'; - } else { - return 'sync'; - } - }; - $scope.override = function (folder) { $http.post(urlbase + "/db/override?folder=" + encodeURIComponent(folder)); }; @@ -1220,10 +1260,14 @@ angular.module('syncthing.core') }; $scope.bumpFile = function (folder, file) { - $http.post(urlbase + "/db/prio?folder=" + encodeURIComponent(folder) + "&file=" + encodeURIComponent(file)).success(function (data) { + var url = urlbase + "/db/prio?folder=" + encodeURIComponent(folder) + "&file=" + encodeURIComponent(file); + // In order to get the right view of data in the response. + url += "&page=" + $scope.neededCurrentPage; + url += "&perpage=" + $scope.neededPageSize; + $http.post(url).success(function (data) { if ($scope.neededFolder == folder) { console.log("bumpFile", folder, data); - $scope.needed = data; + parseNeeded(data); } }).error($scope.emitHTTPError); }; diff --git a/internal/model/model.go b/internal/model/model.go index 37bc01161..853c6f586 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -375,53 +375,71 @@ func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) { return } -// NeedFiles returns the list of currently needed files in progress, queued, -// and to be queued on next puller iteration. Also takes a soft cap which is -// only respected when adding files from the model rather than the runner queue. -func (m *Model) NeedFolderFiles(folder string, max int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) { +// NeedFiles returns paginated list of currently needed files in progress, queued, +// and to be queued on next puller iteration, as well as the total number of +// files currently needed. +func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, int) { m.fmut.RLock() defer m.fmut.RUnlock() - if rf, ok := m.folderFiles[folder]; ok { - var progress, queued, rest []db.FileInfoTruncated - var seen map[string]bool + total := 0 - runner, ok := m.folderRunners[folder] - if ok { - progressNames, queuedNames := runner.Jobs() - - progress = make([]db.FileInfoTruncated, len(progressNames)) - queued = make([]db.FileInfoTruncated, len(queuedNames)) - seen = make(map[string]bool, len(progressNames)+len(queuedNames)) - - for i, name := range progressNames { - if f, ok := rf.GetGlobalTruncated(name); ok { - progress[i] = f - seen[name] = true - } - } - - for i, name := range queuedNames { - if f, ok := rf.GetGlobalTruncated(name); ok { - queued[i] = f - seen[name] = true - } - } - } - left := max - len(progress) - len(queued) - if max < 1 || left > 0 { - rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { - left-- - ft := f.(db.FileInfoTruncated) - if !seen[ft.Name] { - rest = append(rest, ft) - } - return max < 1 || left > 0 - }) - } - return progress, queued, rest + rf, ok := m.folderFiles[folder] + if !ok { + return nil, nil, nil, 0 } - return nil, nil, nil + + var progress, queued, rest []db.FileInfoTruncated + var seen map[string]struct{} + + skip := (page - 1) * perpage + get := perpage + + runner, ok := m.folderRunners[folder] + if ok { + allProgressNames, allQueuedNames := runner.Jobs() + + var progressNames, queuedNames []string + progressNames, skip, get = getChunk(allProgressNames, skip, get) + queuedNames, skip, get = getChunk(allQueuedNames, skip, get) + + progress = make([]db.FileInfoTruncated, len(progressNames)) + queued = make([]db.FileInfoTruncated, len(queuedNames)) + seen = make(map[string]struct{}, len(progressNames)+len(queuedNames)) + + for i, name := range progressNames { + if f, ok := rf.GetGlobalTruncated(name); ok { + progress[i] = f + seen[name] = struct{}{} + } + } + + for i, name := range queuedNames { + if f, ok := rf.GetGlobalTruncated(name); ok { + queued[i] = f + seen[name] = struct{}{} + } + } + } + + rest = make([]db.FileInfoTruncated, 0, perpage) + rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { + total++ + if skip > 0 { + skip-- + return true + } + if get > 0 { + ft := f.(db.FileInfoTruncated) + if _, ok := seen[ft.Name]; !ok { + rest = append(rest, ft) + get-- + } + } + return true + }) + + return progress, queued, rest, total } // Index is called when a new device is connected and we receive their full index. @@ -1616,3 +1634,17 @@ func symlinkInvalid(isLink bool) bool { } return false } + +// Skips `skip` elements and retrieves up to `get` elements from a given slice. +// Returns the resulting slice, plus how much elements are left to skip or +// copy to satisfy the values which were provided, given the slice is not +// big enough. +func getChunk(data []string, skip, get int) ([]string, int, int) { + l := len(data) + if l <= skip { + return []string{}, skip - l, get + } else if l < skip+get { + return data[skip:l], 0, get - (l - skip) + } + return data[skip : skip+get], 0, 0 +}