octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-22 19:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2001 from calmh/failed-files

Browse Source 
Show failed files in web UI
This commit is contained in:
Audrius Butkevicius 2015-06-30 15:26:24 +01:00
commit a03c9f9457
9 changed files with 234 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
gui/assets/lang/lang-en.json
View File

@ -50,6 +50,7 @@
"Enter ignore patterns, one per line.": "Enter ignore patterns, one per line.",
"Error": "Error",
"External File Versioning": "External File Versioning",
"Failed Items": "Failed Items",
"File Pull Order": "File Pull Order",
"File Versioning": "File Versioning",
"File permission bits are ignored when looking for changes. Use on FAT file systems.": "File permission bits are ignored when looking for changes. Use on FAT file systems.",
@ -97,7 +98,7 @@
"OK": "OK",
"Off": "Off",
"Oldest First": "Oldest First",
"Out Of Sync": "Out Of Sync",
"Out of Sync": "Out of Sync",
"Out of Sync Items": "Out of Sync Items",
"Outgoing Rate Limit (KiB/s)": "Outgoing Rate Limit (KiB/s)",
"Override Changes": "Override Changes",
@ -163,6 +164,7 @@
"The folder ID must be unique.": "The folder ID must be unique.",
"The folder path cannot be blank.": "The folder path cannot be blank.",
"The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.",
"The following items could not be synchronized.": "The following items could not be synchronized.",
"The maximum age must be a number and cannot be blank.": "The maximum age must be a number and cannot be blank.",
"The maximum time to keep a version (in days, set to 0 to keep versions forever).": "The maximum time to keep a version (in days, set to 0 to keep versions forever).",
"The number of days must be a number and cannot be blank.": "The number of days must be a number and cannot be blank.",
@ -171,6 +173,7 @@
"The number of versions must be a number and cannot be blank.": "The number of versions must be a number and cannot be blank.",
"The path cannot be blank.": "The path cannot be blank.",
"The rescan interval must be a non-negative number of seconds.": "The rescan interval must be a non-negative number of seconds.",
"They are retried automatically and will be synced when the error is resolved.": "They are retried automatically and will be synced when the error is resolved.",
"This is a major version upgrade.": "This is a major version upgrade.",
"Trash Can File Versioning": "Trash Can File Versioning",
"Unknown": "Unknown",

41
gui/index.html
View File

@ -196,6 +196,7 @@
<span class="hidden-xs" translate>Syncing</span>
({{syncPercentage(folder.id)}}%)
</span>
<span ng-switch-when="outofsync"><span class="hidden-xs" translate>Out of Sync</span><span class="visible-xs">&#9724;</span></span>
</span>
</h3>
</div>
@ -225,6 +226,17 @@
<a ng-click="showNeed(folder.id)" href="">{{model[folder.id].needFiles | alwaysNumber}} <span translate>items</span>, ~{{model[folder.id].needBytes | binary}}B</a>
</td>
</tr>
<tr ng-if="folderStatus(folder) === 'outofsync' || hasFailedFiles(folder.id)">
<th><span class="glyphicon glyphicon-exclamation-sign"></span>&nbsp;<span translate>Failed Items</span></th>
<!-- Show the number of failed items as a link to bring up the list. -->
<td ng-if="hasFailedFiles(folder.id)" class="text-right">
<a ng-click="showFailed(folder.id)" href="">{{failed[folder.id].length | alwaysNumber}}&nbsp;<span translate>items</span></a>
</td>
<!-- The list of failed items hasn't loaded yet; show an ellipsis for the time being. -->
<td ng-if="!hasFailedFiles(folder.id)" class="text-right">
...
</td>
</tr>
<tr ng-if="folder.readOnly">
<th><span class="glyphicon glyphicon-lock"></span>&nbsp;<span translate>Folder Master</span></th>
<td class="text-right">
@ -985,7 +997,7 @@
<table class="table table-striped table-condensed">
<tr dir-paginate="f in needed | itemsPerPage: neededPageSize" current-page="neededCurrentPage" total-items="neededTotal">
<tr dir-paginate="f in needed | itemsPerPage: neededPageSize" current-page="neededCurrentPage" total-items="neededTotal" pagination-id="needed">
<!-- Icon -->
<td class="small-data"><span class="glyphicon glyphicon-{{needIcons[f.action]}}"></span> {{needActions[f.action]}}</td>
@ -1018,15 +1030,37 @@
</tr>
</table>
<dir-pagination-controls on-page-change="neededPageChanged(newPageNumber)"></dir-pagination-controls>
<dir-pagination-controls on-page-change="neededPageChanged(newPageNumber)" pagination-id="needed"></dir-pagination-controls>
<ul class="pagination pull-right">
<li ng-repeat="option in [10, 20, 30, 50, 100]" ng-class="{ active: neededPageSize == option }">
<li ng-repeat="option in [10, 25, 50]" ng-class="{ active: neededPageSize == option }">
<a href="#" ng-click="neededChangePageSize(option)">{{option}}</a>
<li>
</ul>
<div class="clearfix"></div>
</modal>
<!-- Failed Items modal -->
<modal id="failed" large="yes" status="warning" icon="exclamation-sign" close="yes" title="{{'Failed Items' | translate}}">
<p>
<span translate>The following items could not be synchronized.</span>
<span translate>They are retried automatically and will be synced when the error is resolved.</span>
</p>
<table class="table table-striped table-condensed">
<tr dir-paginate="e in failedCurrent | itemsPerPage: failedPageSize" current-page="failedCurrentPage" pagination-id="failed">
<td><abbr title="{{e.path}}">{{e.path | basename}}</abbr></td>
<td><abbr title="{{e.error}}">{{e.error | lastErrorComponent}}</abbr></td>
</tr>
</table>
<dir-pagination-controls on-page-change="failedPageChanged(newPageNumber)" pagination-id="failed"></dir-pagination-controls>
<ul class="pagination pull-right">
<li ng-repeat="option in [10, 25, 50]" ng-class="{ active: failedPageSize == option }">
<a href="#" ng-click="failedChangePageSize(option)">{{option}}</a>
<li>
</ul>
<div class="clearfix"></div>
</modal>
<!-- About modal -->
<modal id="about" large="yes" close="yes" status="info" title="{{'About' | translate}}">
@ -1138,6 +1172,7 @@
<script src="scripts/syncthing/core/filters/binaryFilter.js"></script>
<script src="scripts/syncthing/core/filters/durationFilter.js"></script>
<script src="scripts/syncthing/core/filters/naturalFilter.js"></script>
<script src="scripts/syncthing/core/filters/lastErrorComponentFilter.js"></script>
<script src="scripts/syncthing/core/services/localeService.js"></script>
<script src="assets/lang/valid-langs.js"></script>

98
gui/scripts/syncthing/core/controllers/syncthingController.js
View File

@ -18,8 +18,7 @@ angular.module('syncthing.core')
Events.start();
}
// pubic/scope definitions
// public/scope definitions
$scope.completion = {};
$scope.config = {};
@ -47,6 +46,10 @@ angular.module('syncthing.core')
$scope.neededPageSize = 10;
$scope.foldersTotalLocalBytes = 0;
$scope.foldersTotalLocalFiles = 0;
$scope.failed = {};
$scope.failedCurrentPage = 1;
$scope.failedCurrentFolder = undefined;
$scope.failedPageSize = 10;
$(window).bind('beforeunload', function () {
navigatingAway = true;
@ -144,6 +147,13 @@ angular.module('syncthing.core')
if ($scope.model[data.folder]) {
$scope.model[data.folder].state = data.to;
$scope.model[data.folder].error = data.error;
// If a folder has started syncing, then any old list of
// errors is obsolete. We may get a new list of errors very
// shortly though.
if (data.to === 'syncing') {
$scope.failed[data.folder] = [];
}
}
});
@ -151,14 +161,6 @@ angular.module('syncthing.core')
refreshFolderStats();
});
/* currently not using
$scope.$on('Events.REMOTE_INDEX_UPDATED', function (event, arg) {
// Nothing
});
*/
$scope.$on(Events.DEVICE_DISCONNECTED, function (event, arg) {
delete $scope.connections[arg.data.id];
refreshDeviceStats();
@ -284,6 +286,11 @@ angular.module('syncthing.core')
$scope.completion[data.device]._total = tot / cnt;
});
$scope.$on(Events.FOLDER_ERRORS, function (event, arg) {
var data = arg.data;
$scope.failed[data.folder] = data.errors;
});
$scope.emitHTTPError = function (data, status, headers, config) {
$scope.$emit('HTTPError', {data: data, status: status, headers: headers, config: config});
};
@ -492,6 +499,14 @@ angular.module('syncthing.core')
refreshNeed($scope.neededFolder);
};
$scope.failedPageChanged = function (page) {
$scope.failedCurrentPage = page;
};
$scope.failedChangePageSize = function (perpage) {
$scope.failedPageSize = perpage;
};
var refreshDeviceStats = debounce(function () {
$http.get(urlbase + "/stats/device").success(function (data) {
$scope.deviceStats = data;
@ -526,6 +541,11 @@ angular.module('syncthing.core')
return 'unknown';
}
// after restart syncthing process state may be empty
if (!$scope.model[folderCfg.id].state) {
return 'unknown';
}
if (folderCfg.devices.length <= 1) {
return 'unshared';
}
@ -534,47 +554,36 @@ angular.module('syncthing.core')
return 'stopped';
}
if ($scope.model[folderCfg.id].state == 'error') {
var state = '' + $scope.model[folderCfg.id].state;
if (state === 'error') {
return 'stopped'; // legacy, the state is called "stopped" in the GUI
}
// after restart syncthing process state may be empty
if (!$scope.model[folderCfg.id].state) {
return 'unknown';
if (state === 'idle' && $scope.model[folderCfg.id].needFiles > 0) {
return 'outofsync';
}
return '' + $scope.model[folderCfg.id].state;
return state;
};
$scope.folderClass = function (folderCfg) {
if (typeof $scope.model[folderCfg.id] === 'undefined') {
// Unknown
return 'info';
}
var status = $scope.folderStatus(folderCfg);
if (folderCfg.devices.length <= 1) {
// Unshared
return 'warning';
}
if ($scope.model[folderCfg.id].invalid !== '') {
// Errored
return 'danger';
}
var state = '' + $scope.model[folderCfg.id].state;
if (state == 'idle') {
if (status == 'idle') {
return 'success';
}
if (state == 'syncing') {
if (status == 'syncing' || status == 'scanning') {
return 'primary';
}
if (state == 'scanning') {
return 'primary';
if (status === 'unknown') {
return 'info';
}
if (state == 'error') {
if (status === 'unshared') {
return 'warning';
}
if (status === 'stopped' || status === 'outofsync' || status === 'error') {
return 'danger';
}
return 'info';
};
@ -1277,6 +1286,23 @@ angular.module('syncthing.core')
});
};
$scope.showFailed = function (folder) {
$scope.failedCurrent = $scope.failed[folder]
$('#failed').modal().on('hidden.bs.modal', function () {
$scope.failedCurrent = undefined;
});
};
$scope.hasFailedFiles = function (folder) {
if (!$scope.failed[folder]) {
return false;
}
if ($scope.failed[folder].length == 0) {
return false;
}
return true
};
$scope.override = function (folder) {
$http.post(urlbase + "/db/override?folder=" + encodeURIComponent(folder));
};

12
gui/scripts/syncthing/core/filters/lastErrorComponentFilter.js Normal file
View File

@ -0,0 +1,12 @@
angular.module('syncthing.core')
.filter('lastErrorComponent', function () {
return function (input) {
if (input === undefined)
return "";
var parts = input.split(/:\s*/);
if (!parts || parts.length < 1) {
return input;
}
return parts[parts.length - 1];
};
});

1
gui/scripts/syncthing/core/services/events.js
View File

@ -75,6 +75,7 @@ angular.module('syncthing.core')
STARTING: 'Starting', // Emitted exactly once, when Syncthing starts, before parsing configuration etc
STARTUP_COMPLETED: 'StartupCompleted', // Emitted exactly once, when initialization is complete and Syncthing is ready to start exchanging data with other devices
STATE_CHANGED: 'StateChanged', // Emitted when a folder changes state
FOLDER_ERRORS: 'FolderErrors', // Emitted when a folder has errors preventing a full sync
start: function() {
$http.get(urlbase + '/events?limit=1')

13
internal/auto/gui.files.go
View File

File diff suppressed because one or more lines are too long

3
internal/events/events.go
View File

@ -35,6 +35,7 @@ const (
DownloadProgress
FolderSummary
FolderCompletion
FolderErrors
AllEvents = (1 << iota) - 1
)
@ -75,6 +76,8 @@ func (t EventType) String() string {
return "FolderSummary"
case FolderCompletion:
return "FolderCompletion"
case FolderErrors:
return "FolderErrors"
default:
return "Unknown"
}

83
internal/model/rwfolder.go
View File

@ -13,6 +13,7 @@ import (
"math/rand"
"os"
"path/filepath"
"sort"
"time"
"github.com/syncthing/protocol"
@ -92,6 +93,9 @@ type rwFolder struct {
delayScan chan time.Duration
scanNow chan rescanRequest
remoteIndex chan struct{} // An index update was received, we should re-evaluate needs
errors map[string]string // path -> error string
errorsMut sync.Mutex
}
func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFolder {
@ -121,6 +125,8 @@ func newRWFolder(m *Model, shortID uint64, cfg config.FolderConfiguration) *rwFo
delayScan: make(chan time.Duration),
scanNow: make(chan rescanRequest),
remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
errorsMut: sync.NewMutex(),
}
}
@ -216,8 +222,11 @@ func (p *rwFolder) Serve() {
if debug {
l.Debugln(p, "pulling", prevVer, curVer)
}
p.setState(FolderSyncing)
p.clearErrors()
tries := 0
for {
tries++
@ -256,10 +265,18 @@ func (p *rwFolder) Serve() {
// we're not making it. Probably there are write
// errors preventing us. Flag this with a warning and
// wait a bit longer before retrying.
l.Warnf("Folder %q isn't making progress - check logs for possible root cause. Pausing puller for %v.", p.folder, pauseIntv)
l.Infof("Folder %q isn't making progress. Pausing puller for %v.", p.folder, pauseIntv)
if debug {
l.Debugln(p, "next pull in", pauseIntv)
}
if folderErrors := p.currentErrors(); len(folderErrors) > 0 {
events.Default.Log(events.FolderErrors, map[string]interface{}{
"folder": p.folder,
"errors": folderErrors,
})
}
p.pullTimer.Reset(pauseIntv)
break
}
@ -612,6 +629,7 @@ func (p *rwFolder) handleDir(file protocol.FileInfo) {
err = osutil.InWritableDir(osutil.Remove, realName)
if err != nil {
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
p.newError(file.Name, err)
return
}
fallthrough
@ -633,12 +651,14 @@ func (p *rwFolder) handleDir(file protocol.FileInfo) {
p.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
} else {
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
p.newError(file.Name, err)
}
return
// Weird error when stat()'ing the dir. Probably won't work to do
// anything else with it if we can't even stat() it.
case err != nil:
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
p.newError(file.Name, err)
return
}
@ -652,6 +672,7 @@ func (p *rwFolder) handleDir(file protocol.FileInfo) {
p.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
} else {
l.Infof("Puller (folder %q, dir %q): %v", p.folder, file.Name, err)
p.newError(file.Name, err)
}
}
@ -698,6 +719,7 @@ func (p *rwFolder) deleteDir(file protocol.FileInfo) {
p.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteDir}
} else {
l.Infof("Puller (folder %q, dir %q): delete: %v", p.folder, file.Name, err)
p.newError(file.Name, err)
}
}
@ -746,6 +768,7 @@ func (p *rwFolder) deleteFile(file protocol.FileInfo) {
p.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile}
} else {
l.Infof("Puller (folder %q, file %q): delete: %v", p.folder, file.Name, err)
p.newError(file.Name, err)
}
}
@ -808,6 +831,7 @@ func (p *rwFolder) renameFile(source, target protocol.FileInfo) {
err = p.shortcutFile(target)
if err != nil {
l.Infof("Puller (folder %q, file %q): rename from %q metadata: %v", p.folder, target.Name, source.Name, err)
p.newError(target.Name, err)
return
}
@ -820,6 +844,7 @@ func (p *rwFolder) renameFile(source, target protocol.FileInfo) {
err = osutil.InWritableDir(osutil.Remove, from)
if err != nil {
l.Infof("Puller (folder %q, file %q): delete %q after failed rename: %v", p.folder, target.Name, source.Name, err)
p.newError(target.Name, err)
return
}
@ -900,6 +925,7 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
if err != nil {
l.Infoln("Puller: shortcut:", err)
p.newError(file.Name, err)
} else {
p.dbUpdates <- dbUpdateJob{file, dbUpdateShortcutFile}
}
@ -988,6 +1014,7 @@ func (p *rwFolder) shortcutFile(file protocol.FileInfo) error {
if !p.ignorePermissions(file) {
if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil {
l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", p.folder, file.Name, err)
p.newError(file.Name, err)
return err
}
}
@ -998,6 +1025,7 @@ func (p *rwFolder) shortcutFile(file protocol.FileInfo) error {
info, err := os.Stat(realName)
if err != nil {
l.Infof("Puller (folder %q, file %q): shortcut: unable to stat file: %v", p.folder, file.Name, err)
p.newError(file.Name, err)
return err
}
@ -1018,6 +1046,7 @@ func (p *rwFolder) shortcutSymlink(file protocol.FileInfo) (err error) {
err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
if err != nil {
l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
p.newError(file.Name, err)
}
return
}
@ -1255,6 +1284,7 @@ func (p *rwFolder) finisherRoutine(in <-chan *sharedPullerState) {
if err != nil {
l.Infoln("Puller: final:", err)
p.newError(state.file.Name, err)
}
events.Default.Log(events.ItemFinished, map[string]interface{}{
"folder": p.folder,
@ -1402,3 +1432,54 @@ func moveForConflict(name string) error {
}
return err
}
func (p *rwFolder) newError(path string, err error) {
p.errorsMut.Lock()
defer p.errorsMut.Unlock()
// We might get more than one error report for a file (i.e. error on
// Write() followed by Close()); we keep the first error as that is
// probably closer to the root cause.
if _, ok := p.errors[path]; ok {
return
}
p.errors[path] = err.Error()
}
func (p *rwFolder) clearErrors() {
p.errorsMut.Lock()
p.errors = make(map[string]string)
p.errorsMut.Unlock()
}
func (p *rwFolder) currentErrors() []fileError {
p.errorsMut.Lock()
errors := make([]fileError, 0, len(p.errors))
for path, err := range p.errors {
errors = append(errors, fileError{path, err})
}
sort.Sort(fileErrorList(errors))
p.errorsMut.Unlock()
return errors
}
// A []fileError is sent as part of an event and will be JSON serialized.
type fileError struct {
Path string `json:"path"`
Err string `json:"error"`
}
type fileErrorList []fileError
func (l fileErrorList) Len() int {
return len(l)
}
func (l fileErrorList) Less(a, b int) bool {
return l[a].Path < l[b].Path
}
func (l fileErrorList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}

37
internal/model/rwfolder_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/sync"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
@ -73,9 +74,11 @@ func TestHandleFile(t *testing.T) {
m.updateLocals("default", []protocol.FileInfo{existingFile})
p := rwFolder{
folder: "default",
dir: "testdata",
model: m,
folder: "default",
dir: "testdata",
model: m,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
copyChan := make(chan copyBlocksState, 1)
@ -127,9 +130,11 @@ func TestHandleFileWithTemp(t *testing.T) {
m.updateLocals("default", []protocol.FileInfo{existingFile})
p := rwFolder{
folder: "default",
dir: "testdata",
model: m,
folder: "default",
dir: "testdata",
model: m,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
copyChan := make(chan copyBlocksState, 1)
@ -198,9 +203,11 @@ func TestCopierFinder(t *testing.T) {
}
p := rwFolder{
folder: "default",
dir: "testdata",
model: m,
folder: "default",
dir: "testdata",
model: m,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
copyChan := make(chan copyBlocksState)
@ -332,9 +339,11 @@ func TestLastResortPulling(t *testing.T) {
}
p := rwFolder{
folder: "default",
dir: "testdata",
model: m,
folder: "default",
dir: "testdata",
model: m,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
copyChan := make(chan copyBlocksState)
@ -390,6 +399,8 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
model: m,
queue: newJobQueue(),
progressEmitter: emitter,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
// queue.Done should be called by the finisher routine
@ -477,6 +488,8 @@ func TestDeregisterOnFailInPull(t *testing.T) {
model: m,
queue: newJobQueue(),
progressEmitter: emitter,
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
// queue.Done should be called by the finisher routine