lib/api, lib/model: Improve folder completion API (fixes #6075) (#6808)

lib/api/api.go

@@ -245,7 +245,7 @@ func (s *service) serve(ctx context.Context) {
 
 	// The GET handlers
 	getRestMux := http.NewServeMux()
-	getRestMux.HandleFunc("/rest/db/completion", s.getDBCompletion)              // device folder
+	getRestMux.HandleFunc("/rest/db/completion", s.getDBCompletion)              // [device] [folder]
 	getRestMux.HandleFunc("/rest/db/file", s.getDBFile)                          // folder file
 	getRestMux.HandleFunc("/rest/db/ignores", s.getDBIgnores)                    // folder
 	getRestMux.HandleFunc("/rest/db/need", s.getDBNeed)                          // folder [perpage] [page]
@@ -673,14 +673,21 @@ func (s *service) getDBBrowse(w http.ResponseWriter, r *http.Request) {
 
 func (s *service) getDBCompletion(w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var folder = qs.Get("folder")
-	var deviceStr = qs.Get("device")
+	var folder = qs.Get("folder")    // empty means all folders
+	var deviceStr = qs.Get("device") // empty means local device ID
 
-	device, err := protocol.DeviceIDFromString(deviceStr)
+	// We will check completion status for either the local device, or a
+	// specific given device ID.
+
+	device := protocol.LocalDeviceID
+	if deviceStr != "" {
+		var err error
+		device, err = protocol.DeviceIDFromString(deviceStr)
 		if err != nil {
-		http.Error(w, err.Error(), 500)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
+	}
 
 	sendJSON(w, s.model.Completion(device, folder).Map())
 }

lib/model/model.go

@@ -716,26 +716,91 @@ func (m *model) FolderStatistics() (map[string]stats.FolderStatistics, error) {
 
 type FolderCompletion struct {
 	CompletionPct float64
-	NeedBytes     int64
 	GlobalBytes   int64
+	NeedBytes     int64
+	GlobalItems   int32
 	NeedItems     int32
 	NeedDeletes   int32
 }
 
+func newFolderCompletion(global, need db.Counts) FolderCompletion {
+	comp := FolderCompletion{
+		GlobalBytes: global.Bytes,
+		NeedBytes:   need.Bytes,
+		GlobalItems: global.Files + global.Directories + global.Symlinks,
+		NeedItems:   need.Files + need.Directories + need.Symlinks,
+		NeedDeletes: need.Deleted,
+	}
+	comp.setComplectionPct()
+	return comp
+}
+
+func (comp *FolderCompletion) add(other FolderCompletion) {
+	comp.GlobalBytes += other.GlobalBytes
+	comp.NeedBytes += other.NeedBytes
+	comp.GlobalItems += other.GlobalItems
+	comp.NeedItems += other.NeedItems
+	comp.NeedDeletes += other.NeedDeletes
+	comp.setComplectionPct()
+}
+
+func (comp *FolderCompletion) setComplectionPct() {
+	if comp.GlobalBytes == 0 {
+		comp.CompletionPct = 100
+	} else {
+		needRatio := float64(comp.NeedBytes) / float64(comp.GlobalBytes)
+		comp.CompletionPct = 100 * (1 - needRatio)
+	}
+
+	// If the completion is 100% but there are deletes we need to handle,
+	// drop it down a notch. Hack for consumers that look only at the
+	// percentage (our own GUI does the same calculation as here on its own
+	// and needs the same fixup).
+	if comp.NeedBytes == 0 && comp.NeedDeletes > 0 {
+		comp.CompletionPct = 95 // chosen by fair dice roll
+	}
+}
+
 // Map returns the members as a map, e.g. used in api to serialize as Json.
 func (comp FolderCompletion) Map() map[string]interface{} {
 	return map[string]interface{}{
 		"completion":  comp.CompletionPct,
-		"needBytes":   comp.NeedBytes,
-		"needItems":   comp.NeedItems,
 		"globalBytes": comp.GlobalBytes,
+		"needBytes":   comp.NeedBytes,
+		"globalItems": comp.GlobalItems,
+		"needItems":   comp.NeedItems,
 		"needDeletes": comp.NeedDeletes,
 	}
 }
 
-// Completion returns the completion status, in percent, for the given device
-// and folder.
+// Completion returns the completion status, in percent with some counters,
+// for the given device and folder. The device can be any known device ID
+// (including the local device) or explicitly protocol.LocalDeviceID. An
+// empty folder string means the aggregate of all folders shared with the
+// given device.
 func (m *model) Completion(device protocol.DeviceID, folder string) FolderCompletion {
+	// The user specifically asked for our own device ID. Internally that is
+	// known as protocol.LocalDeviceID so translate.
+	if device == m.id {
+		device = protocol.LocalDeviceID
+	}
+
+	if folder != "" {
+		// We want completion for a specific folder.
+		return m.folderCompletion(device, folder)
+	}
+
+	// We want completion for all (shared) folders as an aggregate.
+	var comp FolderCompletion
+	for _, fcfg := range m.cfg.FolderList() {
+		if device == protocol.LocalDeviceID || fcfg.SharedWith(device) {
+			comp.add(m.folderCompletion(device, fcfg.ID))
+		}
+	}
+	return comp
+}
+
+func (m *model) folderCompletion(device protocol.DeviceID, folder string) FolderCompletion {
 	m.fmut.RLock()
 	rf, ok := m.folderFiles[folder]
 	m.fmut.RUnlock()
@@ -746,8 +811,8 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 	snap := rf.Snapshot()
 	defer snap.Release()
 
-	tot := snap.GlobalSize().Bytes
-	if tot == 0 {
+	global := snap.GlobalSize()
+	if global.Bytes == 0 {
 		// Folder is empty, so we have all of it
 		return FolderCompletion{
 			CompletionPct: 100,
@@ -765,26 +830,10 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
 		need.Bytes = 0
 	}
 
-	needRatio := float64(need.Bytes) / float64(tot)
-	completionPct := 100 * (1 - needRatio)
+	comp := newFolderCompletion(global, need)
 
-	// If the completion is 100% but there are deletes we need to handle,
-	// drop it down a notch. Hack for consumers that look only at the
-	// percentage (our own GUI does the same calculation as here on its own
-	// and needs the same fixup).
-	if need.Bytes == 0 && need.Deleted > 0 {
-		completionPct = 95 // chosen by fair dice roll
-	}
-
-	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need.Bytes, tot, needRatio)
-
-	return FolderCompletion{
-		CompletionPct: completionPct,
-		NeedBytes:     need.Bytes,
-		NeedItems:     need.Files + need.Directories + need.Symlinks,
-		GlobalBytes:   tot,
-		NeedDeletes:   need.Deleted,
-	}
+	l.Debugf("%v Completion(%s, %q): %v", m, device, folder, comp.Map())
+	return comp
 }
 
 // DBSnapshot returns a snapshot of the database content relevant to the given folder.

lib/model/model_test.go

@@ -3888,6 +3888,22 @@ func TestConnectionTerminationOnFolderUnpause(t *testing.T) {
 	})
 }
 
+func TestAddFolderCompletion(t *testing.T) {
+	// Empty folders are always 100% complete.
+	comp := newFolderCompletion(db.Counts{}, db.Counts{})
+	comp.add(newFolderCompletion(db.Counts{}, db.Counts{}))
+	if comp.CompletionPct != 100 {
+		t.Error(comp.CompletionPct)
+	}
+
+	// Completion is of the whole
+	comp = newFolderCompletion(db.Counts{Bytes: 100}, db.Counts{})             // 100% complete
+	comp.add(newFolderCompletion(db.Counts{Bytes: 400}, db.Counts{Bytes: 50})) // 82.5% complete
+	if comp.CompletionPct != 90 {                                              // 100 * (1 - 50/500)
+		t.Error(comp.CompletionPct)
+	}
+}
+
 func testConfigChangeClosesConnections(t *testing.T, expectFirstClosed, expectSecondClosed bool, pre func(config.Wrapper), fn func(config.Wrapper)) {
 	t.Helper()
 	wcfg, _ := tmpDefaultWrapper()