Add support for multiple announce servers (fixes #677)

Somebody owes me a beer.

cmd/syncthing/main.go

@@ -826,7 +826,7 @@ func renewUPnP(port int) {
 		if forwardedPort != 0 {
 			externalPort = forwardedPort
 			discoverer.StopGlobal()
-			discoverer.StartGlobal(opts.GlobalAnnServer, uint16(forwardedPort))
+			discoverer.StartGlobal(opts.GlobalAnnServers, uint16(forwardedPort))
 			if debugNet {
 				l.Debugf("Updated UPnP port mapping for external port %d on device %s.", forwardedPort, igd.FriendlyIdentifier())
 			}
@@ -1098,7 +1098,7 @@ func discovery(extPort int) *discover.Discoverer {
 
 	if opts.GlobalAnnEnabled {
 		l.Infoln("Starting global discovery announcements")
-		disc.StartGlobal(opts.GlobalAnnServer, uint16(extPort))
+		disc.StartGlobal(opts.GlobalAnnServers, uint16(extPort))
 	}
 
 	return disc

gui/index.html

@@ -203,11 +203,17 @@
                     <th><span class="glyphicon glyphicon-dashboard"></span>&emsp;<span translate>CPU Utilization</span></th>
                     <td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
                   </tr>
-                  <tr ng-if="system.extAnnounceOK != undefined">
+                  <tr ng-if="system.extAnnounceOK != undefined && announceServersTotal > 0">
-                    <th><span class="glyphicon glyphicon-bullhorn"></span>&emsp;<span translate>Global Discovery Server</span></th>
+                    <th><span class="glyphicon glyphicon-bullhorn"></span>&emsp;<span translate>Global Discovery Servers</span></th>
                     <td class="text-right">
-                      <span class="data text-success" ng-if="system.extAnnounceOK"><span translate>Online</span></span>
+                      <span ng-if="announceServersFailed.length == 0" class="data text-success">
-                      <span class="data text-danger" ng-if="!system.extAnnounceOK"><span translate>Offline</span></span>
+                        <span>OK</span>
+                      </span>
+                      <span ng-if="announceServersFailed.length != 0" class="data text-danger">
+                        <span popover data-trigger="hover" data-placement="bottom" data-content="{{announceServersFailed.join('\n')}}">
+                          {{announceServersTotal-announceServersFailed.length}}/{{announceServersTotal}}
+                        </span>
+                      </span>
                     </td>
                   </tr>
                   <tr>
@@ -609,8 +615,8 @@
                   <input id="DeviceName" class="form-control" type="text" ng-model="tmpOptions.DeviceName">
                 </div>
                 <div class="form-group">
-                  <label translate for="ListenStr">Sync Protocol Listen Addresses</label>
+                  <label translate for="ListenAddressStr">Sync Protocol Listen Addresses</label>
-                  <input id="ListenStr" class="form-control" type="text" ng-model="tmpOptions.ListenStr">
+                  <input id="ListenAddressStr" class="form-control" type="text" ng-model="tmpOptions.ListenAddressStr">
                 </div>
                 <div class="form-group">
                   <label translate for="MaxRecvKbps">Incoming Rate Limit (KiB/s)</label>
@@ -653,8 +659,8 @@
                   </div>
                 </div>
                 <div class="form-group">
-                  <label translate for="GlobalAnnServer">Global Discovery Server</label>
+                  <label translate for="GlobalAnnServersStr">Global Discovery Server</label>
-                  <input ng-disabled="!tmpOptions.GlobalAnnEnabled" id="GlobalAnnServer" class="form-control" type="text" ng-model="tmpOptions.GlobalAnnServer">
+                  <input ng-disabled="!tmpOptions.GlobalAnnEnabled" id="GlobalAnnServersStr" class="form-control" type="text" ng-model="tmpOptions.GlobalAnnServersStr">
                 </div>
               </div>
 
@@ -865,6 +871,7 @@
   <script src="scripts/syncthing/core/directives/modalDirective.js"></script>
   <script src="scripts/syncthing/core/directives/uniqueFolderDirective.js"></script>
   <script src="scripts/syncthing/core/directives/validDeviceidDirective.js"></script>
+  <script src="scripts/syncthing/core/directives/popoverDirective.js"></script>
   <script src="scripts/syncthing/core/filters/alwaysNumberFilter.js"></script>
   <script src="scripts/syncthing/core/filters/basenameFilter.js"></script>
   <script src="scripts/syncthing/core/filters/binaryFilter.js"></script>

gui/scripts/syncthing/core/controllers/syncthingController.js

@@ -245,7 +245,8 @@ angular.module('syncthing.core')
             var hasConfig = !isEmptyObject($scope.config);
 
             $scope.config = config;
-            $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
+            $scope.config.Options.ListenAddressStr = $scope.config.Options.ListenAddress.join(', ');
+            $scope.config.Options.GlobalAnnServersStr = $scope.config.Options.GlobalAnnServers.join(', ');
 
             $scope.devices = $scope.config.Devices;
             $scope.devices.forEach(function (deviceCfg) {
@@ -272,6 +273,14 @@ angular.module('syncthing.core')
             $http.get(urlbase + '/system').success(function (data) {
                 $scope.myID = data.myID;
                 $scope.system = data;
+                $scope.announceServersTotal = Object.keys(data.extAnnounceOK).length;
+                var failed = [];
+                for (var server in data.extAnnounceOK) {
+                    if (!data.extAnnounceOK[server]) {
+                        failed.push(server);
+                    }
+                }
+                $scope.announceServersFailed = failed;
                 console.log("refreshSystem", data);
             });
         }
@@ -599,9 +608,12 @@ angular.module('syncthing.core')
                 $scope.thisDevice().Name = $scope.tmpOptions.DeviceName;
                 $scope.config.Options = angular.copy($scope.tmpOptions);
                 $scope.config.GUI = angular.copy($scope.tmpGUI);
-                $scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) {
+
+                ['ListenAddress', 'GlobalAnnServers'].forEach(function (key) {
+                   $scope.config.Options[key] = $scope.config.Options[key + "Str"].split(/[ ,]+/).map(function (x) {
                         return x.trim();
                     });
+                });
 
                 $scope.saveConfig();
             }

gui/scripts/syncthing/core/directives/popoverDirective.js

@@ -0,0 +1,9 @@
+angular.module('syncthing.core')
+    .directive('popover', function () {
+        return {
+            restrict: 'A',
+            link: function (scope, element, attributes) {
+                $(element).popover();
+            }
+        };
+});

internal/config/config.go

@@ -160,7 +160,7 @@ type FolderDeviceConfiguration struct {
 
 type OptionsConfiguration struct {
 	ListenAddress           []string `xml:"listenAddress" default:"0.0.0.0:22000"`
-	GlobalAnnServer         string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
 	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" default:"true"`
 	LocalAnnPort            int      `xml:"localAnnouncePort" default:"21025"`
@@ -239,8 +239,6 @@ func (cfg *Configuration) WriteXML(w io.Writer) error {
 func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	fillNilSlices(&cfg.Options)
 
-	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
-
 	// Initialize an empty slice for folders if the config has none
 	if cfg.Folders == nil {
 		cfg.Folders = []FolderConfiguration{}
@@ -362,6 +360,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 			n.Addresses = []string{"dynamic"}
 		}
 	}
+
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
 }
 
 // ChangeRequiresRestart returns true if updating the configuration requires a
@@ -469,8 +470,8 @@ func convertV2V3(cfg *Configuration) {
 
 	// The global discovery format and port number changed in v0.9. Having the
 	// default announce server but old port number is guaranteed to be legacy.
-	if cfg.Options.GlobalAnnServer == "announce.syncthing.net:22025" {
+	if len(cfg.Options.GlobalAnnServers) == 1 && cfg.Options.GlobalAnnServers[0] == "announce.syncthing.net:22025" {
-		cfg.Options.GlobalAnnServer = "announce.syncthing.net:22026"
+		cfg.Options.GlobalAnnServers = []string{"announce.syncthing.net:22026"}
 	}
 
 	cfg.Version = 3

internal/config/config_test.go

@@ -36,7 +36,7 @@ func init() {
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
 		ListenAddress:           []string{"0.0.0.0:22000"},
-		GlobalAnnServer:         "announce.syncthing.net:22026",
+		GlobalAnnServers:        []string{"announce.syncthing.net:22026"},
 		GlobalAnnEnabled:        true,
 		LocalAnnEnabled:         true,
 		LocalAnnPort:            21025,
@@ -138,7 +138,7 @@ func TestNoListenAddress(t *testing.T) {
 func TestOverriddenValues(t *testing.T) {
 	expected := OptionsConfiguration{
 		ListenAddress:           []string{":23000"},
-		GlobalAnnServer:         "syncthing.nym.se:22026",
+		GlobalAnnServers:        []string{"syncthing.nym.se:22026"},
 		GlobalAnnEnabled:        false,
 		LocalAnnEnabled:         false,
 		LocalAnnPort:            42123,
@@ -163,7 +163,7 @@ func TestOverriddenValues(t *testing.T) {
 	}
 
 	if !reflect.DeepEqual(cfg.Options(), expected) {
-		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options())
 	}
 }
 

internal/discover/discover.go

@@ -42,13 +42,13 @@ type Discoverer struct {
 	multicastBeacon  beacon.Interface
 	registry         map[protocol.DeviceID][]CacheEntry
 	registryLock     sync.RWMutex
-	extServer        string
+	extServers       []string
 	extPort          uint16
 	localBcastTick   <-chan time.Time
 	stopGlobal       chan struct{}
 	globalWG         sync.WaitGroup
 	forcedBcastTick  chan time.Time
-	extAnnounceOK    bool
+	extAnnounceOK    map[string]bool
 	extAnnounceOKmut sync.Mutex
 }
 
@@ -70,6 +70,7 @@ func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
 		errorRetryIntv:  60 * time.Second,
 		cacheLifetime:   5 * time.Minute,
 		registry:        make(map[protocol.DeviceID][]CacheEntry),
+		extAnnounceOK:   make(map[string]bool),
 	}
 }
 
@@ -110,14 +111,26 @@ func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
 	}
 }
 
-func (d *Discoverer) StartGlobal(server string, extPort uint16) {
+func (d *Discoverer) StartGlobal(servers []string, extPort uint16) {
 	// Wait for any previous announcer to stop before starting a new one.
 	d.globalWG.Wait()
-	d.extServer = server
+	d.extServers = servers
 	d.extPort = extPort
 	d.stopGlobal = make(chan struct{})
 	d.globalWG.Add(1)
-	go d.sendExternalAnnouncements()
+	go func() {
+		defer d.globalWG.Done()
+
+		buf := d.announcementPkt()
+
+		for _, extServer := range d.extServers {
+			d.globalWG.Add(1)
+			go func(server string) {
+				d.sendExternalAnnouncements(server, buf)
+				d.globalWG.Done()
+			}(extServer)
+		}
+	}()
 }
 
 func (d *Discoverer) StopGlobal() {
@@ -127,7 +140,7 @@ func (d *Discoverer) StopGlobal() {
 	}
 }
 
-func (d *Discoverer) ExtAnnounceOK() bool {
+func (d *Discoverer) ExtAnnounceOK() map[string]bool {
 	d.extAnnounceOKmut.Lock()
 	defer d.extAnnounceOKmut.Unlock()
 	return d.extAnnounceOK
@@ -144,7 +157,7 @@ func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
 			addrs[i] = cached[i].Address
 		}
 		return addrs
-	} else if len(d.extServer) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
+	} else if len(d.extServers) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
 		// Only perform external lookups if we have at least one external
 		// server and one local announcement interval has passed. This is to
 		// avoid finding local peers on their remote address at startup.
@@ -188,6 +201,9 @@ func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
 
 func (d *Discoverer) announcementPkt() []byte {
 	var addrs []Address
+	if d.extPort != 0 {
+		addrs = []Address{{Port: d.extPort}}
+	} else {
 		for _, astr := range d.listenAddrs {
 			addr, err := net.ResolveTCPAddr("tcp", astr)
 			if err != nil {
@@ -204,6 +220,7 @@ func (d *Discoverer) announcementPkt() []byte {
 				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
 			}
 		}
+	}
 	var pkt = Announce{
 		Magic: AnnouncementMagic,
 		This:  Device{d.myID[:], addrs},
@@ -235,44 +252,43 @@ func (d *Discoverer) sendLocalAnnouncements() {
 	}
 }
 
-func (d *Discoverer) sendExternalAnnouncements() {
+func (d *Discoverer) sendExternalAnnouncements(extServer string, buf []byte) {
-	defer d.globalWG.Done()
+	timer := time.NewTimer(0)
-
-	remote, err := net.ResolveUDPAddr("udp", d.extServer)
-	for err != nil {
-		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
-		time.Sleep(d.errorRetryIntv)
-		remote, err = net.ResolveUDPAddr("udp", d.extServer)
-	}
 
 	conn, err := net.ListenUDP("udp", nil)
 	for err != nil {
+		timer.Reset(d.errorRetryIntv)
 		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
-		time.Sleep(d.errorRetryIntv)
+		select {
+		case <-d.stopGlobal:
+			return
+		case <-timer.C:
+		}
 		conn, err = net.ListenUDP("udp", nil)
 	}
 
-	var buf []byte
+	remote, err := net.ResolveUDPAddr("udp", extServer)
-	if d.extPort != 0 {
+	for err != nil {
-		var pkt = Announce{
+		timer.Reset(d.errorRetryIntv)
-			Magic: AnnouncementMagic,
+		l.Warnf("Global discovery: %s: %v; trying again in %v", extServer, err, d.errorRetryIntv)
-			This:  Device{d.myID[:], []Address{{Port: d.extPort}}},
+		select {
+		case <-d.stopGlobal:
+			return
+		case <-timer.C:
 		}
-		buf = pkt.MustMarshalXDR()
+		remote, err = net.ResolveUDPAddr("udp", extServer)
-	} else {
-		buf = d.announcementPkt()
 	}
 
 	// Delay the first announcement until after a full local announcement
 	// cycle, to increase the chance of other peers finding us locally first.
-	nextAnnouncement := time.NewTimer(d.localBcastIntv)
+	timer.Reset(d.localBcastIntv)
 
 	for {
 		select {
 		case <-d.stopGlobal:
 			return
 
-		case <-nextAnnouncement.C:
+		case <-timer.C:
 			var ok bool
 
 			if debug {
@@ -282,28 +298,29 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			_, err := conn.WriteTo(buf, remote)
 			if err != nil {
 				if debug {
-					l.Debugln("discover: warning:", err)
+					l.Debugln("discover: %s: warning:", extServer, err)
 				}
 				ok = false
 			} else {
 				// Verify that the announce server responds positively for our device ID
 
 				time.Sleep(1 * time.Second)
-				res := d.externalLookup(d.myID)
+				res := d.externalLookupOnServer(extServer, d.myID)
+
 				if debug {
-					l.Debugln("discover: external lookup check:", res)
+					l.Debugln("discover:", extServer, "external lookup check:", res)
 				}
 				ok = len(res) > 0
 			}
 
 			d.extAnnounceOKmut.Lock()
-			d.extAnnounceOK = ok
+			d.extAnnounceOK[extServer] = ok
 			d.extAnnounceOKmut.Unlock()
 
 			if ok {
-				nextAnnouncement.Reset(d.globalBcastIntv)
+				timer.Reset(d.globalBcastIntv)
 			} else {
-				nextAnnouncement.Reset(d.errorRetryIntv)
+				timer.Reset(d.errorRetryIntv)
 			}
 		}
 	}
@@ -390,10 +407,39 @@ func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
 }
 
 func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
-	extIP, err := net.ResolveUDPAddr("udp", d.extServer)
+	// Buffer up to as many answers as we have servers to query.
+	results := make(chan []string, len(d.extServers))
+
+	// Query all servers.
+	wg := sync.WaitGroup{}
+	for _, extServer := range d.extServers {
+		wg.Add(1)
+		go func(server string) {
+			result := d.externalLookupOnServer(server, device)
+			if debug {
+				l.Debugln("discover:", result, "from", server, "for", device)
+			}
+			results <- result
+			wg.Done()
+		}(extServer)
+	}
+
+	wg.Wait()
+	close(results)
+
+	addrs := []string{}
+	for result := range results {
+		addrs = append(addrs, result...)
+	}
+
+	return addrs
+}
+
+func (d *Discoverer) externalLookupOnServer(extServer string, device protocol.DeviceID) []string {
+	extIP, err := net.ResolveUDPAddr("udp", extServer)
 	if err != nil {
 		if debug {
-			l.Debugf("discover: %v; no external lookup", err)
+			l.Debugf("discover: %s: %v; no external lookup", extServer, err)
 		}
 		return nil
 	}
@@ -401,7 +447,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
 	conn, err := net.DialUDP("udp", nil, extIP)
 	if err != nil {
 		if debug {
-			l.Debugf("discover: %v; no external lookup", err)
+			l.Debugf("discover: %s: %v; no external lookup", extServer, err)
 		}
 		return nil
 	}
@@ -410,7 +456,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
 	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
 	if err != nil {
 		if debug {
-			l.Debugf("discover: %v; no external lookup", err)
+			l.Debugf("discover: %s: %v; no external lookup", extServer, err)
 		}
 		return nil
 	}
@@ -419,7 +465,7 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
 	_, err = conn.Write(buf)
 	if err != nil {
 		if debug {
-			l.Debugf("discover: %v; no external lookup", err)
+			l.Debugf("discover: %s: %v; no external lookup", extServer, err)
 		}
 		return nil
 	}
@@ -432,20 +478,20 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
 			return nil
 		}
 		if debug {
-			l.Debugf("discover: %v; no external lookup", err)
+			l.Debugf("discover: %s: %v; no external lookup", extServer, err)
 		}
 		return nil
 	}
 
 	if debug {
-		l.Debugf("discover: read external:\n%s", hex.Dump(buf[:n]))
+		l.Debugf("discover: %s: read external:\n%s", extServer, hex.Dump(buf[:n]))
 	}
 
 	var pkt Announce
 	err = pkt.UnmarshalXDR(buf[:n])
 	if err != nil && err != io.EOF {
 		if debug {
-			l.Debugln("discover:", err)
+			l.Debugln("discover:", extServer, err)
 		}
 		return nil
 	}

internal/model/model.go

@@ -1125,7 +1125,9 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 
 			if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
 				// File has been ignored or an unsupported symlink. Set invalid bit.
+				if debug {
 					l.Debugln("setting invalid bit on ignored", f)
+				}
 				nf := protocol.FileInfo{
 					Name:     f.Name,
 					Flags:    f.Flags | protocol.FlagInvalid,