// Copyright (C) 2014 The Syncthing Authors. // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. // Package config implements reading and writing of the syncthing configuration file. package config import ( "encoding/json" "encoding/xml" "fmt" "io" "io/ioutil" "net" "net/url" "os" "path" "path/filepath" "runtime" "sort" "strconv" "strings" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/util" ) const ( OldestHandledVersion = 10 CurrentVersion = 28 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) var ( // DefaultTCPPort defines default TCP port used if the URI does not specify one, for example tcp://0.0.0.0 DefaultTCPPort = 22000 // DefaultListenAddresses should be substituted when the configuration // contains default. This is done by the // "consumer" of the configuration as we don't want these saved to the // config. DefaultListenAddresses = []string{ util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))), "dynamic+https://relays.syncthing.net/endpoint", } // DefaultDiscoveryServersV4 should be substituted when the configuration // contains default-v4. DefaultDiscoveryServersV4 = []string{ "https://discovery.syncthing.net/v2/?noannounce&id=LYXKCHX-VI3NYZR-ALCJBHF-WMZYSPK-QG6QJA3-MPFYMSO-U56GTUK-NA2MIAW", "https://discovery-v4.syncthing.net/v2/?nolookup&id=LYXKCHX-VI3NYZR-ALCJBHF-WMZYSPK-QG6QJA3-MPFYMSO-U56GTUK-NA2MIAW", } // DefaultDiscoveryServersV6 should be substituted when the configuration // contains default-v6. DefaultDiscoveryServersV6 = []string{ "https://discovery.syncthing.net/v2/?noannounce&id=LYXKCHX-VI3NYZR-ALCJBHF-WMZYSPK-QG6QJA3-MPFYMSO-U56GTUK-NA2MIAW", "https://discovery-v6.syncthing.net/v2/?nolookup&id=LYXKCHX-VI3NYZR-ALCJBHF-WMZYSPK-QG6QJA3-MPFYMSO-U56GTUK-NA2MIAW", } // DefaultDiscoveryServers should be substituted when the configuration // contains default. DefaultDiscoveryServers = append(DefaultDiscoveryServersV4, DefaultDiscoveryServersV6...) // DefaultTheme is the default and fallback theme for the web UI. DefaultTheme = "default" ) func New(myID protocol.DeviceID) Configuration { var cfg Configuration cfg.Version = CurrentVersion cfg.OriginalVersion = CurrentVersion util.SetDefaults(&cfg) util.SetDefaults(&cfg.Options) util.SetDefaults(&cfg.GUI) // Can't happen. if err := cfg.prepare(myID); err != nil { panic("bug: error in preparing new folder: " + err.Error()) } return cfg } func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, error) { var cfg Configuration util.SetDefaults(&cfg) util.SetDefaults(&cfg.Options) util.SetDefaults(&cfg.GUI) if err := xml.NewDecoder(r).Decode(&cfg); err != nil { return Configuration{}, err } cfg.OriginalVersion = cfg.Version if err := cfg.prepare(myID); err != nil { return Configuration{}, err } return cfg, nil } func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) { var cfg Configuration util.SetDefaults(&cfg) util.SetDefaults(&cfg.Options) util.SetDefaults(&cfg.GUI) bs, err := ioutil.ReadAll(r) if err != nil { return Configuration{}, err } if err := json.Unmarshal(bs, &cfg); err != nil { return Configuration{}, err } cfg.OriginalVersion = cfg.Version if err := cfg.prepare(myID); err != nil { return Configuration{}, err } return cfg, nil } type Configuration struct { Version int `xml:"version,attr" json:"version"` Folders []FolderConfiguration `xml:"folder" json:"folders"` Devices []DeviceConfiguration `xml:"device" json:"devices"` GUI GUIConfiguration `xml:"gui" json:"gui"` LDAP LDAPConfiguration `xml:"ldap" json:"ldap"` Options OptionsConfiguration `xml:"options" json:"options"` IgnoredDevices []ObservedDevice `xml:"remoteIgnoredDevice" json:"remoteIgnoredDevices"` PendingDevices []ObservedDevice `xml:"pendingDevice" json:"pendingDevices"` XMLName xml.Name `xml:"configuration" json:"-"` MyID protocol.DeviceID `xml:"-" json:"-"` // Provided by the instantiator. OriginalVersion int `xml:"-" json:"-"` // The version we read from disk, before any conversion } func (cfg Configuration) Copy() Configuration { newCfg := cfg // Deep copy FolderConfigurations newCfg.Folders = make([]FolderConfiguration, len(cfg.Folders)) for i := range newCfg.Folders { newCfg.Folders[i] = cfg.Folders[i].Copy() } // Deep copy DeviceConfigurations newCfg.Devices = make([]DeviceConfiguration, len(cfg.Devices)) for i := range newCfg.Devices { newCfg.Devices[i] = cfg.Devices[i].Copy() } newCfg.Options = cfg.Options.Copy() newCfg.GUI = cfg.GUI.Copy() // DeviceIDs are values newCfg.IgnoredDevices = make([]ObservedDevice, len(cfg.IgnoredDevices)) copy(newCfg.IgnoredDevices, cfg.IgnoredDevices) newCfg.PendingDevices = make([]ObservedDevice, len(cfg.PendingDevices)) copy(newCfg.PendingDevices, cfg.PendingDevices) return newCfg } func (cfg *Configuration) WriteXML(w io.Writer) error { e := xml.NewEncoder(w) e.Indent("", " ") err := e.Encode(cfg) if err != nil { return err } _, err = w.Write([]byte("\n")) return err } func (cfg *Configuration) prepare(myID protocol.DeviceID) error { var myName string cfg.MyID = myID // Ensure this device is present in the config for _, device := range cfg.Devices { if device.DeviceID == myID { goto found } } myName, _ = os.Hostname() cfg.Devices = append(cfg.Devices, DeviceConfiguration{ DeviceID: myID, Name: myName, }) found: if err := cfg.clean(); err != nil { return err } // Ensure that we are part of the devices for i := range cfg.Folders { cfg.Folders[i].Devices = ensureDevicePresent(cfg.Folders[i].Devices, myID) } return nil } func (cfg *Configuration) clean() error { util.FillNilSlices(&cfg.Options) // Prepare folders and check for duplicates. Duplicates are bad and // dangerous, can't currently be resolved in the GUI, and shouldn't // happen when configured by the GUI. We return with an error in that // situation. existingFolders := make(map[string]*FolderConfiguration) for i := range cfg.Folders { folder := &cfg.Folders[i] folder.prepare() if folder.ID == "" { return fmt.Errorf("folder with empty ID in configuration") } if _, ok := existingFolders[folder.ID]; ok { return fmt.Errorf("duplicate folder ID %q in configuration", folder.ID) } existingFolders[folder.ID] = folder } cfg.Options.ListenAddresses = util.UniqueStrings(cfg.Options.ListenAddresses) cfg.Options.GlobalAnnServers = util.UniqueStrings(cfg.Options.GlobalAnnServers) if cfg.Version > 0 && cfg.Version < OldestHandledVersion { l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version) } // Upgrade configuration versions as appropriate if cfg.Version <= 10 { convertV10V11(cfg) } if cfg.Version == 11 { convertV11V12(cfg) } if cfg.Version == 12 { convertV12V13(cfg) } if cfg.Version == 13 { convertV13V14(cfg) } if cfg.Version == 14 { convertV14V15(cfg) } if cfg.Version == 15 { convertV15V16(cfg) } if cfg.Version == 16 { convertV16V17(cfg) } if cfg.Version == 17 { convertV17V18(cfg) } if cfg.Version == 18 { convertV18V19(cfg) } if cfg.Version == 19 { convertV19V20(cfg) } if cfg.Version == 20 { convertV20V21(cfg) } if cfg.Version == 21 { convertV21V22(cfg) } if cfg.Version == 22 { convertV22V23(cfg) } if cfg.Version == 23 { convertV23V24(cfg) } if cfg.Version == 24 { convertV24V25(cfg) } if cfg.Version == 25 { convertV25V26(cfg) } if cfg.Version == 26 { convertV26V27(cfg) } if cfg.Version == 27 { convertV27V28(cfg) } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) for _, device := range cfg.Devices { existingDevices[device.DeviceID] = true } // Ensure that the device list is // - free from duplicates // - sorted by ID cfg.Devices = ensureNoDuplicateDevices(cfg.Devices) sort.Slice(cfg.Devices, func(a, b int) bool { return cfg.Devices[a].DeviceID.Compare(cfg.Devices[b].DeviceID) == -1 }) // Ensure that the folder list is sorted by ID sort.Slice(cfg.Folders, func(a, b int) bool { return cfg.Folders[a].ID < cfg.Folders[b].ID }) // Ensure that in all folder configs // - any loose devices are not present in the wrong places // - there are no duplicate devices // - the versioning configuration parameter map is not nil sharedFolders := make(map[protocol.DeviceID][]string, len(cfg.Devices)) for i := range cfg.Folders { cfg.Folders[i].Devices = ensureExistingDevices(cfg.Folders[i].Devices, existingDevices) cfg.Folders[i].Devices = ensureNoDuplicateFolderDevices(cfg.Folders[i].Devices) if cfg.Folders[i].Versioning.Params == nil { cfg.Folders[i].Versioning.Params = map[string]string{} } sort.Slice(cfg.Folders[i].Devices, func(a, b int) bool { return cfg.Folders[i].Devices[a].DeviceID.Compare(cfg.Folders[i].Devices[b].DeviceID) == -1 }) for _, dev := range cfg.Folders[i].Devices { sharedFolders[dev.DeviceID] = append(sharedFolders[dev.DeviceID], cfg.Folders[i].ID) } } for i := range cfg.Devices { cfg.Devices[i].prepare(sharedFolders[cfg.Devices[i].DeviceID]) } // Very short reconnection intervals are annoying if cfg.Options.ReconnectIntervalS < 5 { cfg.Options.ReconnectIntervalS = 5 } if cfg.GUI.APIKey == "" { cfg.GUI.APIKey = rand.String(32) } // The list of ignored devices should not contain any devices that have // been manually added to the config. var newIgnoredDevices []ObservedDevice ignoredDevices := make(map[protocol.DeviceID]bool) for _, dev := range cfg.IgnoredDevices { if !existingDevices[dev.ID] { ignoredDevices[dev.ID] = true newIgnoredDevices = append(newIgnoredDevices, dev) } } cfg.IgnoredDevices = newIgnoredDevices // The list of pending devices should not contain devices that were added manually, nor should it contain // ignored devices. // Sort by time, so that in case of duplicates latest "time" is used. sort.Slice(cfg.PendingDevices, func(i, j int) bool { return cfg.PendingDevices[i].Time.Before(cfg.PendingDevices[j].Time) }) var newPendingDevices []ObservedDevice nextPendingDevice: for _, pendingDevice := range cfg.PendingDevices { if !existingDevices[pendingDevice.ID] && !ignoredDevices[pendingDevice.ID] { // Deduplicate for _, existingPendingDevice := range newPendingDevices { if existingPendingDevice.ID == pendingDevice.ID { continue nextPendingDevice } } newPendingDevices = append(newPendingDevices, pendingDevice) } } cfg.PendingDevices = newPendingDevices // Deprecated protocols are removed from the list of listeners and // device addresses. So far just kcp*. for _, prefix := range []string{"kcp"} { cfg.Options.ListenAddresses = filterURLSchemePrefix(cfg.Options.ListenAddresses, prefix) for i := range cfg.Devices { dev := &cfg.Devices[i] dev.Addresses = filterURLSchemePrefix(dev.Addresses, prefix) } } // Initialize any empty slices if cfg.Folders == nil { cfg.Folders = []FolderConfiguration{} } if cfg.IgnoredDevices == nil { cfg.IgnoredDevices = []ObservedDevice{} } if cfg.PendingDevices == nil { cfg.PendingDevices = []ObservedDevice{} } if cfg.Options.AlwaysLocalNets == nil { cfg.Options.AlwaysLocalNets = []string{} } if cfg.Options.UnackedNotificationIDs == nil { cfg.Options.UnackedNotificationIDs = []string{} } return nil } // DeviceMap returns a map of device ID to device configuration for the given configuration. func (cfg *Configuration) DeviceMap() map[protocol.DeviceID]DeviceConfiguration { m := make(map[protocol.DeviceID]DeviceConfiguration, len(cfg.Devices)) for _, dev := range cfg.Devices { m[dev.DeviceID] = dev } return m } func convertV27V28(cfg *Configuration) { // Show a notification about enabling filesystem watching cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs, "fsWatcherNotification") cfg.Version = 28 } func convertV26V27(cfg *Configuration) { for i := range cfg.Folders { f := &cfg.Folders[i] if f.DeprecatedPullers != 0 { f.PullerMaxPendingKiB = 128 * f.DeprecatedPullers f.DeprecatedPullers = 0 } } cfg.Version = 27 } func convertV25V26(cfg *Configuration) { // triggers database update cfg.Version = 26 } func convertV24V25(cfg *Configuration) { for i := range cfg.Folders { cfg.Folders[i].FSWatcherDelayS = 10 } cfg.Version = 25 } func convertV23V24(cfg *Configuration) { cfg.Options.URSeen = 2 cfg.Version = 24 } func convertV22V23(cfg *Configuration) { permBits := fs.FileMode(0777) if runtime.GOOS == "windows" { // Windows has no umask so we must chose a safer set of bits to // begin with. permBits = 0700 } // Upgrade code remains hardcoded for .stfolder despite configurable // marker name in later versions. for i := range cfg.Folders { fs := cfg.Folders[i].Filesystem() // Invalid config posted, or tests. if fs == nil { continue } if stat, err := fs.Stat(DefaultMarkerName); err == nil && !stat.IsDir() { err = fs.Remove(DefaultMarkerName) if err == nil { err = fs.Mkdir(DefaultMarkerName, permBits) fs.Hide(DefaultMarkerName) // ignore error } if err != nil { l.Infoln("Failed to upgrade folder marker:", err) } } } cfg.Version = 23 } func convertV21V22(cfg *Configuration) { for i := range cfg.Folders { cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic // Migrate to templated external versioner commands if cfg.Folders[i].Versioning.Type == "external" { cfg.Folders[i].Versioning.Params["command"] += " %FOLDER_PATH% %FILE_PATH%" } } cfg.Version = 22 } func convertV20V21(cfg *Configuration) { for _, folder := range cfg.Folders { if folder.FilesystemType != fs.FilesystemTypeBasic { continue } switch folder.Versioning.Type { case "simple", "trashcan": // Clean out symlinks in the known place cleanSymlinks(folder.Filesystem(), ".stversions") case "staggered": versionDir := folder.Versioning.Params["versionsPath"] if versionDir == "" { // default place cleanSymlinks(folder.Filesystem(), ".stversions") } else if filepath.IsAbs(versionDir) { // absolute cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".") } else { // relative to folder cleanSymlinks(folder.Filesystem(), versionDir) } } } cfg.Version = 21 } func convertV19V20(cfg *Configuration) { cfg.Options.MinHomeDiskFree = Size{Value: cfg.Options.DeprecatedMinHomeDiskFreePct, Unit: "%"} cfg.Options.DeprecatedMinHomeDiskFreePct = 0 for i := range cfg.Folders { cfg.Folders[i].MinDiskFree = Size{Value: cfg.Folders[i].DeprecatedMinDiskFreePct, Unit: "%"} cfg.Folders[i].DeprecatedMinDiskFreePct = 0 } cfg.Version = 20 } func convertV18V19(cfg *Configuration) { // Triggers a database tweak cfg.Version = 19 } func convertV17V18(cfg *Configuration) { // Do channel selection for existing users. Those who have auto upgrades // and usage reporting on default to the candidate channel. Others get // stable. if cfg.Options.URAccepted > 0 && cfg.Options.AutoUpgradeIntervalH > 0 { cfg.Options.UpgradeToPreReleases = true } // Show a notification to explain what's going on, except if upgrades // are disabled by compilation or environment variable in which case // it's not relevant. if !upgrade.DisabledByCompilation && os.Getenv("STNOUPGRADE") == "" { cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs, "channelNotification") } cfg.Version = 18 } func convertV16V17(cfg *Configuration) { // Fsync = true removed cfg.Version = 17 } func convertV15V16(cfg *Configuration) { // Triggers a database tweak cfg.Version = 16 } func convertV14V15(cfg *Configuration) { // Undo v0.13.0 broken migration for i, addr := range cfg.Options.GlobalAnnServers { switch addr { case "default-v4v2/": cfg.Options.GlobalAnnServers[i] = "default-v4" case "default-v6v2/": cfg.Options.GlobalAnnServers[i] = "default-v6" } } cfg.Version = 15 } func convertV13V14(cfg *Configuration) { // Not using the ignore cache is the new default. Disable it on existing // configurations. cfg.Options.CacheIgnoredFiles = false // Migrate UPnP -> NAT options cfg.Options.NATEnabled = cfg.Options.DeprecatedUPnPEnabled cfg.Options.DeprecatedUPnPEnabled = false cfg.Options.NATLeaseM = cfg.Options.DeprecatedUPnPLeaseM cfg.Options.DeprecatedUPnPLeaseM = 0 cfg.Options.NATRenewalM = cfg.Options.DeprecatedUPnPRenewalM cfg.Options.DeprecatedUPnPRenewalM = 0 cfg.Options.NATTimeoutS = cfg.Options.DeprecatedUPnPTimeoutS cfg.Options.DeprecatedUPnPTimeoutS = 0 // Replace the default listen address "tcp://0.0.0.0:22000" with the // string "default", but only if we also have the default relay pool // among the relay servers as this is implied by the new "default" // entry. hasDefault := false for _, raddr := range cfg.Options.DeprecatedRelayServers { if raddr == "dynamic+https://relays.syncthing.net/endpoint" { for i, addr := range cfg.Options.ListenAddresses { if addr == "tcp://0.0.0.0:22000" { cfg.Options.ListenAddresses[i] = "default" hasDefault = true break } } break } } // Copy relay addresses into listen addresses. for _, addr := range cfg.Options.DeprecatedRelayServers { if hasDefault && addr == "dynamic+https://relays.syncthing.net/endpoint" { // Skip the default relay address if we already have the // "default" entry in the list. continue } if addr == "" { continue } cfg.Options.ListenAddresses = append(cfg.Options.ListenAddresses, addr) } cfg.Options.DeprecatedRelayServers = nil // For consistency sort.Strings(cfg.Options.ListenAddresses) var newAddrs []string for _, addr := range cfg.Options.GlobalAnnServers { uri, err := url.Parse(addr) if err != nil { // That's odd. Skip the broken address. continue } if uri.Scheme == "https" { uri.Path = path.Join(uri.Path, "v2") + "/" addr = uri.String() } newAddrs = append(newAddrs, addr) } cfg.Options.GlobalAnnServers = newAddrs for i, fcfg := range cfg.Folders { if fcfg.DeprecatedReadOnly { cfg.Folders[i].Type = FolderTypeSendOnly } else { cfg.Folders[i].Type = FolderTypeSendReceive } cfg.Folders[i].DeprecatedReadOnly = false } // v0.13-beta already had config version 13 but did not get the new URL if cfg.Options.ReleasesURL == "https://api.github.com/repos/syncthing/syncthing/releases?per_page=30" { cfg.Options.ReleasesURL = "https://upgrades.syncthing.net/meta.json" } cfg.Version = 14 } func convertV12V13(cfg *Configuration) { if cfg.Options.ReleasesURL == "https://api.github.com/repos/syncthing/syncthing/releases?per_page=30" { cfg.Options.ReleasesURL = "https://upgrades.syncthing.net/meta.json" } cfg.Version = 13 } func convertV11V12(cfg *Configuration) { // Change listen address schema for i, addr := range cfg.Options.ListenAddresses { if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") { cfg.Options.ListenAddresses[i] = util.Address("tcp", addr) } } for i, device := range cfg.Devices { for j, addr := range device.Addresses { if addr != "dynamic" && addr != "" { cfg.Devices[i].Addresses[j] = util.Address("tcp", addr) } } } // Use new discovery server var newDiscoServers []string var useDefault bool for _, addr := range cfg.Options.GlobalAnnServers { if addr == "udp4://announce.syncthing.net:22026" { useDefault = true } else if addr == "udp6://announce-v6.syncthing.net:22026" { useDefault = true } else { newDiscoServers = append(newDiscoServers, addr) } } if useDefault { newDiscoServers = append(newDiscoServers, "default") } cfg.Options.GlobalAnnServers = newDiscoServers // Use new multicast group if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" { cfg.Options.LocalAnnMCAddr = "[ff12::8384]:21027" } // Use new local discovery port if cfg.Options.LocalAnnPort == 21025 { cfg.Options.LocalAnnPort = 21027 } // Set MaxConflicts to unlimited for i := range cfg.Folders { cfg.Folders[i].MaxConflicts = -1 } cfg.Version = 12 } func convertV10V11(cfg *Configuration) { // Set minimum disk free of existing folders to 1% for i := range cfg.Folders { cfg.Folders[i].DeprecatedMinDiskFreePct = 1 } cfg.Version = 11 } func ensureDevicePresent(devices []FolderDeviceConfiguration, myID protocol.DeviceID) []FolderDeviceConfiguration { for _, device := range devices { if device.DeviceID.Equals(myID) { return devices } } devices = append(devices, FolderDeviceConfiguration{ DeviceID: myID, }) return devices } func ensureExistingDevices(devices []FolderDeviceConfiguration, existingDevices map[protocol.DeviceID]bool) []FolderDeviceConfiguration { count := len(devices) i := 0 loop: for i < count { if _, ok := existingDevices[devices[i].DeviceID]; !ok { devices[i] = devices[count-1] count-- continue loop } i++ } return devices[0:count] } func ensureNoDuplicateFolderDevices(devices []FolderDeviceConfiguration) []FolderDeviceConfiguration { count := len(devices) i := 0 seenDevices := make(map[protocol.DeviceID]bool) loop: for i < count { id := devices[i].DeviceID if _, ok := seenDevices[id]; ok { devices[i] = devices[count-1] count-- continue loop } seenDevices[id] = true i++ } return devices[0:count] } func ensureNoDuplicateDevices(devices []DeviceConfiguration) []DeviceConfiguration { count := len(devices) i := 0 seenDevices := make(map[protocol.DeviceID]bool) loop: for i < count { id := devices[i].DeviceID if _, ok := seenDevices[id]; ok { devices[i] = devices[count-1] count-- continue loop } seenDevices[id] = true i++ } return devices[0:count] } func cleanSymlinks(filesystem fs.Filesystem, dir string) { if runtime.GOOS == "windows" { // We don't do symlinks on Windows. Additionally, there may // be things that look like symlinks that are not, which we // should leave alone. Deduplicated files, for example. return } filesystem.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } if info.IsSymlink() { l.Infoln("Removing incorrectly versioned symlink", path) filesystem.Remove(path) return fs.SkipDir } return nil }) } // filterURLSchemePrefix returns the list of addresses after removing all // entries whose URL scheme matches the given prefix. func filterURLSchemePrefix(addrs []string, prefix string) []string { for i := 0; i < len(addrs); i++ { uri, err := url.Parse(addrs[i]) if err != nil { continue } if strings.HasPrefix(uri.Scheme, prefix) { // Remove this entry copy(addrs[i:], addrs[i+1:]) addrs = addrs[:len(addrs)-1] i-- } } return addrs }