cmd/syncthing: Add debug commands to cli (#7503)

build.go

@@ -472,7 +472,7 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"install", "-v", "-trimpath"}
+	args := []string{"install", "-v"}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
@@ -504,7 +504,7 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"build", "-v", "-trimpath"}
+	args := []string{"build", "-v"}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
@@ -538,13 +538,13 @@ func appendParameters(args []string, tags []string, pkgs ...string) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags(tags))
+		args = append(args, "-trimpath", "-ldflags", ldflags(tags))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
 		// offset 0x0: too short` on 'dlv exec ...' see
 		// https://github.com/go-delve/delve/issues/79
-		args = append(args, "-gcflags", "-N -l")
+		args = append(args, "-gcflags", "all=-N -l")
 	}
 
 	return append(args, pkgs...)

cmd/stindex/main.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/db/backend"
-)
-
-func main() {
-	var mode string
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
-
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
-
-	flag.Parse()
-
-	path := flag.Arg(0)
-	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
-	}
-
-	ldb, err := backend.OpenLevelDBRO(path)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	switch mode {
-	case "dump":
-		dump(ldb)
-	case "dumpsize":
-		dumpsize(ldb)
-	case "idxck":
-		if !idxck(ldb) {
-			os.Exit(1)
-		}
-	case "account":
-		account(ldb)
-	default:
-		fmt.Println("Unknown mode")
-	}
-}

cmd/stindex/util.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"log"
-	"os"
-	"path/filepath"
-	"runtime"
-
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func nulString(bs []byte) string {
-	for i := range bs {
-		if bs[i] == 0 {
-			return string(bs[:i])
-		}
-	}
-	return string(bs)
-}
-
-func defaultConfigDir() string {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing")
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing")
-
-	case "darwin":
-		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing")
-		}
-		dir, err := fs.ExpandTilde("~/.config/syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-	}
-}

cmd/syncthing/cli/debug.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var debugCommand = cli.Command{
+	Name:     "debug",
+	HideHelp: true,
+	Usage:    "Debug command group",
+	Subcommands: []cli.Command{
+		{
+			Name:      "file",
+			Usage:     "Show information about a file (or directory/symlink)",
+			ArgsUsage: "FOLDER-ID PATH",
+			Action:    expects(2, debugFile()),
+		},
+		indexCommand,
+		{
+			Name:      "profile",
+			Usage:     "Save a profile to help figuring out what Syncthing does.",
+			ArgsUsage: "cpu | heap",
+			Action:    expects(1, profile()),
+		},
+	},
+}
+
+func debugFile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		return indexDumpOutput(fmt.Sprintf("debug/file?folder=%v&file=%v", c.Args()[0], normalizePath(c.Args()[1])))(c)
+	}
+}
+
+func profile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		switch t := c.Args()[0]; t {
+		case "cpu", "heap":
+			return saveToFile(fmt.Sprintf("debug/%vprof", c.Args()[0]))(c)
+		default:
+			return fmt.Errorf("expected cpu or heap as argument, got %v", t)
+		}
+	}
+}

cmd/syncthing/cli/errors.go

@@ -22,7 +22,7 @@ var errorsCommand = cli.Command{
 		{
 			Name:   "show",
 			Usage:  "Show pending errors",
-			Action: expects(0, dumpOutput("system/error")),
+			Action: expects(0, indexDumpOutput("system/error")),
 		},
 		{
 			Name:      "push",

cmd/syncthing/cli/index.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"github.com/urfave/cli"
+)
+
+var indexCommand = cli.Command{
+	Name:  "index",
+	Usage: "Show information about the index (database)",
+	Subcommands: []cli.Command{
+		{
+			Name:   "dump",
+			Usage:  "Print the entire db",
+			Action: expects(0, indexDump),
+		},
+		{
+			Name:   "dump-size",
+			Usage:  "Print the db size of different categories of information",
+			Action: expects(0, indexDumpSize),
+		},
+		{
+			Name:   "check",
+			Usage:  "Check the database for inconsistencies",
+			Action: expects(0, indexCheck),
+		},
+		{
+			Name:   "account",
+			Usage:  "Print key and value size statistics per key type",
+			Action: expects(0, indexAccount),
+		},
+	},
+}

cmd/syncthing/cli/index_accounting.go

@@ -4,22 +4,26 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"text/tabwriter"
 
-	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/urfave/cli"
 )
 
-// account prints key and data size statistics per class
+// indexAccount prints key and data size statistics per class
-func account(ldb backend.Backend) {
+func indexAccount(*cli.Context) error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	var ksizes [256]int
@@ -55,4 +59,6 @@ func account(ldb backend.Backend) {
 	}
 	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
 	tw.Flush()
+
+	return nil
 }

cmd/syncthing/cli/index_dump.go

@@ -4,23 +4,27 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"encoding/binary"
 	"fmt"
-	"log"
 	"time"
 
+	"github.com/urfave/cli"
+
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb backend.Backend) {
+func indexDump(*cli.Context) error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	for it.Next() {
 		key := it.Key()
@@ -34,7 +38,7 @@ func dump(ldb backend.Backend) {
 			var f protocol.FileInfo
 			err := f.Unmarshal(it.Value())
 			if err != nil {
-				log.Fatal(err)
+				return err
 			}
 			fmt.Printf(" V:%v\n", f)
 
@@ -61,10 +65,10 @@ func dump(ldb backend.Backend) {
 			folder := binary.BigEndian.Uint32(key[1:])
 			name := nulString(key[1+4:])
 			val := it.Value()
-			var real, virt time.Time
+			var realTime, virtualTime time.Time
-			real.UnmarshalBinary(val[:len(val)/2])
+			realTime.UnmarshalBinary(val[:len(val)/2])
-			virt.UnmarshalBinary(val[len(val)/2:])
+			virtualTime.UnmarshalBinary(val[len(val)/2:])
-			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
 
 		case db.KeyTypeFolderIdx:
 			key := binary.BigEndian.Uint32(key[1:])
@@ -152,4 +156,5 @@ func dump(ldb backend.Backend) {
 			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
 		}
 	}
+	return nil
 }

cmd/syncthing/cli/index_dumpsize.go

@@ -4,16 +4,16 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"container/heap"
 	"encoding/binary"
 	"fmt"
-	"log"
+
+	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 type SizedElement struct {
@@ -39,13 +39,18 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb backend.Backend) {
+func indexDumpSize(*cli.Context) error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
 	h := &ElementHeap{}
 	heap.Init(h)
 
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	var ele SizedElement
 	for it.Next() {
@@ -96,4 +101,6 @@ func dumpsize(ldb backend.Backend) {
 		ele = heap.Pop(h).(SizedElement)
 		fmt.Println(ele.key, ele.size)
 	}
+
+	return nil
 }

cmd/syncthing/cli/index_idxck.go

@@ -4,17 +4,18 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
-	"log"
 	"sort"
 
+	"github.com/urfave/cli"
+
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -34,7 +35,12 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func idxck(ldb backend.Backend) (success bool) {
+func indexCheck(*cli.Context) (err error) {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
@@ -47,11 +53,20 @@ func idxck(ldb backend.Backend) (success bool) {
 	usedBlocklists := make(map[string]struct{})
 	usedVersions := make(map[string]struct{})
 	var localDeviceKey uint32
-	success = true
+	success := true
+	defer func() {
+		if err == nil {
+			if success {
+				fmt.Println("Index check completed succesfully.")
+			} else {
+				err = errors.New("Inconsistencies found in the index")
+			}
+		}
+	}()
 
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	for it.Next() {
 		key := it.Key()
@@ -329,7 +344,7 @@ func idxck(ldb backend.Backend) (success bool) {
 		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
 	}
 
-	return
+	return nil
 }
 
 func needsLocally(vl db.VersionList) bool {

cmd/syncthing/cli/main.go

@@ -20,18 +20,21 @@ import (
 	"github.com/flynn-archive/go-shlex"
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/urfave/cli"
 )
 
 type preCli struct {
 	GUIAddress string `name:"gui-address"`
 	GUIAPIKey  string `name:"gui-apikey"`
 	HomeDir    string `name:"home"`
-	ConfDir    string `name:"conf"`
+	ConfDir    string `name:"config"`
+	DataDir    string `name:"data"`
 }
 
 func Run() error {
@@ -44,17 +47,7 @@ func Run() error {
 	parseFlags(&c)
 
 	// Not set as default above because the strings can be really long.
-	var err error
+	err := cmdutil.SetConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
-	homeSet := c.HomeDir != ""
-	confSet := c.ConfDir != ""
-	switch {
-	case homeSet && confSet:
-		err = errors.New("-home must not be used together with -conf")
-	case homeSet:
-		err = locations.SetBaseDir(locations.ConfigBaseDir, c.HomeDir)
-	case confSet:
-		err = locations.SetBaseDir(locations.ConfigBaseDir, c.ConfDir)
-	}
 	if err != nil {
 		return errors.Wrap(err, "Command line options:")
 	}
@@ -98,20 +91,28 @@ func Run() error {
 
 	client := getClient(guiCfg)
 
-	cfg, err := getConfig(client)
+	cfg, cfgErr := getConfig(client)
 	original := cfg.Copy()
-	if err != nil {
-		return errors.Wrap(err, "getting config")
-	}
 
 	// Copy the config and set the default flags
 	recliCfg := recli.DefaultConfig
 	recliCfg.IDTag.Name = "xml"
 	recliCfg.SkipTag.Name = "json"
 
-	commands, err := recli.New(recliCfg).Construct(&cfg)
+	configCommand := cli.Command{
-	if err != nil {
+		Name:     "config",
-		return errors.Wrap(err, "config reflect")
+		HideHelp: true,
+		Usage:    "Configuration modification command group",
+	}
+	if cfgErr != nil {
+		configCommand.Action = func(*cli.Context) error {
+			return cfgErr
+		}
+	} else {
+		configCommand.Subcommands, err = recli.New(recliCfg).Construct(&cfg)
+		if err != nil {
+			return errors.Wrap(err, "config reflect")
+		}
 	}
 
 	// Implement the same flags at the upper CLI, but do nothing with them.
@@ -150,15 +151,11 @@ func Run() error {
 		Usage: "Syncthing command line interface",
 		Flags: fakeFlags,
 		Subcommands: []cli.Command{
-			{
+			configCommand,
-				Name:        "config",
-				HideHelp:    true,
-				Usage:       "Configuration modification command group",
-				Subcommands: commands,
-			},
 			showCommand,
 			operationCommand,
 			errorsCommand,
+			debugCommand,
 		},
 	}}
 
@@ -190,7 +187,7 @@ func Run() error {
 		}
 	}
 
-	if !reflect.DeepEqual(cfg, original) {
+	if cfgErr == nil && !reflect.DeepEqual(cfg, original) {
 		body, err := json.MarshalIndent(cfg, "", "  ")
 		if err != nil {
 			return err

cmd/syncthing/cli/show.go

@@ -18,27 +18,27 @@ var showCommand = cli.Command{
 		{
 			Name:   "version",
 			Usage:  "Show syncthing client version",
-			Action: expects(0, dumpOutput("system/version")),
+			Action: expects(0, indexDumpOutput("system/version")),
 		},
 		{
 			Name:   "config-status",
 			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
-			Action: expects(0, dumpOutput("config/restart-required")),
+			Action: expects(0, indexDumpOutput("config/restart-required")),
 		},
 		{
 			Name:   "system",
 			Usage:  "Show system status",
-			Action: expects(0, dumpOutput("system/status")),
+			Action: expects(0, indexDumpOutput("system/status")),
 		},
 		{
 			Name:   "connections",
 			Usage:  "Report about connections to other devices",
-			Action: expects(0, dumpOutput("system/connections")),
+			Action: expects(0, indexDumpOutput("system/connections")),
 		},
 		{
 			Name:   "usage",
 			Usage:  "Show usage report",
-			Action: expects(0, dumpOutput("svc/report")),
+			Action: expects(0, indexDumpOutput("svc/report")),
 		},
 	},
 }

cmd/syncthing/cli/utils.go

@@ -8,12 +8,17 @@ package cli
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"mime"
 	"net/http"
 	"os"
+	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/urfave/cli"
 )
 
@@ -33,7 +38,7 @@ func emptyPost(url string) cli.ActionFunc {
 	}
 }
 
-func dumpOutput(url string) cli.ActionFunc {
+func indexDumpOutput(url string) cli.ActionFunc {
 	return func(c *cli.Context) error {
 		client := c.App.Metadata["client"].(*APIClient)
 		response, err := client.Get(url)
@@ -44,6 +49,39 @@ func dumpOutput(url string) cli.ActionFunc {
 	}
 }
 
+func saveToFile(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		response, err := client.Get(url)
+		if err != nil {
+			return err
+		}
+		_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
+		if err != nil {
+			return err
+		}
+		filename := params["filename"]
+		if filename == "" {
+			return errors.New("Missing filename in response")
+		}
+		bs, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		f, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = f.Write(bs)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Wrote results to", filename)
+		return err
+	}
+}
+
 func getConfig(c *APIClient) (config.Configuration, error) {
 	cfg := config.Configuration{}
 	response, err := c.Get("system/config")
@@ -92,3 +130,20 @@ func prettyPrintResponse(c *cli.Context, response *http.Response) error {
 	// TODO: Check flag for pretty print format
 	return prettyPrintJSON(data)
 }
+
+func getDB() (backend.Backend, error) {
+	return backend.OpenLevelDBRO(locations.Get(locations.Database))
+}
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
+func normalizePath(path string) string {
+	return filepath.ToSlash(filepath.Clean(path))
+}

cmd/syncthing/cmdutil/util.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+import (
+	"errors"
+
+	"github.com/syncthing/syncthing/lib/locations"
+)
+
+func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
+	homeSet := homeDir != ""
+	confSet := confDir != ""
+	dataSet := dataDir != ""
+	switch {
+	case dataSet != confSet:
+		return errors.New("either both or none of -conf and -data must be given, use -home to set both at once")
+	case homeSet && dataSet:
+		return errors.New("-home must not be used together with -conf and -data")
+	case homeSet:
+		confDir = homeDir
+		dataDir = homeDir
+		fallthrough
+	case dataSet:
+		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
+			return err
+		}
+		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
+	}
+	return nil
+}

cmd/syncthing/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/thejerf/suture/v4"
 
 	"github.com/syncthing/syncthing/cmd/syncthing/cli"
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
@@ -290,24 +291,7 @@ func (options serveOptions) Run() error {
 	}
 
 	// Not set as default above because the strings can be really long.
-	var err error
+	err := cmdutil.SetConfigDataLocationsFromFlags(options.HomeDir, options.ConfDir, options.DataDir)
-	homeSet := options.HomeDir != ""
-	confSet := options.ConfDir != ""
-	dataSet := options.DataDir != ""
-	switch {
-	case dataSet != confSet:
-		err = errors.New("either both or none of -conf and -data must be given, use -home to set both at once")
-	case homeSet && dataSet:
-		err = errors.New("-home must not be used together with -conf and -data")
-	case homeSet:
-		if err = locations.SetBaseDir(locations.ConfigBaseDir, options.HomeDir); err == nil {
-			err = locations.SetBaseDir(locations.DataBaseDir, options.HomeDir)
-		}
-	case dataSet:
-		if err = locations.SetBaseDir(locations.ConfigBaseDir, options.ConfDir); err == nil {
-			err = locations.SetBaseDir(locations.DataBaseDir, options.DataDir)
-		}
-	}
 	if err != nil {
 		l.Warnln("Command line options:", err)
 		os.Exit(svcutil.ExitError.AsInt())