syncthing/vendor/github.com/magefile/mage/sh/cmd.go
Jakob Borg 916ec63af6 cmd/stdiscosrv: New discovery server (fixes #4618)
This is a new revision of the discovery server. Relevant changes and
non-changes:

- Protocol towards clients is unchanged.

- Recommended large scale design is still to be deployed nehind nginx (I
  tested, and it's still a lot faster at terminating TLS).

- Database backend is leveldb again, only. It scales enough, is easy to
  setup, and we don't need any backend to take care of.

- Server supports replication. This is a simple TCP channel - protect it
  with a firewall when deploying over the internet. (We deploy this within
  the same datacenter, and with firewall.) Any incoming client announces
  are sent over the replication channel(s) to other peer discosrvs.
  Incoming replication changes are applied to the database as if they came
  from clients, but without the TLS/certificate overhead.

- Metrics are exposed using the prometheus library, when enabled.

- The database values and replication protocol is protobuf, because JSON
  was quite CPU intensive when I tried that and benchmarked it.

- The "Retry-After" value for failed lookups gets slowly increased from
  a default of 120 seconds, by 5 seconds for each failed lookup,
  independently by each discosrv. This lowers the query load over time for
  clients that are never seen. The Retry-After maxes out at 3600 after a
  couple of weeks of this increase. The number of failed lookups is
  stored in the database, now and then (avoiding making each lookup a
  database put).

All in all this means clients can be pointed towards a cluster using
just multiple A / AAAA records to gain both load sharing and redundancy
(if one is down, clients will talk to the remaining ones).

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4648
2018-01-14 08:52:31 +00:00

166 lines
5.0 KiB
Go

package sh
import (
"bytes"
"fmt"
"io"
"log"
"os"
"os/exec"
"strings"
"github.com/magefile/mage/mg"
)
// RunCmd returns a function that will call Run with the given command. This is
// useful for creating command aliases to make your scripts easier to read, like
// this:
//
// // in a helper file somewhere
// var g0 = sh.RunCmd("go") // go is a keyword :(
//
// // somewhere in your main code
// if err := g0("install", "github.com/gohugo/hugo"); err != nil {
// return err
// }
//
// Args passed to command get baked in as args to the command when you run it.
// Any args passed in when you run the returned function will be appended to the
// original args. For example, this is equivalent to the above:
//
// var goInstall = sh.RunCmd("go", "install") goInstall("github.com/gohugo/hugo")
//
// RunCmd uses Exec underneath, so see those docs for more details.
func RunCmd(cmd string, args ...string) func(args ...string) error {
return func(args2 ...string) error {
return Run(cmd, append(args, args2...)...)
}
}
// OutCmd is like RunCmd except the command returns the output of the
// command.
func OutCmd(cmd string, args ...string) func(args ...string) (string, error) {
return func(args2 ...string) (string, error) {
return Output(cmd, append(args, args2...)...)
}
}
// Run is like RunWith, but doesn't specify any environment variables.
func Run(cmd string, args ...string) error {
return RunWith(nil, cmd, args...)
}
// RunV is like Run, but always sends the command's stdout to os.Stdout.
func RunV(cmd string, args ...string) error {
_, err := Exec(nil, os.Stdout, os.Stderr, cmd, args...)
return err
}
// RunWith runs the given command, directing stderr to this program's stderr and
// printing stdout to stdout if mage was run with -v. It adds adds env to the
// environment variables for the command being run. Environment variables should
// be in the format name=value.
func RunWith(env map[string]string, cmd string, args ...string) error {
var output io.Writer
if mg.Verbose() {
output = os.Stdout
}
_, err := Exec(env, output, os.Stderr, cmd, args...)
return err
}
// Output runs the command and returns the text from stdout.
func Output(cmd string, args ...string) (string, error) {
buf := &bytes.Buffer{}
_, err := Exec(nil, buf, os.Stderr, cmd, args...)
return strings.TrimSuffix(buf.String(), "\n"), err
}
// OutputWith is like RunWith, ubt returns what is written to stdout.
func OutputWith(env map[string]string, cmd string, args ...string) (string, error) {
buf := &bytes.Buffer{}
_, err := Exec(env, buf, os.Stderr, cmd, args...)
return strings.TrimSuffix(buf.String(), "\n"), err
}
// Exec executes the command, piping its stderr to mage's stderr and
// piping its stdout to the given writer. If the command fails, it will return
// an error that, if returned from a target or mg.Deps call, will cause mage to
// exit with the same code as the command failed with. Env is a list of
// environment variables to set when running the command, these override the
// current environment variables set (which are also passed to the command). cmd
// and args may include references to environment variables in $FOO format, in
// which case these will be expanded before the command is run.
//
// Ran reports if the command ran (rather than was not found or not executable).
// Code reports the exit code the command returned if it ran. If err == nil, ran
// is always true and code is always 0.
func Exec(env map[string]string, stdout, stderr io.Writer, cmd string, args ...string) (ran bool, err error) {
expand := func(s string) string {
s2, ok := env[s]
if ok {
return s2
}
return os.Getenv(s)
}
cmd = os.Expand(cmd, expand)
for i := range args {
args[i] = os.Expand(args[i], expand)
}
ran, code, err := run(env, stdout, stderr, cmd, args...)
if err == nil {
return true, nil
}
if ran {
return ran, mg.Fatalf(code, `running "%s %s" failed with exit code %d`, cmd, strings.Join(args, " "), code)
}
return ran, fmt.Errorf(`failed to run "%s %s: %v"`, cmd, strings.Join(args, " "), err)
}
func run(env map[string]string, stdout, stderr io.Writer, cmd string, args ...string) (ran bool, code int, err error) {
c := exec.Command(cmd, args...)
c.Env = os.Environ()
for k, v := range env {
c.Env = append(c.Env, k+"="+v)
}
c.Stderr = stderr
c.Stdout = stdout
c.Stdin = os.Stdin
log.Println("exec:", cmd, strings.Join(args, " "))
err = c.Run()
return cmdRan(err), ExitStatus(err), err
}
func cmdRan(err error) bool {
if err == nil {
return true
}
ee, ok := err.(*exec.ExitError)
if ok {
return ee.Exited()
}
return false
}
type exitStatus interface {
ExitStatus() int
}
// ExitStatus returns the exit status of the error if it is an exec.ExitError
// or if it implements ExitStatus() int.
// 0 if it is nil or 1 if it is a different error.
func ExitStatus(err error) int {
if err == nil {
return 0
}
if e, ok := err.(exitStatus); ok {
return e.ExitStatus()
}
if e, ok := err.(*exec.ExitError); ok {
if ex, ok := e.Sys().(exitStatus); ok {
return ex.ExitStatus()
}
}
return 1
}