// Copyright (C) 2014 The Syncthing Authors. // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. package api import ( "bytes" "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "io" "io/ioutil" "log" "net" "net/http" "net/url" "os" "path/filepath" "reflect" "regexp" "runtime" "runtime/pprof" "sort" "strconv" "strings" "time" "github.com/julienschmidt/httprouter" metrics "github.com/rcrowley/go-metrics" "github.com/thejerf/suture/v4" "github.com/vitrun/qart/qr" "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/connections" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/discover" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/locations" "github.com/syncthing/syncthing/lib/logger" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/ur" ) // matches a bcrypt hash and not too much else var bcryptExpr = regexp.MustCompile(`^\$2[aby]\$\d+\$.{50,}`) const ( DefaultEventMask = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected DiskEventMask = events.LocalChangeDetected | events.RemoteChangeDetected EventSubBufferSize = 1000 defaultEventTimeout = time.Minute httpsCertLifetimeDays = 820 featureFlagUntrusted = "untrusted" ) type service struct { suture.Service id protocol.DeviceID cfg config.Wrapper statics *staticsServer model model.Model eventSubs map[events.EventType]events.BufferedSubscription eventSubsMut sync.Mutex evLogger events.Logger discoverer discover.Manager connectionsService connections.Service fss model.FolderSummaryService urService *ur.Service noUpgrade bool tlsDefaultCommonName string configChanged chan struct{} // signals intentional listener close due to config change started chan string // signals startup complete by sending the listener address, for testing only startedOnce chan struct{} // the service has started successfully at least once startupErr error listenerAddr net.Addr exitChan chan *svcutil.FatalErr guiErrors logger.Recorder systemLog logger.Recorder } type Service interface { suture.Service config.Committer WaitForStart() error } func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool) Service { return &service{ id: id, cfg: cfg, statics: newStaticsServer(cfg.GUI().Theme, assetDir, cfg.Options().FeatureFlag(featureFlagUntrusted)), model: m, eventSubs: map[events.EventType]events.BufferedSubscription{ DefaultEventMask: defaultSub, DiskEventMask: diskSub, }, eventSubsMut: sync.NewMutex(), evLogger: evLogger, discoverer: discoverer, connectionsService: connectionsService, fss: fss, urService: urService, guiErrors: errors, systemLog: systemLog, noUpgrade: noUpgrade, tlsDefaultCommonName: tlsDefaultCommonName, configChanged: make(chan struct{}), startedOnce: make(chan struct{}), exitChan: make(chan *svcutil.FatalErr, 1), } } func (s *service) WaitForStart() error { <-s.startedOnce return s.startupErr } func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, error) { httpsCertFile := locations.Get(locations.HTTPSCertFile) httpsKeyFile := locations.Get(locations.HTTPSKeyFile) cert, err := tls.LoadX509KeyPair(httpsCertFile, httpsKeyFile) // If the certificate has expired or will expire in the next month, fail // it and generate a new one. if err == nil { err = shouldRegenerateCertificate(cert) } if err != nil { l.Infoln("Loading HTTPS certificate:", err) l.Infoln("Creating new HTTPS certificate") // When generating the HTTPS certificate, use the system host name per // default. If that isn't available, use the "syncthing" default. var name string name, err = os.Hostname() if err != nil { name = s.tlsDefaultCommonName } cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name, httpsCertLifetimeDays) } if err != nil { return nil, err } tlsCfg := tlsutil.SecureDefault() tlsCfg.Certificates = []tls.Certificate{cert} if guiCfg.Network() == "unix" { // When listening on a UNIX socket we should unlink before bind, // lest we get a "bind: address already in use". We don't // particularly care if this succeeds or not. os.Remove(guiCfg.Address()) } rawListener, err := net.Listen(guiCfg.Network(), guiCfg.Address()) if err != nil { return nil, err } if guiCfg.Network() == "unix" && guiCfg.UnixSocketPermissions() != 0 { // We should error if this fails under the assumption that these permissions are // required for operation. err = os.Chmod(guiCfg.Address(), guiCfg.UnixSocketPermissions()) if err != nil { return nil, err } } listener := &tlsutil.DowngradingListener{ Listener: rawListener, TLSConfig: tlsCfg, } return listener, nil } func sendJSON(w http.ResponseWriter, jsonObject interface{}) { w.Header().Set("Content-Type", "application/json; charset=utf-8") // Marshalling might fail, in which case we should return a 500 with the // actual error. bs, err := json.MarshalIndent(jsonObject, "", " ") if err != nil { // This Marshal() can't fail though. bs, _ = json.Marshal(map[string]string{"error": err.Error()}) http.Error(w, string(bs), http.StatusInternalServerError) return } fmt.Fprintf(w, "%s\n", bs) } func (s *service) Serve(ctx context.Context) error { listener, err := s.getListener(s.cfg.GUI()) if err != nil { select { case <-s.startedOnce: // We let this be a loud user-visible warning as it may be the only // indication they get that the GUI won't be available. l.Warnln("Starting API/GUI:", err) default: // This is during initialization. A failure here should be fatal // as there will be no way for the user to communicate with us // otherwise anyway. s.startupErr = err close(s.startedOnce) } return err } if listener == nil { // Not much we can do here other than exit quickly. The supervisor // will log an error at some point. return nil } s.listenerAddr = listener.Addr() defer listener.Close() s.cfg.Subscribe(s) defer s.cfg.Unsubscribe(s) restMux := httprouter.New() // The GET handlers restMux.HandlerFunc(http.MethodGet, "/rest/cluster/pending/devices", s.getPendingDevices) // - restMux.HandlerFunc(http.MethodGet, "/rest/cluster/pending/folders", s.getPendingFolders) // [device] restMux.HandlerFunc(http.MethodGet, "/rest/db/completion", s.getDBCompletion) // [device] [folder] restMux.HandlerFunc(http.MethodGet, "/rest/db/file", s.getDBFile) // folder file restMux.HandlerFunc(http.MethodGet, "/rest/db/ignores", s.getDBIgnores) // folder restMux.HandlerFunc(http.MethodGet, "/rest/db/need", s.getDBNeed) // folder [perpage] [page] restMux.HandlerFunc(http.MethodGet, "/rest/db/remoteneed", s.getDBRemoteNeed) // device folder [perpage] [page] restMux.HandlerFunc(http.MethodGet, "/rest/db/localchanged", s.getDBLocalChanged) // folder restMux.HandlerFunc(http.MethodGet, "/rest/db/status", s.getDBStatus) // folder restMux.HandlerFunc(http.MethodGet, "/rest/db/browse", s.getDBBrowse) // folder [prefix] [dirsonly] [levels] restMux.HandlerFunc(http.MethodGet, "/rest/folder/versions", s.getFolderVersions) // folder restMux.HandlerFunc(http.MethodGet, "/rest/folder/errors", s.getFolderErrors) // folder restMux.HandlerFunc(http.MethodGet, "/rest/folder/pullerrors", s.getFolderErrors) // folder (deprecated) restMux.HandlerFunc(http.MethodGet, "/rest/events", s.getIndexEvents) // [since] [limit] [timeout] [events] restMux.HandlerFunc(http.MethodGet, "/rest/events/disk", s.getDiskEvents) // [since] [limit] [timeout] restMux.HandlerFunc(http.MethodGet, "/rest/stats/device", s.getDeviceStats) // - restMux.HandlerFunc(http.MethodGet, "/rest/stats/folder", s.getFolderStats) // - restMux.HandlerFunc(http.MethodGet, "/rest/svc/deviceid", s.getDeviceID) // id restMux.HandlerFunc(http.MethodGet, "/rest/svc/lang", s.getLang) // - restMux.HandlerFunc(http.MethodGet, "/rest/svc/report", s.getReport) // - restMux.HandlerFunc(http.MethodGet, "/rest/svc/random/string", s.getRandomString) // [length] restMux.HandlerFunc(http.MethodGet, "/rest/system/browse", s.getSystemBrowse) // current restMux.HandlerFunc(http.MethodGet, "/rest/system/connections", s.getSystemConnections) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/discovery", s.getSystemDiscovery) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/error", s.getSystemError) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/ping", s.restPing) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/status", s.getSystemStatus) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/upgrade", s.getSystemUpgrade) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/version", s.getSystemVersion) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/debug", s.getSystemDebug) // - restMux.HandlerFunc(http.MethodGet, "/rest/system/log", s.getSystemLog) // [since] restMux.HandlerFunc(http.MethodGet, "/rest/system/log.txt", s.getSystemLogTxt) // [since] // The POST handlers restMux.HandlerFunc(http.MethodPost, "/rest/db/prio", s.postDBPrio) // folder file [perpage] [page] restMux.HandlerFunc(http.MethodPost, "/rest/db/ignores", s.postDBIgnores) // folder restMux.HandlerFunc(http.MethodPost, "/rest/db/override", s.postDBOverride) // folder restMux.HandlerFunc(http.MethodPost, "/rest/db/revert", s.postDBRevert) // folder restMux.HandlerFunc(http.MethodPost, "/rest/db/scan", s.postDBScan) // folder [sub...] [delay] restMux.HandlerFunc(http.MethodPost, "/rest/folder/versions", s.postFolderVersionsRestore) // folder restMux.HandlerFunc(http.MethodPost, "/rest/system/error", s.postSystemError) // restMux.HandlerFunc(http.MethodPost, "/rest/system/error/clear", s.postSystemErrorClear) // - restMux.HandlerFunc(http.MethodPost, "/rest/system/ping", s.restPing) // - restMux.HandlerFunc(http.MethodPost, "/rest/system/reset", s.postSystemReset) // [folder] restMux.HandlerFunc(http.MethodPost, "/rest/system/restart", s.postSystemRestart) // - restMux.HandlerFunc(http.MethodPost, "/rest/system/shutdown", s.postSystemShutdown) // - restMux.HandlerFunc(http.MethodPost, "/rest/system/upgrade", s.postSystemUpgrade) // - restMux.HandlerFunc(http.MethodPost, "/rest/system/pause", s.makeDevicePauseHandler(true)) // [device] restMux.HandlerFunc(http.MethodPost, "/rest/system/resume", s.makeDevicePauseHandler(false)) // [device] restMux.HandlerFunc(http.MethodPost, "/rest/system/debug", s.postSystemDebug) // [enable] [disable] // Config endpoints configBuilder := &configMuxBuilder{ Router: restMux, id: s.id, cfg: s.cfg, } configBuilder.registerConfig("/rest/config") configBuilder.registerConfigInsync("/rest/config/insync") // deprecated configBuilder.registerConfigInsync("/rest/config/restart-required") configBuilder.registerFolders("/rest/config/folders") configBuilder.registerDevices("/rest/config/devices") configBuilder.registerFolder("/rest/config/folders/:id") configBuilder.registerDevice("/rest/config/devices/:id") configBuilder.registerDefaultFolder("/rest/config/defaults/folder") configBuilder.registerDefaultDevice("/rest/config/defaults/device") configBuilder.registerOptions("/rest/config/options") configBuilder.registerLDAP("/rest/config/ldap") configBuilder.registerGUI("/rest/config/gui") // Deprecated config endpoints configBuilder.registerConfigDeprecated("/rest/system/config") // POST instead of PUT configBuilder.registerConfigInsync("/rest/system/config/insync") // Debug endpoints, not for general use debugMux := http.NewServeMux() debugMux.HandleFunc("/rest/debug/peerCompletion", s.getPeerCompletion) debugMux.HandleFunc("/rest/debug/httpmetrics", s.getSystemHTTPMetrics) debugMux.HandleFunc("/rest/debug/cpuprof", s.getCPUProf) // duration debugMux.HandleFunc("/rest/debug/heapprof", s.getHeapProf) debugMux.HandleFunc("/rest/debug/support", s.getSupportBundle) debugMux.HandleFunc("/rest/debug/file", s.getDebugFile) restMux.Handler(http.MethodGet, "/rest/debug/*method", s.whenDebugging(debugMux)) // A handler that disables caching noCacheRestMux := noCacheMiddleware(metricsMiddleware(restMux)) // The main routing handler mux := http.NewServeMux() mux.Handle("/rest/", noCacheRestMux) mux.HandleFunc("/qr/", s.getQR) // Serve compiled in assets unless an asset directory was set (for development) mux.Handle("/", s.statics) // Handle the special meta.js path mux.HandleFunc("/meta.js", s.getJSMetadata) guiCfg := s.cfg.GUI() // Wrap everything in CSRF protection. The /rest prefix should be // protected, other requests will grant cookies. var handler http.Handler = newCsrfManager(s.id.String()[:5], "/rest", guiCfg, mux, locations.Get(locations.CsrfTokens)) // Add our version and ID as a header to responses handler = withDetailsMiddleware(s.id, handler) // Wrap everything in basic auth, if user/password is set. if guiCfg.IsAuthEnabled() { handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler, s.evLogger) } // Redirect to HTTPS if we are supposed to if guiCfg.UseTLS() { handler = redirectToHTTPSMiddleware(handler) } // Add the CORS handling handler = corsMiddleware(handler, guiCfg.InsecureAllowFrameLoading) if addressIsLocalhost(guiCfg.Address()) && !guiCfg.InsecureSkipHostCheck { // Verify source host handler = localhostMiddleware(handler) } handler = debugMiddleware(handler) srv := http.Server{ Handler: handler, // ReadTimeout must be longer than SyncthingController $scope.refresh // interval to avoid HTTP keepalive/GUI refresh race. ReadTimeout: 15 * time.Second, // Prevent the HTTP server from logging stuff on its own. The things we // care about we log ourselves from the handlers. ErrorLog: log.New(ioutil.Discard, "", 0), } l.Infoln("GUI and API listening on", listener.Addr()) l.Infoln("Access the GUI via the following URL:", guiCfg.URL()) if s.started != nil { // only set when run by the tests select { case <-ctx.Done(): // Shouldn't return directly due to cleanup below case s.started <- listener.Addr().String(): } } // Indicate successful initial startup, to ourselves and to interested // listeners (i.e. the thing that starts the browser). select { case <-s.startedOnce: default: close(s.startedOnce) } // Serve in the background serveError := make(chan error, 1) go func() { select { case serveError <- srv.Serve(listener): case <-ctx.Done(): } }() // Wait for stop, restart or error signals err = nil select { case <-ctx.Done(): // Shutting down permanently l.Debugln("shutting down (stop)") case <-s.configChanged: // Soft restart due to configuration change l.Debugln("restarting (config changed)") case err = <-s.exitChan: case err = <-serveError: // Restart due to listen/serve failure l.Warnln("GUI/API:", err, "(restarting)") } // Give it a moment to shut down gracefully, e.g. if we are restarting // due to a config change through the API, let that finish successfully. timeout, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() if err := srv.Shutdown(timeout); err == timeout.Err() { srv.Close() } return err } // Complete implements suture.IsCompletable, which signifies to the supervisor // whether to stop restarting the service. func (s *service) Complete() bool { select { case <-s.startedOnce: return s.startupErr != nil default: } return false } func (s *service) String() string { return fmt.Sprintf("api.service@%p", s) } func (s *service) VerifyConfiguration(from, to config.Configuration) error { if to.GUI.Network() != "tcp" { return nil } _, err := net.ResolveTCPAddr("tcp", to.GUI.Address()) return err } func (s *service) CommitConfiguration(from, to config.Configuration) bool { // No action required when this changes, so mask the fact that it changed at all. from.GUI.Debugging = to.GUI.Debugging if untrusted := to.Options.FeatureFlag(featureFlagUntrusted); untrusted != from.Options.FeatureFlag(featureFlagUntrusted) { s.statics.setUntrusted(untrusted) } if to.GUI == from.GUI { // No GUI changes, we're done here. return true } if to.GUI.Theme != from.GUI.Theme { s.statics.setTheme(to.GUI.Theme) } // Tell the serve loop to restart s.configChanged <- struct{}{} return true } func (s *service) fatal(err *svcutil.FatalErr) { // s.exitChan is 1-buffered and whoever is first gets handled. select { case s.exitChan <- err: default: } } func debugMiddleware(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t0 := time.Now() h.ServeHTTP(w, r) if shouldDebugHTTP() { ms := 1000 * time.Since(t0).Seconds() // The variable `w` is most likely a *http.response, which we can't do // much with since it's a non exported type. We can however peek into // it with reflection to get at the status code and number of bytes // written. var status, written int64 if rw := reflect.Indirect(reflect.ValueOf(w)); rw.IsValid() && rw.Kind() == reflect.Struct { if rf := rw.FieldByName("status"); rf.IsValid() && rf.Kind() == reflect.Int { status = rf.Int() } if rf := rw.FieldByName("written"); rf.IsValid() && rf.Kind() == reflect.Int64 { written = rf.Int() } } l.Debugf("http: %s %q: status %d, %d bytes in %.02f ms", r.Method, r.URL.String(), status, written, ms) } }) } func corsMiddleware(next http.Handler, allowFrameLoading bool) http.Handler { // Handle CORS headers and CORS OPTIONS request. // CORS OPTIONS request are typically sent by browser during AJAX preflight // when the browser initiate a POST request. // // As the OPTIONS request is unauthorized, this handler must be the first // of the chain (hence added at the end). // // See https://www.w3.org/TR/cors/ for details. return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Process OPTIONS requests if r.Method == "OPTIONS" { // Add a generous access-control-allow-origin header for CORS requests w.Header().Add("Access-Control-Allow-Origin", "*") // Only GET/POST/OPTIONS Methods are supported w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS") // Only these headers can be set w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key") // The request is meant to be cached 10 minutes w.Header().Set("Access-Control-Max-Age", "600") // Indicate that no content will be returned w.WriteHeader(204) return } // Other security related headers that should be present. // https://www.owasp.org/index.php/Security_Headers if !allowFrameLoading { // We don't want to be rendered in an