mirror of
https://github.com/octoleo/syncthing.git
synced 2024-11-17 18:45:13 +00:00
c1ec9a8826
This reduces allocations, in number and in size, while getting extended attributes. This is mostly noticable when there is a large number of new files to scan and we're running with the default scanProgressInterval -- then a queue of files is built in-memory, and this queue includes extended attributes as part of file metadata. (Arguable it shouldn't, but that's a more difficult and involved change.) With 1M files to scan, each with one extended attribute, current peak memory usage looks like this: Showing nodes accounting for 1425.30MB, 98.19% of 1451.64MB total Dropped 1435 nodes (cum <= 7.26MB) Showing top 10 nodes out of 54 flat flat% sum% cum cum% 976.56MB 67.27% 67.27% 976.56MB 67.27% github.com/syncthing/syncthing/lib/fs.getXattr 305.44MB 21.04% 88.31% 305.44MB 21.04% github.com/syncthing/syncthing/lib/scanner.(*walker).walk.func1 45.78MB 3.15% 91.47% 1045.23MB 72.00% github.com/syncthing/syncthing/lib/fs.(*BasicFilesystem).GetXattr 22.89MB 1.58% 93.04% 22.89MB 1.58% github.com/syncthing/syncthing/lib/fs.listXattr 22.89MB 1.58% 94.62% 22.89MB 1.58% github.com/syncthing/syncthing/lib/protocol.(*PlatformData).SetXattrs 16MB 1.10% 95.72% 16.01MB 1.10% github.com/syndtr/goleveldb/leveldb/memdb.New After the change, it's this: Showing nodes accounting for 502.32MB, 95.70% of 524.88MB total Dropped 1400 nodes (cum <= 2.62MB) Showing top 10 nodes out of 91 flat flat% sum% cum cum% 305.43MB 58.19% 58.19% 305.43MB 58.19% github.com/syncthing/syncthing/lib/scanner.(*walker).walk.func1 45.79MB 8.72% 66.91% 68.68MB 13.09% github.com/syncthing/syncthing/lib/fs.(*BasicFilesystem).GetXattr 32MB 6.10% 73.01% 32.01MB 6.10% github.com/syndtr/goleveldb/leveldb/memdb.New 22.89MB 4.36% 77.37% 22.89MB 4.36% github.com/syncthing/syncthing/lib/fs.listXattr 22.89MB 4.36% 81.73% 22.89MB 4.36% github.com/syncthing/syncthing/lib/protocol.(*PlatformData).SetXattrs 15.35MB 2.92% 84.66% 15.36MB 2.93% github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).Get 15.28MB 2.91% 87.57% 15.28MB 2.91% strings.(*Builder).grow (The usage for xattrs is reduced from 976 MB to 68 MB)
169 lines
4.4 KiB
Go
169 lines
4.4 KiB
Go
// Copyright (C) 2022 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
//go:build !windows && !dragonfly && !illumos && !solaris && !openbsd
|
|
// +build !windows,!dragonfly,!illumos,!solaris,!openbsd
|
|
|
|
package fs
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"syscall"
|
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
func (f *BasicFilesystem) GetXattr(path string, xattrFilter XattrFilter) ([]protocol.Xattr, error) {
|
|
path, err := f.rooted(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get xattr %s: %w", path, err)
|
|
}
|
|
|
|
attrs, err := listXattr(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get xattr %s: %w", path, err)
|
|
}
|
|
|
|
res := make([]protocol.Xattr, 0, len(attrs))
|
|
var totSize int
|
|
for _, attr := range attrs {
|
|
if !xattrFilter.Permit(attr) {
|
|
l.Debugf("get xattr %s: skipping attribute %q denied by filter", path, attr)
|
|
continue
|
|
}
|
|
val, err := getXattr(path, attr)
|
|
var errNo syscall.Errno
|
|
if errors.As(err, &errNo) && errNo == 0x5d {
|
|
// ENOATTR, returned on BSD when asking for an attribute that
|
|
// doesn't exist (any more?)
|
|
continue
|
|
} else if err != nil {
|
|
return nil, fmt.Errorf("get xattr %s: %w", path, err)
|
|
}
|
|
if max := xattrFilter.GetMaxSingleEntrySize(); max > 0 && len(attr)+len(val) > max {
|
|
l.Debugf("get xattr %s: attribute %q exceeds max size", path, attr)
|
|
continue
|
|
}
|
|
totSize += len(attr) + len(val)
|
|
if max := xattrFilter.GetMaxTotalSize(); max > 0 && totSize > max {
|
|
l.Debugf("get xattr %s: attribute %q would cause max size to be exceeded", path, attr)
|
|
continue
|
|
}
|
|
res = append(res, protocol.Xattr{
|
|
Name: attr,
|
|
Value: val,
|
|
})
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
var xattrBufPool = sync.Pool{
|
|
New: func() any { return make([]byte, 1024) },
|
|
}
|
|
|
|
func getXattr(path, name string) ([]byte, error) {
|
|
buf := xattrBufPool.Get().([]byte)
|
|
defer func() {
|
|
// Put the buffer back in the pool, or not if we're not supposed to
|
|
// (we returned it to the caller).
|
|
if buf != nil {
|
|
xattrBufPool.Put(buf)
|
|
}
|
|
}()
|
|
|
|
size, err := unix.Lgetxattr(path, name, buf)
|
|
if errors.Is(err, unix.ERANGE) {
|
|
// Buffer was too small. Figure out how large it needs to be, and
|
|
// allocate.
|
|
size, err = unix.Lgetxattr(path, name, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Lgetxattr %s %q: %w", path, name, err)
|
|
}
|
|
if size > len(buf) {
|
|
xattrBufPool.Put(buf)
|
|
buf = make([]byte, size)
|
|
}
|
|
size, err = unix.Lgetxattr(path, name, buf)
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Lgetxattr %s %q: %w", path, name, err)
|
|
}
|
|
|
|
if size >= len(buf)/4*3 {
|
|
// The buffer is adequately sized (at least three quarters of it is
|
|
// used), return it as-is.
|
|
val := buf[:size]
|
|
buf = nil // Don't put it back in the pool.
|
|
return val, nil
|
|
}
|
|
|
|
// The buffer is larger than required, copy the data to a new buffer of
|
|
// the correct size. This avoids having lots of 1024-sized allocations
|
|
// sticking around when 24 bytes or whatever would be enough.
|
|
val := make([]byte, size)
|
|
copy(val, buf)
|
|
return val, nil
|
|
}
|
|
|
|
func (f *BasicFilesystem) SetXattr(path string, xattrs []protocol.Xattr, xattrFilter XattrFilter) error {
|
|
// Index the new attribute set.
|
|
xattrsIdx := make(map[string]int)
|
|
for i, xa := range xattrs {
|
|
xattrsIdx[xa.Name] = i
|
|
}
|
|
|
|
// Get and index the existing attribute set
|
|
current, err := f.GetXattr(path, xattrFilter)
|
|
if err != nil {
|
|
return fmt.Errorf("set xattrs %s: GetXattr: %w", path, err)
|
|
}
|
|
currentIdx := make(map[string]int)
|
|
for i, xa := range current {
|
|
currentIdx[xa.Name] = i
|
|
}
|
|
|
|
path, err = f.rooted(path)
|
|
if err != nil {
|
|
return fmt.Errorf("set xattrs %s: %w", path, err)
|
|
}
|
|
|
|
// Remove all existing xattrs that are not in the new set
|
|
for _, xa := range current {
|
|
if _, ok := xattrsIdx[xa.Name]; !ok {
|
|
if err := unix.Lremovexattr(path, xa.Name); err != nil {
|
|
return fmt.Errorf("set xattrs %s: Removexattr %q: %w", path, xa.Name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set all xattrs that are different in the new set
|
|
for _, xa := range xattrs {
|
|
if old, ok := currentIdx[xa.Name]; ok && bytes.Equal(xa.Value, current[old].Value) {
|
|
continue
|
|
}
|
|
if err := unix.Lsetxattr(path, xa.Name, xa.Value, 0); err != nil {
|
|
return fmt.Errorf("set xattrs %s: Setxattr %q: %w", path, xa.Name, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func compact(ss []string) []string {
|
|
i := 0
|
|
for _, s := range ss {
|
|
if s != "" {
|
|
ss[i] = s
|
|
i++
|
|
}
|
|
}
|
|
return ss[:i]
|
|
}
|