syncthing/cmd/infra/stcrashreceiver/diskstore.go
Jakob Borg d4770ddc77
chore(cmd): clean up commands (#9705)
Move infrastructure related commands to under `cmd/infra` and
development stuff to `cmd/dev`. The default build command builds the
regular user facing binaries: syncthing, stdiscosrv, and strelaysrv.
2024-09-21 09:04:22 +02:00

199 lines
4.5 KiB
Go

// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"compress/gzip"
"context"
"io"
"log"
"math"
"os"
"path/filepath"
"sort"
"time"
)
type diskStore struct {
dir string
inbox chan diskEntry
maxBytes int64
maxFiles int
currentFiles []currentFile
currentSize int64
}
type diskEntry struct {
path string
data []byte
}
type currentFile struct {
path string
size int64
mtime int64
}
func (d *diskStore) Serve(ctx context.Context) {
if err := os.MkdirAll(d.dir, 0o700); err != nil {
log.Println("Creating directory:", err)
return
}
if err := d.inventory(); err != nil {
log.Println("Failed to inventory disk store:", err)
}
d.clean()
cleanTimer := time.NewTicker(time.Minute)
inventoryTimer := time.NewTicker(24 * time.Hour)
buf := new(bytes.Buffer)
gw := gzip.NewWriter(buf)
for {
select {
case entry := <-d.inbox:
path := d.fullPath(entry.path)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
log.Println("Creating directory:", err)
continue
}
buf.Reset()
gw.Reset(buf)
if _, err := gw.Write(entry.data); err != nil {
log.Println("Failed to compress crash report:", err)
continue
}
if err := gw.Close(); err != nil {
log.Println("Failed to compress crash report:", err)
continue
}
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
log.Printf("Failed to write %s: %v", entry.path, err)
_ = os.Remove(path)
continue
}
d.currentSize += int64(buf.Len())
d.currentFiles = append(d.currentFiles, currentFile{
size: int64(len(entry.data)),
path: path,
})
case <-cleanTimer.C:
d.clean()
case <-inventoryTimer.C:
if err := d.inventory(); err != nil {
log.Println("Failed to inventory disk store:", err)
}
case <-ctx.Done():
return
}
}
}
func (d *diskStore) Put(path string, data []byte) bool {
select {
case d.inbox <- diskEntry{
path: path,
data: data,
}:
return true
default:
return false
}
}
func (d *diskStore) Get(path string) ([]byte, error) {
path = d.fullPath(path)
bs, err := os.ReadFile(path)
if err != nil {
return nil, err
}
gr, err := gzip.NewReader(bytes.NewReader(bs))
if err != nil {
return nil, err
}
defer gr.Close()
return io.ReadAll(gr)
}
func (d *diskStore) Exists(path string) bool {
path = d.fullPath(path)
_, err := os.Lstat(path)
return err == nil
}
func (d *diskStore) clean() {
for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
f := d.currentFiles[0]
log.Println("Removing", f.path)
if err := os.Remove(f.path); err != nil {
log.Println("Failed to remove file:", err)
}
d.currentFiles = d.currentFiles[1:]
d.currentSize -= f.size
}
var oldest time.Duration
if len(d.currentFiles) > 0 {
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
}
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
}
func (d *diskStore) inventory() error {
d.currentFiles = nil
d.currentSize = 0
err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if filepath.Ext(path) != ".gz" {
return nil
}
d.currentSize += info.Size()
d.currentFiles = append(d.currentFiles, currentFile{
path: path,
size: info.Size(),
mtime: info.ModTime().Unix(),
})
return nil
})
sort.Slice(d.currentFiles, func(i, j int) bool {
return d.currentFiles[i].mtime < d.currentFiles[j].mtime
})
var oldest time.Duration
if len(d.currentFiles) > 0 {
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
}
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
return err
}
func (d *diskStore) fullPath(path string) string {
return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
}