syncthing/lib/db/blockmap.go
Jakob Borg 0d9a04c713 Reuse blockkey, speeds up large Update and Replace calls
benchmark                         old ns/op      new ns/op      delta
BenchmarkReplaceAll-8             2866418930     2880834572     +0.50%
BenchmarkUpdateOneChanged-8       226635         236596         +4.40%
BenchmarkUpdateOneUnchanged-8     229090         227326         -0.77%
BenchmarkNeedHalf-8               104483393      105151538      +0.64%
BenchmarkHave-8                   29288220       28827492       -1.57%
BenchmarkGlobal-8                 159269126      150768724      -5.34%
BenchmarkNeedHalfTruncated-8      108235000      104434216      -3.51%
BenchmarkHaveTruncated-8          28945489       27860093       -3.75%
BenchmarkGlobalTruncated-8        149355833      149972888      +0.41%

benchmark                         old allocs     new allocs     delta
BenchmarkReplaceAll-8             1054944        555451         -47.35%
BenchmarkUpdateOneChanged-8       1135           1135           +0.00%
BenchmarkUpdateOneUnchanged-8     1135           1135           +0.00%
BenchmarkNeedHalf-8               374777         374779         +0.00%
BenchmarkHave-8                   151995         151996         +0.00%
BenchmarkGlobal-8                 530063         530066         +0.00%
BenchmarkNeedHalfTruncated-8      374699         374702         +0.00%
BenchmarkHaveTruncated-8          151834         151834         +0.00%
BenchmarkGlobalTruncated-8        530021         530049         +0.01%

benchmark                         old bytes      new bytes      delta
BenchmarkReplaceAll-8             5074297112     5018351912     -1.10%
BenchmarkUpdateOneChanged-8       135097         135085         -0.01%
BenchmarkUpdateOneUnchanged-8     134976         134976         +0.00%
BenchmarkNeedHalf-8               44759436       44769400       +0.02%
BenchmarkHave-8                   11911138       11930612       +0.16%
BenchmarkGlobal-8                 81609867       81523668       -0.11%
BenchmarkNeedHalfTruncated-8      46588024       46692342       +0.22%
BenchmarkHaveTruncated-8          11348354       11348357       +0.00%
BenchmarkGlobalTruncated-8        79485168       81843956       +2.97%
2015-10-21 23:05:23 +02:00

209 lines
5.5 KiB
Go

// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// Package db provides a set type to track local/remote files with newness
// checks. We must do a certain amount of normalization in here. We will get
// fed paths with either native or wire-format separators and encodings
// depending on who calls us. We transform paths to wire-format (NFC and
// slashes) on the way to the database, and transform to native format
// (varying separator and encoding) on the way back out.
package db
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
var blockFinder *BlockFinder
type BlockMap struct {
db *leveldb.DB
folder string
}
func NewBlockMap(db *leveldb.DB, folder string) *BlockMap {
return &BlockMap{
db: db,
folder: folder,
}
}
// Add files to the block map, ignoring any deleted or invalid files.
func (m *BlockMap) Add(files []protocol.FileInfo) error {
batch := new(leveldb.Batch)
buf := make([]byte, 4)
var key []byte
for _, file := range files {
if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() {
continue
}
for i, block := range file.Blocks {
binary.BigEndian.PutUint32(buf, uint32(i))
key = m.blockKeyInto(key, block.Hash, file.Name)
batch.Put(key, buf)
}
}
return m.db.Write(batch, nil)
}
// Update block map state, removing any deleted or invalid files.
func (m *BlockMap) Update(files []protocol.FileInfo) error {
batch := new(leveldb.Batch)
buf := make([]byte, 4)
var key []byte
for _, file := range files {
if file.IsDirectory() {
continue
}
if file.IsDeleted() || file.IsInvalid() {
for _, block := range file.Blocks {
key = m.blockKeyInto(key, block.Hash, file.Name)
batch.Delete(key)
}
continue
}
for i, block := range file.Blocks {
binary.BigEndian.PutUint32(buf, uint32(i))
key = m.blockKeyInto(key, block.Hash, file.Name)
batch.Put(key, buf)
}
}
return m.db.Write(batch, nil)
}
// Discard block map state, removing the given files
func (m *BlockMap) Discard(files []protocol.FileInfo) error {
batch := new(leveldb.Batch)
var key []byte
for _, file := range files {
for _, block := range file.Blocks {
key = m.blockKeyInto(key, block.Hash, file.Name)
batch.Delete(key)
}
}
return m.db.Write(batch, nil)
}
// Drop block map, removing all entries related to this block map from the db.
func (m *BlockMap) Drop() error {
batch := new(leveldb.Batch)
iter := m.db.NewIterator(util.BytesPrefix(m.blockKeyInto(nil, nil, "")[:1+64]), nil)
defer iter.Release()
for iter.Next() {
batch.Delete(iter.Key())
}
if iter.Error() != nil {
return iter.Error()
}
return m.db.Write(batch, nil)
}
func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
return blockKeyInto(o, hash, m.folder, file)
}
type BlockFinder struct {
db *leveldb.DB
}
func NewBlockFinder(db *leveldb.DB) *BlockFinder {
if blockFinder != nil {
return blockFinder
}
f := &BlockFinder{
db: db,
}
return f
}
func (f *BlockFinder) String() string {
return fmt.Sprintf("BlockFinder@%p", f)
}
// Iterate takes an iterator function which iterates over all matching blocks
// for the given hash. The iterator function has to return either true (if
// they are happy with the block) or false to continue iterating for whatever
// reason. The iterator finally returns the result, whether or not a
// satisfying block was eventually found.
func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
var key []byte
for _, folder := range folders {
key = blockKeyInto(key, hash, folder, "")
iter := f.db.NewIterator(util.BytesPrefix(key), nil)
defer iter.Release()
for iter.Next() && iter.Error() == nil {
folder, file := fromBlockKey(iter.Key())
index := int32(binary.BigEndian.Uint32(iter.Value()))
if iterFn(folder, osutil.NativeFilename(file), index) {
return true
}
}
}
return false
}
// Fix repairs incorrect blockmap entries, removing the old entry and
// replacing it with a new entry for the given block
func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error {
buf := make([]byte, 4)
binary.BigEndian.PutUint32(buf, uint32(index))
batch := new(leveldb.Batch)
batch.Delete(blockKeyInto(nil, oldHash, folder, file))
batch.Put(blockKeyInto(nil, newHash, folder, file), buf)
return f.db.Write(batch, nil)
}
// m.blockKey returns a byte slice encoding the following information:
// keyTypeBlock (1 byte)
// folder (64 bytes)
// block hash (32 bytes)
// file name (variable size)
func blockKeyInto(o, hash []byte, folder, file string) []byte {
reqLen := 1 + 64 + 32 + len(file)
if cap(o) < reqLen {
o = make([]byte, reqLen)
} else {
o = o[:reqLen]
}
o[0] = KeyTypeBlock
copy(o[1:], []byte(folder))
copy(o[1+64:], []byte(hash))
copy(o[1+64+32:], []byte(file))
return o
}
func fromBlockKey(data []byte) (string, string) {
if len(data) < 1+64+32+1 {
panic("Incorrect key length")
}
if data[0] != KeyTypeBlock {
panic("Incorrect key type")
}
file := string(data[1+64+32:])
slice := data[1 : 1+64]
izero := bytes.IndexByte(slice, 0)
if izero > -1 {
return string(slice[:izero]), file
}
return string(slice), file
}