From 09046c8f6e49f09a0d4d5fa331b236cbb9ac7865 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Mar 2016 15:12:51 +0100 Subject: [PATCH] initial binlog package --- go/binlog/binlog.go | 145 ++++++++++++++++++++++++++++++++ go/binlog/binlog_reader.go | 12 +++ go/binlog/binlog_test.go | 132 +++++++++++++++++++++++++++++ go/binlog/mysqlbinlog_reader.go | 12 +++ 4 files changed, 301 insertions(+) create mode 100644 go/binlog/binlog.go create mode 100644 go/binlog/binlog_reader.go create mode 100644 go/binlog/binlog_test.go create mode 100644 go/binlog/mysqlbinlog_reader.go diff --git a/go/binlog/binlog.go b/go/binlog/binlog.go new file mode 100644 index 0000000..6925fa1 --- /dev/null +++ b/go/binlog/binlog.go @@ -0,0 +1,145 @@ +/* + Copyright 2015 Shlomi Noach, courtesy Booking.com +*/ + +package binlog + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +type BinlogType int + +const ( + BinaryLog BinlogType = iota + RelayLog +) + +// BinlogCoordinates described binary log coordinates in the form of log file & log position. +type BinlogCoordinates struct { + LogFile string + LogPos int64 + Type BinlogType +} + +// ParseBinlogCoordinates will parse an InstanceKey from a string representation such as 127.0.0.1:3306 +func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { + tokens := strings.SplitN(logFileLogPos, ":", 2) + if len(tokens) != 2 { + return nil, fmt.Errorf("ParseBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) + } + + if logPos, err := strconv.ParseInt(tokens[1], 10, 0); err != nil { + return nil, fmt.Errorf("ParseBinlogCoordinates: invalid pos: %s", tokens[1]) + } else { + return &BinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil + } +} + +// DisplayString returns a user-friendly string representation of these coordinates +func (this *BinlogCoordinates) DisplayString() string { + return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) +} + +// String returns a user-friendly string representation of these coordinates +func (this BinlogCoordinates) String() string { + return this.DisplayString() +} + +// Equals tests equality of this corrdinate and another one. +func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { + if other == nil { + return false + } + return this.LogFile == other.LogFile && this.LogPos == other.LogPos && this.Type == other.Type +} + +// IsEmpty returns true if the log file is empty, unnamed +func (this *BinlogCoordinates) IsEmpty() bool { + return this.LogFile == "" +} + +// SmallerThan returns true if this coordinate is strictly smaller than the other. +func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool { + if this.LogFile < other.LogFile { + return true + } + if this.LogFile == other.LogFile && this.LogPos < other.LogPos { + return true + } + return false +} + +// SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. +// We do NOT compare the type so we can not use this.Equals() +func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) bool { + if this.SmallerThan(other) { + return true + } + return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison +} + +// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's. +func (this *BinlogCoordinates) FileSmallerThan(other *BinlogCoordinates) bool { + return this.LogFile < other.LogFile +} + +// FileNumberDistance returns the numeric distance between this corrdinate's file number and the other's. +// Effectively it means "how many roatets/FLUSHes would make these coordinates's file reach the other's" +func (this *BinlogCoordinates) FileNumberDistance(other *BinlogCoordinates) int { + thisNumber, _ := this.FileNumber() + otherNumber, _ := other.FileNumber() + return otherNumber - thisNumber +} + +// FileNumber returns the numeric value of the file, and the length in characters representing the number in the filename. +// Example: FileNumber() of mysqld.log.000789 is (789, 6) +func (this *BinlogCoordinates) FileNumber() (int, int) { + tokens := strings.Split(this.LogFile, ".") + numPart := tokens[len(tokens)-1] + numLen := len(numPart) + fileNum, err := strconv.Atoi(numPart) + if err != nil { + return 0, 0 + } + return fileNum, numLen +} + +// PreviousFileCoordinatesBy guesses the filename of the previous binlog/relaylog, by given offset (number of files back) +func (this *BinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoordinates, error) { + result := BinlogCoordinates{LogPos: 0, Type: this.Type} + + fileNum, numLen := this.FileNumber() + if fileNum == 0 { + return result, errors.New("Log file number is zero, cannot detect previous file") + } + newNumStr := fmt.Sprintf("%d", (fileNum - offset)) + newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr + + tokens := strings.Split(this.LogFile, ".") + tokens[len(tokens)-1] = newNumStr + result.LogFile = strings.Join(tokens, ".") + return result, nil +} + +// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog +func (this *BinlogCoordinates) PreviousFileCoordinates() (BinlogCoordinates, error) { + return this.PreviousFileCoordinatesBy(1) +} + +// NextFileCoordinates guesses the filename of the next binlog/relaylog +func (this *BinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) { + result := BinlogCoordinates{LogPos: 0, Type: this.Type} + + fileNum, numLen := this.FileNumber() + newNumStr := fmt.Sprintf("%d", (fileNum + 1)) + newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr + + tokens := strings.Split(this.LogFile, ".") + tokens[len(tokens)-1] = newNumStr + result.LogFile = strings.Join(tokens, ".") + return result, nil +} diff --git a/go/binlog/binlog_reader.go b/go/binlog/binlog_reader.go new file mode 100644 index 0000000..ed5591b --- /dev/null +++ b/go/binlog/binlog_reader.go @@ -0,0 +1,12 @@ +/* + Copyright 2016 GitHub Inc. +*/ + +package binlog + +type BinlogEntry struct { +} + +type BinlogReader interface { + ReadEntries(logFile string, startPos uint, endPos uint) (entries [](*BinlogEntry), err error) +} diff --git a/go/binlog/binlog_test.go b/go/binlog/binlog_test.go new file mode 100644 index 0000000..1314d1c --- /dev/null +++ b/go/binlog/binlog_test.go @@ -0,0 +1,132 @@ +/* + Copyright 2014 Outbrain Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package binlog + +import ( + "github.com/outbrain/golib/log" + test "github.com/outbrain/golib/tests" + "testing" +) + +func init() { + log.SetLevel(log.ERROR) +} + +func TestBinlogCoordinates(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c2 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 5000} + c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} + + test.S(t).ExpectTrue(c1.Equals(&c2)) + test.S(t).ExpectFalse(c1.Equals(&c3)) + test.S(t).ExpectFalse(c1.Equals(&c4)) + test.S(t).ExpectFalse(c1.SmallerThan(&c2)) + test.S(t).ExpectTrue(c1.SmallerThan(&c3)) + test.S(t).ExpectTrue(c1.SmallerThan(&c4)) + test.S(t).ExpectTrue(c3.SmallerThan(&c4)) + test.S(t).ExpectFalse(c3.SmallerThan(&c2)) + test.S(t).ExpectFalse(c4.SmallerThan(&c2)) + test.S(t).ExpectFalse(c4.SmallerThan(&c3)) + + test.S(t).ExpectTrue(c1.SmallerThanOrEquals(&c2)) + test.S(t).ExpectTrue(c1.SmallerThanOrEquals(&c3)) +} + +func TestBinlogNext(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + cres, err := c1.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00018") + + c2 := BinlogCoordinates{LogFile: "mysql-bin.00099", LogPos: 104} + cres, err = c2.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00100") + + c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00099", LogPos: 104} + cres, err = c3.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00100") +} + +func TestBinlogPrevious(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + cres, err := c1.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00016") + + c2 := BinlogCoordinates{LogFile: "mysql-bin.00100", LogPos: 104} + cres, err = c2.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00099") + + c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00100", LogPos: 104} + cres, err = c3.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00099") + + c4 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00000", LogPos: 104} + _, err = c4.PreviousFileCoordinates() + + test.S(t).ExpectNotNil(err) +} + +func TestBinlogCoordinatesAsKey(t *testing.T) { + m := make(map[BinlogCoordinates]bool) + + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104} + c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c4 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 222} + + m[c1] = true + m[c2] = true + m[c3] = true + m[c4] = true + + test.S(t).ExpectEquals(len(m), 3) +} + +func TestBinlogFileNumber(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104} + + test.S(t).ExpectEquals(c1.FileNumberDistance(&c1), 0) + test.S(t).ExpectEquals(c1.FileNumberDistance(&c2), 5) + test.S(t).ExpectEquals(c2.FileNumberDistance(&c1), -5) +} + +func TestBinlogFileNumberDistance(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + fileNum, numLen := c1.FileNumber() + + test.S(t).ExpectEquals(fileNum, 17) + test.S(t).ExpectEquals(numLen, 5) +} diff --git a/go/binlog/mysqlbinlog_reader.go b/go/binlog/mysqlbinlog_reader.go new file mode 100644 index 0000000..6b23bc5 --- /dev/null +++ b/go/binlog/mysqlbinlog_reader.go @@ -0,0 +1,12 @@ +/* + Copyright 2016 GitHub Inc. +*/ + +package binlog + +type MySQLBinlogReader struct { +} + +func (this *MySQLBinlogReader) ReadEntries(logFile string, startPos uint, endPos uint) (entries [](*BinlogEntry), err error) { + return entries, err +}