initial binlog package

go/binlog/binlog.go

@@ -0,0 +1,145 @@
+/*
+   Copyright 2015 Shlomi Noach, courtesy Booking.com
+*/
+
+package binlog
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type BinlogType int
+
+const (
+	BinaryLog BinlogType = iota
+	RelayLog
+)
+
+// BinlogCoordinates described binary log coordinates in the form of log file & log position.
+type BinlogCoordinates struct {
+	LogFile string
+	LogPos  int64
+	Type    BinlogType
+}
+
+// ParseBinlogCoordinates will parse an InstanceKey from a string representation such as 127.0.0.1:3306
+func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) {
+	tokens := strings.SplitN(logFileLogPos, ":", 2)
+	if len(tokens) != 2 {
+		return nil, fmt.Errorf("ParseBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos)
+	}
+
+	if logPos, err := strconv.ParseInt(tokens[1], 10, 0); err != nil {
+		return nil, fmt.Errorf("ParseBinlogCoordinates: invalid pos: %s", tokens[1])
+	} else {
+		return &BinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil
+	}
+}
+
+// DisplayString returns a user-friendly string representation of these coordinates
+func (this *BinlogCoordinates) DisplayString() string {
+	return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos)
+}
+
+// String returns a user-friendly string representation of these coordinates
+func (this BinlogCoordinates) String() string {
+	return this.DisplayString()
+}
+
+// Equals tests equality of this corrdinate and another one.
+func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool {
+	if other == nil {
+		return false
+	}
+	return this.LogFile == other.LogFile && this.LogPos == other.LogPos && this.Type == other.Type
+}
+
+// IsEmpty returns true if the log file is empty, unnamed
+func (this *BinlogCoordinates) IsEmpty() bool {
+	return this.LogFile == ""
+}
+
+// SmallerThan returns true if this coordinate is strictly smaller than the other.
+func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool {
+	if this.LogFile < other.LogFile {
+		return true
+	}
+	if this.LogFile == other.LogFile && this.LogPos < other.LogPos {
+		return true
+	}
+	return false
+}
+
+// SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one.
+// We do NOT compare the type so we can not use this.Equals()
+func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) bool {
+	if this.SmallerThan(other) {
+		return true
+	}
+	return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison
+}
+
+// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's.
+func (this *BinlogCoordinates) FileSmallerThan(other *BinlogCoordinates) bool {
+	return this.LogFile < other.LogFile
+}
+
+// FileNumberDistance returns the numeric distance between this corrdinate's file number and the other's.
+// Effectively it means "how many roatets/FLUSHes would make these coordinates's file reach the other's"
+func (this *BinlogCoordinates) FileNumberDistance(other *BinlogCoordinates) int {
+	thisNumber, _ := this.FileNumber()
+	otherNumber, _ := other.FileNumber()
+	return otherNumber - thisNumber
+}
+
+// FileNumber returns the numeric value of the file, and the length in characters representing the number in the filename.
+// Example: FileNumber() of mysqld.log.000789 is (789, 6)
+func (this *BinlogCoordinates) FileNumber() (int, int) {
+	tokens := strings.Split(this.LogFile, ".")
+	numPart := tokens[len(tokens)-1]
+	numLen := len(numPart)
+	fileNum, err := strconv.Atoi(numPart)
+	if err != nil {
+		return 0, 0
+	}
+	return fileNum, numLen
+}
+
+// PreviousFileCoordinatesBy guesses the filename of the previous binlog/relaylog, by given offset (number of files back)
+func (this *BinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoordinates, error) {
+	result := BinlogCoordinates{LogPos: 0, Type: this.Type}
+
+	fileNum, numLen := this.FileNumber()
+	if fileNum == 0 {
+		return result, errors.New("Log file number is zero, cannot detect previous file")
+	}
+	newNumStr := fmt.Sprintf("%d", (fileNum - offset))
+	newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr
+
+	tokens := strings.Split(this.LogFile, ".")
+	tokens[len(tokens)-1] = newNumStr
+	result.LogFile = strings.Join(tokens, ".")
+	return result, nil
+}
+
+// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog
+func (this *BinlogCoordinates) PreviousFileCoordinates() (BinlogCoordinates, error) {
+	return this.PreviousFileCoordinatesBy(1)
+}
+
+// NextFileCoordinates guesses the filename of the next binlog/relaylog
+func (this *BinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) {
+	result := BinlogCoordinates{LogPos: 0, Type: this.Type}
+
+	fileNum, numLen := this.FileNumber()
+	newNumStr := fmt.Sprintf("%d", (fileNum + 1))
+	newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr
+
+	tokens := strings.Split(this.LogFile, ".")
+	tokens[len(tokens)-1] = newNumStr
+	result.LogFile = strings.Join(tokens, ".")
+	return result, nil
+}

go/binlog/binlog_reader.go

@@ -0,0 +1,12 @@
+/*
+  Copyright 2016 GitHub Inc.
+*/
+
+package binlog
+
+type BinlogEntry struct {
+}
+
+type BinlogReader interface {
+	ReadEntries(logFile string, startPos uint, endPos uint) (entries [](*BinlogEntry), err error)
+}

go/binlog/binlog_test.go

@@ -0,0 +1,132 @@
+/*
+   Copyright 2014 Outbrain Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package binlog
+
+import (
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+	"testing"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestBinlogCoordinates(t *testing.T) {
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	c2 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 5000}
+	c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104}
+
+	test.S(t).ExpectTrue(c1.Equals(&c2))
+	test.S(t).ExpectFalse(c1.Equals(&c3))
+	test.S(t).ExpectFalse(c1.Equals(&c4))
+	test.S(t).ExpectFalse(c1.SmallerThan(&c2))
+	test.S(t).ExpectTrue(c1.SmallerThan(&c3))
+	test.S(t).ExpectTrue(c1.SmallerThan(&c4))
+	test.S(t).ExpectTrue(c3.SmallerThan(&c4))
+	test.S(t).ExpectFalse(c3.SmallerThan(&c2))
+	test.S(t).ExpectFalse(c4.SmallerThan(&c2))
+	test.S(t).ExpectFalse(c4.SmallerThan(&c3))
+
+	test.S(t).ExpectTrue(c1.SmallerThanOrEquals(&c2))
+	test.S(t).ExpectTrue(c1.SmallerThanOrEquals(&c3))
+}
+
+func TestBinlogNext(t *testing.T) {
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	cres, err := c1.NextFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00018")
+
+	c2 := BinlogCoordinates{LogFile: "mysql-bin.00099", LogPos: 104}
+	cres, err = c2.NextFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00100")
+
+	c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00099", LogPos: 104}
+	cres, err = c3.NextFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00100")
+}
+
+func TestBinlogPrevious(t *testing.T) {
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	cres, err := c1.PreviousFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00016")
+
+	c2 := BinlogCoordinates{LogFile: "mysql-bin.00100", LogPos: 104}
+	cres, err = c2.PreviousFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00099")
+
+	c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00100", LogPos: 104}
+	cres, err = c3.PreviousFileCoordinates()
+
+	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(c1.Type, cres.Type)
+	test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00099")
+
+	c4 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00000", LogPos: 104}
+	_, err = c4.PreviousFileCoordinates()
+
+	test.S(t).ExpectNotNil(err)
+}
+
+func TestBinlogCoordinatesAsKey(t *testing.T) {
+	m := make(map[BinlogCoordinates]bool)
+
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104}
+	c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	c4 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 222}
+
+	m[c1] = true
+	m[c2] = true
+	m[c3] = true
+	m[c4] = true
+
+	test.S(t).ExpectEquals(len(m), 3)
+}
+
+func TestBinlogFileNumber(t *testing.T) {
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104}
+
+	test.S(t).ExpectEquals(c1.FileNumberDistance(&c1), 0)
+	test.S(t).ExpectEquals(c1.FileNumberDistance(&c2), 5)
+	test.S(t).ExpectEquals(c2.FileNumberDistance(&c1), -5)
+}
+
+func TestBinlogFileNumberDistance(t *testing.T) {
+	c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
+	fileNum, numLen := c1.FileNumber()
+
+	test.S(t).ExpectEquals(fileNum, 17)
+	test.S(t).ExpectEquals(numLen, 5)
+}

go/binlog/mysqlbinlog_reader.go

@@ -0,0 +1,12 @@
+/*
+  Copyright 2016 GitHub Inc.
+*/
+
+package binlog
+
+type MySQLBinlogReader struct {
+}
+
+func (this *MySQLBinlogReader) ReadEntries(logFile string, startPos uint, endPos uint) (entries [](*BinlogEntry), err error) {
+	return entries, err
+}