commit
7d875b415c
@ -82,7 +82,7 @@ The `--dml-batch-size` flag controls the size of the batched write. Allowed valu
|
||||
|
||||
Why is this behavior configurable? Different workloads have different characteristics. Some workloads have very large writes, such that aggregating even `50` writes into a transaction makes for a significant transaction size. On other workloads write rate is high such that one just can't allow for a hundred more syncs to disk per second. The default value of `10` is a modest compromise that should probably work very well for most workloads. Your mileage may vary.
|
||||
|
||||
Noteworthy is that setting `--dml-batch-size` to higher value _does not_ mean `gh-ost` blocks or waits on writes. The batch size is an upper limit on transaction size, not a minimal one. If `gh-ost` doesn't have "enough" events in the pipe, it does not wait on the binary log, it just writes what it already has. This conveniently suggests that if write load is light enough for `gh-ost` to only see a few events in the binary log at a given time, then it is also light neough for `gh-ost` to apply a fraction of the batch size.
|
||||
Noteworthy is that setting `--dml-batch-size` to higher value _does not_ mean `gh-ost` blocks or waits on writes. The batch size is an upper limit on transaction size, not a minimal one. If `gh-ost` doesn't have "enough" events in the pipe, it does not wait on the binary log, it just writes what it already has. This conveniently suggests that if write load is light enough for `gh-ost` to only see a few events in the binary log at a given time, then it is also light enough for `gh-ost` to apply a fraction of the batch size.
|
||||
|
||||
### exact-rowcount
|
||||
|
||||
@ -151,7 +151,7 @@ See also: [`concurrent-migrations`](cheatsheet.md#concurrent-migrations) on the
|
||||
|
||||
### skip-foreign-key-checks
|
||||
|
||||
By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not referenece other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`.
|
||||
By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not reference other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`.
|
||||
|
||||
### skip-renamed-columns
|
||||
|
||||
|
@ -43,7 +43,7 @@ Both interfaces may serve at the same time. Both respond to simple text command,
|
||||
|
||||
### Querying for data
|
||||
|
||||
For commands that accept an argumetn as value, pass `?` (question mark) to _get_ current value rather than _set_ a new one.
|
||||
For commands that accept an argument as value, pass `?` (question mark) to _get_ current value rather than _set_ a new one.
|
||||
|
||||
### Examples
|
||||
|
||||
|
@ -24,15 +24,15 @@ Initial output lines may look like this:
|
||||
2016-05-19 17:57:11 INFO connection validated on 127.0.0.1:3306
|
||||
2016-05-19 17:57:11 INFO rotate to next log name: mysql-bin.002587
|
||||
2016-05-19 17:57:11 INFO connection validated on 127.0.0.1:3306
|
||||
2016-05-19 17:57:11 INFO Droppping table `mydb`.`_mytable_gst`
|
||||
2016-05-19 17:57:11 INFO Dropping table `mydb`.`_mytable_gst`
|
||||
2016-05-19 17:57:11 INFO Table dropped
|
||||
2016-05-19 17:57:11 INFO Droppping table `mydb`.`_mytable_old`
|
||||
2016-05-19 17:57:11 INFO Dropping table `mydb`.`_mytable_old`
|
||||
2016-05-19 17:57:11 INFO Table dropped
|
||||
2016-05-19 17:57:11 INFO Creating ghost table `mydb`.`_mytable_gst`
|
||||
2016-05-19 17:57:11 INFO Ghost table created
|
||||
2016-05-19 17:57:11 INFO Altering ghost table `mydb`.`_mytable_gst`
|
||||
2016-05-19 17:57:11 INFO Ghost table altered
|
||||
2016-05-19 17:57:11 INFO Droppping table `mydb`.`_mytable_osc`
|
||||
2016-05-19 17:57:11 INFO Dropping table `mydb`.`_mytable_osc`
|
||||
2016-05-19 17:57:11 INFO Table dropped
|
||||
2016-05-19 17:57:11 INFO Creating changelog table `mydb`.`_mytable_osc`
|
||||
2016-05-19 17:57:11 INFO Changelog table created
|
||||
|
@ -16,7 +16,7 @@ Use of triggers simplifies a lot of the flow in doing a live table migration, bu
|
||||
|
||||
Triggers are stored routines which are invoked on a per-row operation upon `INSERT`, `DELETE`, `UPDATE` on a table.
|
||||
They were introduced in MySQL `5.0`.
|
||||
A trigger may contain a set of queries, and these queries run in the same transaction space as the query that manipulates the table. This makes for an atomicy of both the original operation on the table and the trigger-invoked operations.
|
||||
A trigger may contain a set of queries, and these queries run in the same transaction space as the query that manipulates the table. This makes for an atomicity of both the original operation on the table and the trigger-invoked operations.
|
||||
|
||||
### Triggers, overhead
|
||||
|
||||
|
@ -57,12 +57,12 @@ func NewGoMySQLReader(connectionConfig *mysql.ConnectionConfig) (binlogReader *G
|
||||
// ConnectBinlogStreamer
|
||||
func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) {
|
||||
if coordinates.IsEmpty() {
|
||||
return log.Errorf("Emptry coordinates at ConnectBinlogStreamer()")
|
||||
return log.Errorf("Empty coordinates at ConnectBinlogStreamer()")
|
||||
}
|
||||
|
||||
this.currentCoordinates = coordinates
|
||||
log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
|
||||
// Start sync with sepcified binlog file and position
|
||||
// Start sync with specified binlog file and position
|
||||
this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)})
|
||||
|
||||
return err
|
||||
@ -114,7 +114,7 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven
|
||||
}
|
||||
}
|
||||
// The channel will do the throttling. Whoever is reding from the channel
|
||||
// decides whether action is taken sycnhronously (meaning we wait before
|
||||
// decides whether action is taken synchronously (meaning we wait before
|
||||
// next iteration) or asynchronously (we keep pushing more events)
|
||||
// In reality, reads will be synchronous
|
||||
entriesChannel <- binlogEntry
|
||||
|
@ -46,7 +46,7 @@ func main() {
|
||||
migrationContext := base.GetMigrationContext()
|
||||
|
||||
flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
|
||||
flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unabel to determine the master")
|
||||
flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master")
|
||||
flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)")
|
||||
flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user")
|
||||
flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password")
|
||||
|
@ -213,7 +213,7 @@ func (this *Applier) dropTable(tableName string) error {
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(tableName),
|
||||
)
|
||||
log.Infof("Droppping table %s.%s",
|
||||
log.Infof("Dropping table %s.%s",
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(tableName),
|
||||
)
|
||||
@ -380,7 +380,7 @@ func (this *Applier) ReadMigrationRangeValues() error {
|
||||
// CalculateNextIterationRangeEndValues reads the next-iteration-range-end unique key values,
|
||||
// which will be used for copying the next chunk of rows. Ir returns "false" if there is
|
||||
// no further chunk to work through, i.e. we're past the last chunk and are done with
|
||||
// itrating the range (and this done with copying row chunks)
|
||||
// iterating the range (and this done with copying row chunks)
|
||||
func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, err error) {
|
||||
this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationIterationRangeMaxValues
|
||||
if this.migrationContext.MigrationIterationRangeMinValues == nil {
|
||||
|
@ -202,7 +202,7 @@ func (this *Inspector) validateConnection() error {
|
||||
}
|
||||
|
||||
// validateGrants verifies the user by which we're executing has necessary grants
|
||||
// to do its thang.
|
||||
// to do its thing.
|
||||
func (this *Inspector) validateGrants() error {
|
||||
query := `show /* gh-ost */ grants for current_user()`
|
||||
foundAll := false
|
||||
@ -261,7 +261,7 @@ func (this *Inspector) validateGrants() error {
|
||||
|
||||
// restartReplication is required so that we are _certain_ the binlog format and
|
||||
// row image settings have actually been applied to the replication thread.
|
||||
// It is entriely possible, for example, that the replication is using 'STATEMENT'
|
||||
// It is entirely possible, for example, that the replication is using 'STATEMENT'
|
||||
// binlog format even as the variable says 'ROW'
|
||||
func (this *Inspector) restartReplication() error {
|
||||
log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
@ -377,7 +377,7 @@ func (this *Inspector) validateLogSlaveUpdates() error {
|
||||
}
|
||||
|
||||
if this.migrationContext.InspectorIsAlsoApplier() {
|
||||
log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -179,7 +179,7 @@ func (this *Migrator) canStopStreaming() bool {
|
||||
|
||||
// onChangelogStateEvent is called when a binlog event operation on the changelog table is intercepted.
|
||||
func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
|
||||
// Hey, I created the changlog table, I know the type of columns it has!
|
||||
// Hey, I created the changelog table, I know the type of columns it has!
|
||||
if hint := dmlEvent.NewColumnValues.StringColumn(2); hint != "state" {
|
||||
return nil
|
||||
}
|
||||
@ -387,7 +387,7 @@ func (this *Migrator) ExecOnFailureHook() (err error) {
|
||||
|
||||
func (this *Migrator) handleCutOverResult(cutOverError error) (err error) {
|
||||
if this.migrationContext.TestOnReplica {
|
||||
// We're merly testing, we don't want to keep this state. Rollback the renames as possible
|
||||
// We're merely testing, we don't want to keep this state. Rollback the renames as possible
|
||||
this.applier.RenameTablesRollback()
|
||||
}
|
||||
if cutOverError == nil {
|
||||
@ -742,7 +742,7 @@ func (this *Migrator) initiateStatus() error {
|
||||
// printMigrationStatusHint prints a detailed configuration dump, that is useful
|
||||
// to keep in mind; such as the name of migrated table, throttle params etc.
|
||||
// This gets printed at beginning and end of migration, every 10 minutes throughout
|
||||
// migration, and as reponse to the "status" interactive command.
|
||||
// migration, and as response to the "status" interactive command.
|
||||
func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) {
|
||||
w := io.MultiWriter(writers...)
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Migrating %s.%s; Ghost table is %s.%s",
|
||||
@ -820,7 +820,7 @@ func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
// printStatus prints the prgoress status, and optionally additionally detailed
|
||||
// printStatus prints the progress status, and optionally additionally detailed
|
||||
// dump of configuration.
|
||||
// `rule` indicates the type of output expected.
|
||||
// By default the status is written to standard output, but other writers can
|
||||
|
@ -147,7 +147,7 @@ sup # Print a short status message
|
||||
coordinates # Print the currently inspected coordinates
|
||||
chunk-size=<newsize> # Set a new chunk-size
|
||||
dml-batch-size=<newsize> # Set a new dml-batch-size
|
||||
nice-ratio=<ratio> # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is agrressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...)
|
||||
nice-ratio=<ratio> # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is aggressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...)
|
||||
critical-load=<load> # Set a new set of max-load thresholds
|
||||
max-lag-millis=<max-lag> # Set a new replication lag threshold
|
||||
replication-lag-query=<query> # Set a new query that determines replication lag (no quotes)
|
||||
@ -305,8 +305,8 @@ help # This message
|
||||
return NoPrintStatusRule, err
|
||||
}
|
||||
if arg != "" && arg != this.migrationContext.OriginalTableName {
|
||||
// User exlpicitly provided table name. This is a courtesy protection mechanism
|
||||
err := fmt.Errorf("User commanded 'unpostpone' on %s, but migrated table is %s; ingoring request.", arg, this.migrationContext.OriginalTableName)
|
||||
// User explicitly provided table name. This is a courtesy protection mechanism
|
||||
err := fmt.Errorf("User commanded 'unpostpone' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName)
|
||||
return NoPrintStatusRule, err
|
||||
}
|
||||
if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) > 0 {
|
||||
|
@ -39,7 +39,7 @@ var (
|
||||
|
||||
const frenoMagicHint = "freno"
|
||||
|
||||
// Throttler collects metrics related to throttling and makes informed decisison
|
||||
// Throttler collects metrics related to throttling and makes informed decision
|
||||
// whether throttling should take place.
|
||||
type Throttler struct {
|
||||
migrationContext *base.MigrationContext
|
||||
@ -139,7 +139,7 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo
|
||||
if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
|
||||
// when running on replica, the heartbeat injection is also done on the replica.
|
||||
// This means we will always get a good heartbeat value.
|
||||
// When runnign on replica, we should instead check the `SHOW SLAVE STATUS` output.
|
||||
// When running on replica, we should instead check the `SHOW SLAVE STATUS` output.
|
||||
if lag, err := mysql.GetReplicationLag(this.inspector.connectionConfig); err != nil {
|
||||
return log.Errore(err)
|
||||
} else {
|
||||
|
@ -57,7 +57,7 @@ func (this BinlogCoordinates) String() string {
|
||||
return this.DisplayString()
|
||||
}
|
||||
|
||||
// Equals tests equality of this corrdinate and another one.
|
||||
// Equals tests equality of this coordinate and another one.
|
||||
func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool {
|
||||
if other == nil {
|
||||
return false
|
||||
@ -95,8 +95,8 @@ func (this *BinlogCoordinates) FileSmallerThan(other *BinlogCoordinates) bool {
|
||||
return this.LogFile < other.LogFile
|
||||
}
|
||||
|
||||
// FileNumberDistance returns the numeric distance between this corrdinate's file number and the other's.
|
||||
// Effectively it means "how many roatets/FLUSHes would make these coordinates's file reach the other's"
|
||||
// FileNumberDistance returns the numeric distance between this coordinate's file number and the other's.
|
||||
// Effectively it means "how many rotates/FLUSHes would make these coordinates's file reach the other's"
|
||||
func (this *BinlogCoordinates) FileNumberDistance(other *BinlogCoordinates) int {
|
||||
thisNumber, _ := this.FileNumber()
|
||||
otherNumber, _ := other.FileNumber()
|
||||
|
@ -15,7 +15,7 @@ const (
|
||||
DefaultInstancePort = 3306
|
||||
)
|
||||
|
||||
// InstanceKey is an instance indicator, identifued by hostname and port
|
||||
// InstanceKey is an instance indicator, identified by hostname and port
|
||||
type InstanceKey struct {
|
||||
Hostname string
|
||||
Port int
|
||||
@ -83,7 +83,7 @@ func (this *InstanceKey) IsValid() bool {
|
||||
return len(this.Hostname) > 0 && this.Port > 0
|
||||
}
|
||||
|
||||
// DetachedKey returns an instance key whose hostname is detahced: invalid, but recoverable
|
||||
// DetachedKey returns an instance key whose hostname is detached: invalid, but recoverable
|
||||
func (this *InstanceKey) DetachedKey() *InstanceKey {
|
||||
if this.IsDetached() {
|
||||
return this
|
||||
@ -91,7 +91,7 @@ func (this *InstanceKey) DetachedKey() *InstanceKey {
|
||||
return &InstanceKey{Hostname: fmt.Sprintf("%s%s", detachHint, this.Hostname), Port: this.Port}
|
||||
}
|
||||
|
||||
// ReattachedKey returns an instance key whose hostname is detahced: invalid, but recoverable
|
||||
// ReattachedKey returns an instance key whose hostname is detached: invalid, but recoverable
|
||||
func (this *InstanceKey) ReattachedKey() *InstanceKey {
|
||||
if !this.IsDetached() {
|
||||
return this
|
||||
|
@ -26,7 +26,7 @@ const (
|
||||
|
||||
const maxMediumintUnsigned int32 = 16777215
|
||||
|
||||
type TimezoneConvertion struct {
|
||||
type TimezoneConversion struct {
|
||||
ToTimezone string
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ type Column struct {
|
||||
IsUnsigned bool
|
||||
Charset string
|
||||
Type ColumnType
|
||||
timezoneConversion *TimezoneConvertion
|
||||
timezoneConversion *TimezoneConversion
|
||||
}
|
||||
|
||||
func (this *Column) convertArg(arg interface{}) interface{} {
|
||||
@ -172,7 +172,7 @@ func (this *ColumnList) GetColumnType(columnName string) ColumnType {
|
||||
}
|
||||
|
||||
func (this *ColumnList) SetConvertDatetimeToTimestamp(columnName string, toTimezone string) {
|
||||
this.GetColumn(columnName).timezoneConversion = &TimezoneConvertion{ToTimezone: toTimezone}
|
||||
this.GetColumn(columnName).timezoneConversion = &TimezoneConversion{ToTimezone: toTimezone}
|
||||
}
|
||||
|
||||
func (this *ColumnList) HasTimezoneConversion(columnName string) bool {
|
||||
|
Loading…
Reference in New Issue
Block a user