Skip to content

Commit 580b714

Browse files
timvaillancourttimvaillancourt
committed
move to just a count of in-order migrations
Signed-off-by: Tim Vaillancourt <[email protected]>
1 parent 3da225d commit 580b714

File tree

9 files changed

+208
-249
lines changed

9 files changed

+208
-249
lines changed

‎go/vt/proto/vtctldata/vtctldata.pb.go‎

Lines changed: 64 additions & 64 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go/vt/proto/vtctldata/vtctldata_vtproto.pb.go‎

Lines changed: 10 additions & 26 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go/vt/sidecardb/schema/onlineddl/schema_migrations.sql‎

Lines changed: 63 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -16,69 +16,69 @@ limitations under the License.
1616

1717
CREATE TABLE IF NOT EXISTS schema_migrations
1818
(
19-
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
20-
`migration_uuid` varchar(64) NOT NULL,
21-
`keyspace` varchar(256) NOT NULL,
22-
`shard` varchar(255) NOT NULL,
23-
`mysql_schema` varchar(128) NOT NULL,
24-
`mysql_table` varchar(128) NOT NULL,
25-
`migration_statement` text NOT NULL,
26-
`strategy` varchar(128) NOT NULL,
27-
`options` varchar(8192) NOT NULL,
28-
`added_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
29-
`requested_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
30-
`ready_timestamp` timestamp NULL DEFAULT NULL,
31-
`started_timestamp` timestamp NULL DEFAULT NULL,
32-
`liveness_timestamp` timestamp NULL DEFAULT NULL,
33-
`completed_timestamp` timestamp(6) NULL DEFAULT NULL,
34-
`cleanup_timestamp` timestamp NULL DEFAULT NULL,
35-
`migration_status` varchar(128) NOT NULL,
36-
`log_path` varchar(1024) NOT NULL,
37-
`artifacts` text NOT NULL,
38-
`retries` int unsigned NOT NULL DEFAULT '0',
39-
`tablet` varchar(128) NOT NULL DEFAULT '',
40-
`tablet_failure` tinyint unsigned NOT NULL DEFAULT '0',
41-
`progress` float NOT NULL DEFAULT '0',
42-
`migration_context` varchar(1024) NOT NULL DEFAULT '',
43-
`ddl_action` varchar(16) NOT NULL DEFAULT '',
44-
`message` text NOT NULL,
45-
`message_timestamp` timestamp(6) NULL DEFAULT NULL,
46-
`eta_seconds` bigint NOT NULL DEFAULT '-1',
47-
`rows_copied` bigint unsigned NOT NULL DEFAULT '0',
48-
`vreplication_lag_seconds` bigint unsigned NOT NULL DEFAULT '0',
49-
`table_rows` bigint NOT NULL DEFAULT '0',
50-
`added_unique_keys` int unsigned NOT NULL DEFAULT '0',
51-
`removed_unique_keys` int unsigned NOT NULL DEFAULT '0',
52-
`log_file` varchar(1024) NOT NULL DEFAULT '',
53-
`retain_artifacts_seconds` bigint NOT NULL DEFAULT '0',
54-
`postpone_completion` tinyint unsigned NOT NULL DEFAULT '0',
55-
`removed_unique_key_names` text NOT NULL,
56-
`dropped_no_default_column_names` text NOT NULL,
57-
`expanded_column_names` text NOT NULL,
58-
`revertible_notes` text NOT NULL,
59-
`allow_concurrent` tinyint unsigned NOT NULL DEFAULT '0',
60-
`reverted_uuid` varchar(64) NOT NULL DEFAULT '',
61-
`is_view` tinyint unsigned NOT NULL DEFAULT '0',
62-
`ready_to_complete` tinyint unsigned NOT NULL DEFAULT '0',
63-
`vitess_liveness_indicator` bigint NOT NULL DEFAULT '0',
64-
`user_throttle_ratio` float NOT NULL DEFAULT '0',
65-
`special_plan` text NOT NULL,
66-
`last_throttled_timestamp` timestamp NULL DEFAULT NULL,
67-
`component_throttled` tinytext NOT NULL,
68-
`reason_throttled` text NOT NULL,
69-
`cancelled_timestamp` timestamp NULL DEFAULT NULL,
70-
`postpone_launch` tinyint unsigned NOT NULL DEFAULT '0',
71-
`stage` text NOT NULL,
72-
`cutover_attempts` int unsigned NOT NULL DEFAULT '0',
73-
`is_immediate_operation` tinyint unsigned NOT NULL DEFAULT '0',
74-
`reviewed_timestamp` timestamp NULL DEFAULT NULL,
75-
`ready_to_complete_timestamp` timestamp NULL DEFAULT NULL,
76-
`shadow_analyzed_timestamp` timestamp NULL DEFAULT NULL,
77-
`removed_foreign_key_names` text NOT NULL,
78-
`last_cutover_attempt_timestamp` timestamp NULL DEFAULT NULL,
79-
`force_cutover` tinyint unsigned NOT NULL DEFAULT '0',
80-
`cutover_threshold_seconds` int unsigned NOT NULL DEFAULT '0',
81-
`dependent_migrations` text NOT NULL,
19+
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
20+
`migration_uuid` varchar(64) NOT NULL,
21+
`keyspace` varchar(256) NOT NULL,
22+
`shard` varchar(255) NOT NULL,
23+
`mysql_schema` varchar(128) NOT NULL,
24+
`mysql_table` varchar(128) NOT NULL,
25+
`migration_statement` text NOT NULL,
26+
`strategy` varchar(128) NOT NULL,
27+
`options` varchar(8192) NOT NULL,
28+
`added_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
29+
`requested_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
30+
`ready_timestamp` timestamp NULL DEFAULT NULL,
31+
`started_timestamp` timestamp NULL DEFAULT NULL,
32+
`liveness_timestamp` timestamp NULL DEFAULT NULL,
33+
`completed_timestamp` timestamp(6) NULL DEFAULT NULL,
34+
`cleanup_timestamp` timestamp NULL DEFAULT NULL,
35+
`migration_status` varchar(128) NOT NULL,
36+
`log_path` varchar(1024) NOT NULL,
37+
`artifacts` text NOT NULL,
38+
`retries` int unsigned NOT NULL DEFAULT '0',
39+
`tablet` varchar(128) NOT NULL DEFAULT '',
40+
`tablet_failure` tinyint unsigned NOT NULL DEFAULT '0',
41+
`progress` float NOT NULL DEFAULT '0',
42+
`migration_context` varchar(1024) NOT NULL DEFAULT '',
43+
`ddl_action` varchar(16) NOT NULL DEFAULT '',
44+
`message` text NOT NULL,
45+
`message_timestamp` timestamp(6) NULL DEFAULT NULL,
46+
`eta_seconds` bigint NOT NULL DEFAULT '-1',
47+
`rows_copied` bigint unsigned NOT NULL DEFAULT '0',
48+
`vreplication_lag_seconds` bigint unsigned NOT NULL DEFAULT '0',
49+
`table_rows` bigint NOT NULL DEFAULT '0',
50+
`added_unique_keys` int unsigned NOT NULL DEFAULT '0',
51+
`removed_unique_keys` int unsigned NOT NULL DEFAULT '0',
52+
`log_file` varchar(1024) NOT NULL DEFAULT '',
53+
`retain_artifacts_seconds` bigint NOT NULL DEFAULT '0',
54+
`postpone_completion` tinyint unsigned NOT NULL DEFAULT '0',
55+
`removed_unique_key_names` text NOT NULL,
56+
`dropped_no_default_column_names` text NOT NULL,
57+
`expanded_column_names` text NOT NULL,
58+
`revertible_notes` text NOT NULL,
59+
`allow_concurrent` tinyint unsigned NOT NULL DEFAULT '0',
60+
`reverted_uuid` varchar(64) NOT NULL DEFAULT '',
61+
`is_view` tinyint unsigned NOT NULL DEFAULT '0',
62+
`ready_to_complete` tinyint unsigned NOT NULL DEFAULT '0',
63+
`vitess_liveness_indicator` bigint NOT NULL DEFAULT '0',
64+
`user_throttle_ratio` float NOT NULL DEFAULT '0',
65+
`special_plan` text NOT NULL,
66+
`last_throttled_timestamp` timestamp NULL DEFAULT NULL,
67+
`component_throttled` tinytext NOT NULL,
68+
`reason_throttled` text NOT NULL,
69+
`cancelled_timestamp` timestamp NULL DEFAULT NULL,
70+
`postpone_launch` tinyint unsigned NOT NULL DEFAULT '0',
71+
`stage` text NOT NULL,
72+
`cutover_attempts` int unsigned NOT NULL DEFAULT '0',
73+
`is_immediate_operation` tinyint unsigned NOT NULL DEFAULT '0',
74+
`reviewed_timestamp` timestamp NULL DEFAULT NULL,
75+
`ready_to_complete_timestamp` timestamp NULL DEFAULT NULL,
76+
`shadow_analyzed_timestamp` timestamp NULL DEFAULT NULL,
77+
`removed_foreign_key_names` text NOT NULL,
78+
`last_cutover_attempt_timestamp` timestamp NULL DEFAULT NULL,
79+
`force_cutover` tinyint unsigned NOT NULL DEFAULT '0',
80+
`cutover_threshold_seconds` int unsigned NOT NULL DEFAULT '0',
81+
`postponed_by_in_order_completions` int unsigned NOT NULL DEFAULT '0',
8282
PRIMARY KEY (`id`),
8383
UNIQUE KEY `uuid_idx` (`migration_uuid`),
8484
KEY `keyspace_shard_idx` (`keyspace`(64), `shard`(64)),

‎go/vt/vtctl/grpcvtctldserver/query.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ func rowToSchemaMigration(row sqltypes.RowNamedValues) (sm *vtctldatapb.SchemaMi
150150
sm.VitessLivenessIndicator = row.AsInt64("vitess_liveness_indicator", 0)
151151
sm.UserThrottleRatio = float32(row.AsFloat64("user_throttle_ratio", 0))
152152
sm.SpecialPlan = row.AsString("special_plan", "")
153-
sm.DependentMigrations = row.AsString("dependent_migrations", "")
153+
sm.PostponedByInOrderCompletions = row.AsUint64("postponed_by_in_order_completions", 0)
154154

155155
sm.LastThrottledAt, err = valueToVTTime(row.AsString("last_throttled_timestamp", ""))
156156
if err != nil {

‎go/vt/vttablet/onlineddl/executor.go‎

Lines changed: 26 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -3098,18 +3098,20 @@ func shouldCutOverAccordingToBackoff(
30983098
return false, false
30993099
}
31003100

3101-
// getDependentMigrations returns a slice of migrations that must cut-over in-order, before the provided
3102-
// migration.
3103-
func getDependentMigrations(onlineDDL *schema.OnlineDDL, pendingMigrationsUUIDs []string) []string {
3104-
dependentMigrations := make([]string, 0, len(pendingMigrationsUUIDs))
3101+
// getInOrderCompletionPendingCount returns a count of migrations that must cut-over in-order, before the
3102+
// provided migration is able to proceed. This count is relevant only if the migration uses the
3103+
// --in-order-completion option.
3104+
func getInOrderCompletionPendingCount(onlineDDL *schema.OnlineDDL, pendingMigrationsUUIDs []string) uint64 {
3105+
var pendingCount uint64
31053106
for _, pendingMigrationsUUID := range pendingMigrationsUUIDs {
31063107
if pendingMigrationsUUID == onlineDDL.UUID {
3107-
// found all dependencies if we found ourself
3108+
// found all migrations we must wait for if
3109+
// we found ourself in the pending list.
31083110
break
31093111
}
3110-
dependentMigrations = append(dependentMigrations, pendingMigrationsUUID)
3112+
pendingCount++
31113113
}
3112-
return dependentMigrations
3114+
return pendingCount
31133115
}
31143116

31153117
// reviewRunningMigrations iterates migrations in 'running' state. Normally there's only one running, which was
@@ -3259,20 +3261,21 @@ func (e *Executor) reviewRunningMigrations(ctx context.Context) (countRunnning i
32593261
// In the case of a postponed migration, we will not complete it, but the user will
32603262
// understand whether "now is a good time" or "not there yet"
32613263
_ = e.updateMigrationReadyToComplete(ctx, uuid, isReady)
3262-
var postponeInOrderCompletion bool
3263-
if strategySetting.IsInOrderCompletion() {
3264-
if len(pendingMigrationsUUIDs) > 0 && pendingMigrationsUUIDs[0] != onlineDDL.UUID {
3265-
if err = e.updateDependentMigrations(ctx, onlineDDL.UUID, getDependentMigrations(onlineDDL, pendingMigrationsUUIDs)); err != nil {
3266-
return err
3267-
}
3268-
postponeInOrderCompletion = true
3269-
}
3270-
}
32713264
if !isReady {
32723265
// The migration is not ready yet.
32733266
return nil
32743267
}
3275-
if postponeCompletion || postponeInOrderCompletion {
3268+
if strategySetting.IsInOrderCompletion() {
3269+
var pendingMigrationsCount uint64
3270+
if len(pendingMigrationsUUIDs) > 0 && pendingMigrationsUUIDs[0] != onlineDDL.UUID {
3271+
pendingMigrationsCount = getInOrderCompletionPendingCount(onlineDDL, pendingMigrationsUUIDs)
3272+
postponeCompletion = true
3273+
}
3274+
if err = e.updatePostponedByInOrderCompletions(ctx, onlineDDL.UUID, pendingMigrationsCount); err != nil {
3275+
return err
3276+
}
3277+
}
3278+
if postponeCompletion {
32763279
// override. Even if migration is ready, we do not complete it.
32773280
return nil
32783281
}
@@ -3791,17 +3794,8 @@ func (e *Executor) updateMigrationStatusFailedOrCancelled(ctx context.Context, u
37913794
}
37923795

37933796
func (e *Executor) updateMigrationStatus(ctx context.Context, uuid string, status schema.OnlineDDLStatus) error {
3794-
var sqlUpdate string
3795-
switch status {
3796-
case schema.OnlineDDLStatusComplete, schema.OnlineDDLStatusFailed, schema.OnlineDDLStatusCancelled:
3797-
// sqlUpdateMigrationStatusFinal represents a final update to migration status.
3798-
sqlUpdate = sqlUpdateMigrationStatusFinal
3799-
default:
3800-
sqlUpdate = sqlUpdateMigrationStatus
3801-
}
3802-
38033797
log.Infof("updateMigrationStatus: transitioning migration: %s into status: %s", uuid, string(status))
3804-
query, err := sqlparser.ParseAndBind(sqlUpdate,
3798+
query, err := sqlparser.ParseAndBind(sqlUpdateMigrationStatus,
38053799
sqltypes.StringBindVariable(string(status)),
38063800
sqltypes.StringBindVariable(uuid),
38073801
)
@@ -4090,17 +4084,15 @@ func (e *Executor) updateMigrationUserThrottleRatio(ctx context.Context, uuid st
40904084
return err
40914085
}
40924086

4093-
func (e *Executor) updateDependentMigrations(
4087+
func (e *Executor) updatePostponedByInOrderCompletions(
40944088
ctx context.Context,
40954089
uuid string,
4096-
dependentMigrations []string,
4090+
pendingCompletions uint64,
40974091
) error {
4098-
if len(dependentMigrations) == 0 {
4099-
return nil
4100-
}
4101-
query, err := sqlparser.ParseAndBind(sqlUpdateDependentMigrations,
4102-
sqltypes.StringBindVariable(strings.Join(dependentMigrations, ",")),
4092+
query, err := sqlparser.ParseAndBind(sqlUpdatePostponedByInOrderCompletions,
4093+
sqltypes.Uint64BindVariable(pendingCompletions),
41034094
sqltypes.StringBindVariable(uuid),
4095+
sqltypes.Uint64BindVariable(pendingCompletions),
41044096
)
41054097
if err != nil {
41064098
return err
@@ -4136,23 +4128,8 @@ func (e *Executor) RetryMigration(ctx context.Context, uuid string) (result *sql
41364128
e.migrationMutex.Lock()
41374129
defer e.migrationMutex.Unlock()
41384130

4139-
onlineDDL, _, err := e.readMigration(ctx, uuid)
4140-
if err != nil {
4141-
return nil, vterrors.New(vtrpcpb.Code_FAILED_PRECONDITION, err.Error())
4142-
}
4143-
4144-
var dependentMigrations []string
4145-
if onlineDDL.StrategySetting().IsInOrderCompletion() {
4146-
pendingMigrationsUUIDs, err := e.readPendingMigrationsUUIDs(ctx)
4147-
if err != nil {
4148-
return nil, vterrors.New(vtrpcpb.Code_FAILED_PRECONDITION, err.Error())
4149-
}
4150-
dependentMigrations = getDependentMigrations(onlineDDL, pendingMigrationsUUIDs)
4151-
}
4152-
41534131
query, err := sqlparser.ParseAndBind(sqlRetryMigration,
41544132
sqltypes.StringBindVariable(e.TabletAliasString()),
4155-
sqltypes.StringBindVariable(strings.Join(dependentMigrations, ",")),
41564133
sqltypes.StringBindVariable(uuid),
41574134
)
41584135
if err != nil {
@@ -4638,15 +4615,6 @@ func (e *Executor) SubmitMigration(
46384615
}
46394616
log.Infof("SubmitMigration: request to submit migration %s; action=%s, table=%s", onlineDDL.UUID, actionStr, onlineDDL.Table)
46404617

4641-
var dependentMigrations []string
4642-
if onlineDDL.StrategySetting().IsInOrderCompletion() {
4643-
pendingMigrationsUUIDs, err := e.readPendingMigrationsUUIDs(ctx)
4644-
if err != nil {
4645-
return nil, err
4646-
}
4647-
dependentMigrations = getDependentMigrations(onlineDDL, pendingMigrationsUUIDs)
4648-
}
4649-
46504618
revertedUUID, _ := onlineDDL.GetRevertUUID(e.env.Environment().Parser()) // Empty value if the migration is not actually a REVERT. Safe to ignore error.
46514619
retainArtifactsSeconds := int64((retainOnlineDDLTables).Seconds())
46524620
if retainArtifacts, _ := onlineDDL.StrategySetting().RetainArtifactsDuration(); retainArtifacts != 0 {
@@ -4682,7 +4650,6 @@ func (e *Executor) SubmitMigration(
46824650
sqltypes.BoolBindVariable(allowConcurrentMigration),
46834651
sqltypes.StringBindVariable(revertedUUID),
46844652
sqltypes.BoolBindVariable(onlineDDL.IsView(e.env.Environment().Parser())),
4685-
sqltypes.StringBindVariable(strings.Join(dependentMigrations, ",")),
46864653
)
46874654
if err != nil {
46884655
return nil, err

0 commit comments

Comments
 (0)