Skip to content

Commit 384f0d1

Browse files
tanjinxtimvaillancourt
tanjinx
and
timvaillancourt
authored
vtorc: add keyspace/shard labels to recoveries stats (vitessio#18304) (#740)
Signed-off-by: Tim Vaillancourt <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> Co-authored-by: Tim Vaillancourt <[email protected]>
1 parent dade949 commit 384f0d1

File tree

4 files changed

+39
-42
lines changed

4 files changed

+39
-42
lines changed

‎go/test/endtoend/vtorc/general/vtorc_test.go‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func TestErrantGTIDOnPreviousPrimary(t *testing.T) {
7575
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
7676
assert.NotNil(t, curPrimary, "should have elected a primary")
7777
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
78-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
78+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
7979
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
8080

8181
var replica, otherReplica *cluster.Vttablet
@@ -134,7 +134,7 @@ func TestSingleKeyspace(t *testing.T) {
134134

135135
utils.CheckPrimaryTablet(t, clusterInfo, shard0.Vttablets[0], true)
136136
utils.CheckReplication(t, clusterInfo, shard0.Vttablets[0], shard0.Vttablets[1:], 10*time.Second)
137-
utils.WaitForSuccessfulRecoveryCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], logic.ElectNewPrimaryRecoveryName, 1)
137+
utils.WaitForSuccessfulRecoveryCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
138138
utils.WaitForSuccessfulPRSCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], keyspace.Name, shard0.Name, 1)
139139
}
140140

@@ -152,7 +152,7 @@ func TestKeyspaceShard(t *testing.T) {
152152

153153
utils.CheckPrimaryTablet(t, clusterInfo, shard0.Vttablets[0], true)
154154
utils.CheckReplication(t, clusterInfo, shard0.Vttablets[0], shard0.Vttablets[1:], 10*time.Second)
155-
utils.WaitForSuccessfulRecoveryCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], logic.ElectNewPrimaryRecoveryName, 1)
155+
utils.WaitForSuccessfulRecoveryCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
156156
utils.WaitForSuccessfulPRSCount(t, clusterInfo.ClusterInstance.VTOrcProcesses[0], keyspace.Name, shard0.Name, 1)
157157
}
158158

@@ -175,7 +175,7 @@ func TestVTOrcRepairs(t *testing.T) {
175175
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
176176
assert.NotNil(t, curPrimary, "should have elected a primary")
177177
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
178-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
178+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
179179
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
180180

181181
var replica, otherReplica *cluster.Vttablet
@@ -203,7 +203,7 @@ func TestVTOrcRepairs(t *testing.T) {
203203
// wait for repair
204204
match := utils.WaitForReadOnlyValue(t, curPrimary, 0)
205205
require.True(t, match)
206-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixPrimaryRecoveryName, 1)
206+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
207207
})
208208

209209
t.Run("ReplicaReadWrite", func(t *testing.T) {
@@ -214,7 +214,7 @@ func TestVTOrcRepairs(t *testing.T) {
214214
// wait for repair
215215
match := utils.WaitForReadOnlyValue(t, replica, 1)
216216
require.True(t, match)
217-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 1)
217+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 1)
218218
})
219219

220220
t.Run("StopReplication", func(t *testing.T) {
@@ -224,23 +224,23 @@ func TestVTOrcRepairs(t *testing.T) {
224224

225225
// check replication is setup correctly
226226
utils.CheckReplication(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
227-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 2)
227+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 2)
228228

229229
// Stop just the IO thread on the replica
230230
_, err = utils.RunSQL(t, "STOP REPLICA IO_THREAD", replica, "")
231231
require.NoError(t, err)
232232

233233
// check replication is setup correctly
234234
utils.CheckReplication(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
235-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 3)
235+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 3)
236236

237237
// Stop just the SQL thread on the replica
238238
_, err = utils.RunSQL(t, "STOP REPLICA SQL_THREAD", replica, "")
239239
require.NoError(t, err)
240240

241241
// check replication is setup correctly
242242
utils.CheckReplication(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
243-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 4)
243+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 4)
244244
})
245245

246246
t.Run("ReplicationFromOtherReplica", func(t *testing.T) {
@@ -256,7 +256,7 @@ func TestVTOrcRepairs(t *testing.T) {
256256

257257
// wait until the source port is set back correctly by vtorc
258258
utils.CheckSourcePort(t, replica, curPrimary, 15*time.Second)
259-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 5)
259+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 5)
260260

261261
// check that writes succeed
262262
utils.VerifyWritesSucceed(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
@@ -268,7 +268,7 @@ func TestVTOrcRepairs(t *testing.T) {
268268

269269
// wait until heart beat interval has been fixed by vtorc.
270270
utils.CheckHeartbeatInterval(t, replica, 16.5, 15*time.Second)
271-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 6)
271+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 6)
272272

273273
// check that writes succeed
274274
utils.VerifyWritesSucceed(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second)
@@ -291,7 +291,7 @@ func TestVTOrcRepairs(t *testing.T) {
291291
// wait for repair
292292
err = utils.WaitForReplicationToStop(t, curPrimary)
293293
require.NoError(t, err)
294-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryHasPrimaryRecoveryName, 1)
294+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryHasPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
295295
// check that the writes still succeed
296296
utils.VerifyWritesSucceed(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 10*time.Second)
297297
})
@@ -514,7 +514,7 @@ func TestVTOrcWithPrs(t *testing.T) {
514514
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
515515
assert.NotNil(t, curPrimary, "should have elected a primary")
516516
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
517-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
517+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
518518
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
519519

520520
// find any replica tablet other than the current primary
@@ -540,13 +540,13 @@ func TestVTOrcWithPrs(t *testing.T) {
540540
// check that the replica gets promoted
541541
utils.CheckPrimaryTablet(t, clusterInfo, replica, true)
542542
// Verify that VTOrc didn't run any other recovery
543-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
543+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
544544
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
545-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, 0)
545+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, keyspace.Name, shard0.Name, 0)
546546
utils.WaitForSuccessfulERSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 0)
547-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixPrimaryRecoveryName, 0)
548-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, 0)
549-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryHasPrimaryRecoveryName, 0)
547+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixPrimaryRecoveryName, keyspace.Name, shard0.Name, 0)
548+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.FixReplicaRecoveryName, keyspace.Name, shard0.Name, 0)
549+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryHasPrimaryRecoveryName, keyspace.Name, shard0.Name, 0)
550550
utils.VerifyWritesSucceed(t, clusterInfo, replica, shard0.Vttablets, 10*time.Second)
551551
}
552552

@@ -673,7 +673,7 @@ func TestFullStatusConnectionPooling(t *testing.T) {
673673
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
674674
assert.NotNil(t, curPrimary, "should have elected a primary")
675675
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
676-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
676+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
677677
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
678678

679679
// Kill the current primary.

‎go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func TestDownPrimary(t *testing.T) {
5252
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
5353
assert.NotNil(t, curPrimary, "should have elected a primary")
5454
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
55-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
55+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
5656
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
5757

5858
// find the replica and rdonly tablets
@@ -99,7 +99,7 @@ func TestDownPrimary(t *testing.T) {
9999

100100
// also check that the replication is working correctly after failover
101101
utils.VerifyWritesSucceed(t, clusterInfo, replica, []*cluster.Vttablet{crossCellReplica}, 10*time.Second)
102-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, 1)
102+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
103103
utils.WaitForSuccessfulERSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
104104
t.Run("Check ERS and PRS Vars and Metrics", func(t *testing.T) {
105105
utils.CheckVarExists(t, vtOrcProcess, "EmergencyReparentCounts")
@@ -164,7 +164,7 @@ func TestDownPrimaryBeforeVTOrc(t *testing.T) {
164164

165165
// also check that the replication is working correctly after failover
166166
utils.VerifyWritesSucceed(t, clusterInfo, replica, []*cluster.Vttablet{rdonly}, 10*time.Second)
167-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, 1)
167+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
168168
utils.WaitForSuccessfulERSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
169169
}
170170

@@ -178,7 +178,7 @@ func TestDeletedPrimaryTablet(t *testing.T) {
178178
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
179179
assert.NotNil(t, curPrimary, "should have elected a primary")
180180
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
181-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
181+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
182182
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
183183

184184
// find the replica and rdonly tablets
@@ -229,7 +229,7 @@ func TestDeletedPrimaryTablet(t *testing.T) {
229229

230230
// also check that the replication is working correctly after failover
231231
utils.VerifyWritesSucceed(t, clusterInfo, replica, []*cluster.Vttablet{rdonly}, 10*time.Second)
232-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryTabletDeletedRecoveryName, 1)
232+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverPrimaryTabletDeletedRecoveryName, keyspace.Name, shard0.Name, 1)
233233
}
234234

235235
// TestDeadPrimaryRecoversImmediately test Vtorc ability to recover immediately if primary is dead.
@@ -249,7 +249,7 @@ func TestDeadPrimaryRecoversImmediately(t *testing.T) {
249249
curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0)
250250
assert.NotNil(t, curPrimary, "should have elected a primary")
251251
vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0]
252-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1)
252+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
253253
utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
254254

255255
// find the replica and rdonly tablets
@@ -286,7 +286,7 @@ func TestDeadPrimaryRecoversImmediately(t *testing.T) {
286286
utils.WaitForInstancePollSecondsExceededCount(t, vtOrcProcess, "InstancePollSecondsExceeded", 2, false)
287287
// also check that the replication is working correctly after failover
288288
utils.VerifyWritesSucceed(t, clusterInfo, replica, []*cluster.Vttablet{crossCellReplica}, 10*time.Second)
289-
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, 1)
289+
utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.RecoverDeadPrimaryRecoveryName, keyspace.Name, shard0.Name, 1)
290290
utils.WaitForSuccessfulERSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1)
291291

292292
// Parse log file and find out how much time it took for DeadPrimary to recover.

‎go/test/endtoend/vtorc/utils/utils.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -988,14 +988,15 @@ func WaitForReadOnlyValue(t *testing.T, curPrimary *cluster.Vttablet, expectValu
988988
}
989989

990990
// WaitForSuccessfulRecoveryCount waits until the given recovery name's count of successful runs matches the count expected
991-
func WaitForSuccessfulRecoveryCount(t *testing.T, vtorcInstance *cluster.VTOrcProcess, recoveryName string, countExpected int) {
991+
func WaitForSuccessfulRecoveryCount(t *testing.T, vtorcInstance *cluster.VTOrcProcess, recoveryName, keyspace, shard string, countExpected int) {
992992
t.Helper()
993993
timeout := 15 * time.Second
994994
startTime := time.Now()
995+
mapKey := fmt.Sprintf("%s.%s.%s", recoveryName, keyspace, shard)
995996
for time.Since(startTime) < timeout {
996997
vars := vtorcInstance.GetVars()
997998
successfulRecoveriesMap := vars["SuccessfulRecoveries"].(map[string]interface{})
998-
successCount := GetIntFromValue(successfulRecoveriesMap[recoveryName])
999+
successCount := GetIntFromValue(successfulRecoveriesMap[mapKey])
9991000
if successCount == countExpected {
10001001
return
10011002
}

‎go/vt/vtorc/logic/topology_recovery.go‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,6 @@ const (
5151
)
5252

5353
var (
54-
actionableRecoveriesNames = []string{
55-
RecoverDeadPrimaryRecoveryName,
56-
RecoverPrimaryHasPrimaryRecoveryName,
57-
ElectNewPrimaryRecoveryName,
58-
FixPrimaryRecoveryName,
59-
FixReplicaRecoveryName,
60-
}
61-
6254
countPendingRecoveries = stats.NewGauge("PendingRecoveries", "Count of the number of pending recoveries")
6355

6456
// detectedProblems is used to track the number of detected problems.
@@ -75,14 +67,17 @@ var (
7567
// shardsLockCounter is a count of in-flight shard locks. Use atomics to read/update.
7668
shardsLockCounter int64
7769

70+
// recoveriesCounterLabels are labels for grouping the counter based stats for recoveries.
71+
recoveriesCounterLabels = []string{"RecoveryType", "Keyspace", "Shard"}
72+
7873
// recoveriesCounter counts the number of recoveries that VTOrc has performed
79-
recoveriesCounter = stats.NewCountersWithSingleLabel("RecoveriesCount", "Count of the different recoveries performed", "RecoveryType", actionableRecoveriesNames...)
74+
recoveriesCounter = stats.NewCountersWithMultiLabels("RecoveriesCount", "Count of the different recoveries performed", recoveriesCounterLabels)
8075

8176
// recoveriesSuccessfulCounter counts the number of successful recoveries that VTOrc has performed
82-
recoveriesSuccessfulCounter = stats.NewCountersWithSingleLabel("SuccessfulRecoveries", "Count of the different successful recoveries performed", "RecoveryType", actionableRecoveriesNames...)
77+
recoveriesSuccessfulCounter = stats.NewCountersWithMultiLabels("SuccessfulRecoveries", "Count of the different successful recoveries performed", recoveriesCounterLabels)
8378

8479
// recoveriesFailureCounter counts the number of failed recoveries that VTOrc has performed
85-
recoveriesFailureCounter = stats.NewCountersWithSingleLabel("FailedRecoveries", "Count of the different failed recoveries performed", "RecoveryType", actionableRecoveriesNames...)
80+
recoveriesFailureCounter = stats.NewCountersWithMultiLabels("FailedRecoveries", "Count of the different failed recoveries performed", recoveriesCounterLabels)
8681

8782
// shardLockTimings measures the timing of LockShard operations.
8883
shardLockTimingsActions = []string{"Lock", "Unlock"}
@@ -637,13 +632,14 @@ func executeCheckAndRecoverFunction(analysisEntry *inst.ReplicationAnalysis) (er
637632
return err
638633
}
639634
recoveryName := getRecoverFunctionName(checkAndRecoverFunctionCode)
640-
recoveriesCounter.Add(recoveryName, 1)
635+
recoveryLabels := []string{recoveryName, analysisEntry.AnalyzedKeyspace, analysisEntry.AnalyzedShard}
636+
recoveriesCounter.Add(recoveryLabels, 1)
641637
if err != nil {
642638
logger.Errorf("Failed to recover: %+v", err)
643-
recoveriesFailureCounter.Add(recoveryName, 1)
639+
recoveriesFailureCounter.Add(recoveryLabels, 1)
644640
} else {
645641
logger.Info("Recovery succeeded")
646-
recoveriesSuccessfulCounter.Add(recoveryName, 1)
642+
recoveriesSuccessfulCounter.Add(recoveryLabels, 1)
647643
}
648644
if topologyRecovery == nil {
649645
logger.Error("Topology recovery is nil - recovery might have failed")

0 commit comments

Comments
 (0)