vtorc: allow recoveries to be disabled from startup (vitessio#18005)

Signed-off-by: Tim Vaillancourt <[email protected]>

Author: timvaillancourt

go/flags/endtoend/vtorc.txt

@@ -16,6 +16,7 @@ vtorc \
 
 Flags:
       --allow-emergency-reparent                                    Whether VTOrc should be allowed to run emergency reparent operation when it detects a dead primary (default true)
+      --allow-recovery                                              Whether VTOrc should be allowed to run recovery actions (default true)
       --alsologtostderr                                             log to standard error as well as files
       --audit-file-location string                                  File location where the audit logs are to be stored
       --audit-purge-duration duration                               Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s)

go/vt/vtorc/config/config.go

@@ -192,6 +192,15 @@ var (
 		},
 	)
 
+	allowRecovery = viperutil.Configure(
+		"allow-recovery",
+		viperutil.Options[bool]{
+			FlagName: "allow-recovery",
+			Default:  true,
+			Dynamic:  true,
+		},
+	)
+
 	convertTabletsWithErrantGTIDs = viperutil.Configure(
 		"change-tablets-with-errant-gtid-to-drained",
 		viperutil.Options[bool]{
@@ -234,6 +243,7 @@ func registerFlags(fs *pflag.FlagSet) {
 	fs.Duration("topo-information-refresh-duration", topoInformationRefreshDuration.Default(), "Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server")
 	fs.Duration("recovery-poll-duration", recoveryPollDuration.Default(), "Timer duration on which VTOrc polls its database to run a recovery")
 	fs.Bool("allow-emergency-reparent", ersEnabled.Default(), "Whether VTOrc should be allowed to run emergency reparent operation when it detects a dead primary")
+	fs.Bool("allow-recovery", allowRecovery.Default(), "Whether VTOrc should be allowed to run recovery actions")
 	fs.Bool("change-tablets-with-errant-gtid-to-drained", convertTabletsWithErrantGTIDs.Default(), "Whether VTOrc should be changing the type of tablets with errant GTIDs to DRAINED")
 	fs.Bool("enable-primary-disk-stalled-recovery", enablePrimaryDiskStalledRecovery.Default(), "Whether VTOrc should detect a stalled disk on the primary and failover")
 
@@ -255,6 +265,7 @@ func registerFlags(fs *pflag.FlagSet) {
 		topoInformationRefreshDuration,
 		recoveryPollDuration,
 		ersEnabled,
+		allowRecovery,
 		convertTabletsWithErrantGTIDs,
 		enablePrimaryDiskStalledRecovery,
 	)
@@ -380,6 +391,11 @@ func SetERSEnabled(val bool) {
 	ersEnabled.Set(val)
 }
 
+// GetAllowRecovery is a getter function.
+func GetAllowRecovery() bool {
+	return allowRecovery.Get()
+}
+
 // ConvertTabletWithErrantGTIDs reports whether VTOrc is allowed to change the tablet type of tablets with errant GTIDs to DRAINED.
 func ConvertTabletWithErrantGTIDs() bool {
 	return convertTabletsWithErrantGTIDs.Get()

go/vt/vtorc/logic/vtorc.go

@@ -274,6 +274,14 @@ func ContinuousDiscovery() {
 	log.Infof("continuous discovery: setting up")
 	recentDiscoveryOperationKeys = cache.New(config.GetInstancePollTime(), time.Second)
 
+	if !config.GetAllowRecovery() {
+		log.Info("--allow-recovery is set to 'false', disabling recovery actions")
+		if err := DisableRecovery(); err != nil {
+			log.Errorf("failed to disable recoveries: %+v", err)
+			return
+		}
+	}
+
 	go handleDiscoveryRequests()
 
 	healthTick := time.Tick(config.HealthPollSeconds * time.Second)