rework agent recovery

agent/agent_windows.go

@@ -574,7 +574,7 @@ func (a *WindowsAgent) SyncMeshNodeID() {
 
 //RecoverMesh recovers mesh agent
 func (a *WindowsAgent) RecoverMesh() {
-	a.Logger.Debugln("Attempting mesh recovery on", a.Hostname)
+	a.Logger.Infoln("Attempting mesh recovery")
 	defer CMD("net", []string{"start", a.MeshSVC}, 60, false)
 
 	_, _ = CMD("net", []string{"stop", a.MeshSVC}, 60, false)
@@ -584,7 +584,7 @@ func (a *WindowsAgent) RecoverMesh() {
 
 //RecoverRPC recovers nats rpc service
 func (a *WindowsAgent) RecoverRPC() {
-	a.Logger.Debugln("Attempting rpc recovery on", a.Hostname)
+	a.Logger.Infoln("Attempting rpc recovery")
 	_, _ = CMD("net", []string{"stop", "tacticalrpc"}, 90, false)
 	time.Sleep(2 * time.Second)
 	_, _ = CMD("net", []string{"start", "tacticalrpc"}, 90, false)
@@ -958,3 +958,31 @@ func (a *WindowsAgent) RunMigrations() {
 	a.deleteOldTacticalServices()
 	CMD("schtasks.exe", []string{"/delete", "/TN", "TacticalRMM_fixmesh", "/f"}, 10, false)
 }
+
+func (a *WindowsAgent) CheckForRecovery() {
+	url := fmt.Sprintf("/api/v3/%s/recovery/", a.AgentID)
+	r, err := a.rClient.R().SetResult(&rmm.RecoveryAction{}).Get(url)
+
+	if err != nil {
+		a.Logger.Debugln("Recovery:", err)
+		return
+	}
+	if r.IsError() {
+		a.Logger.Debugln("Recovery status code:", r.StatusCode())
+		return
+	}
+
+	mode := r.Result().(*rmm.RecoveryAction).Mode
+	command := r.Result().(*rmm.RecoveryAction).ShellCMD
+
+	switch mode {
+	case "mesh":
+		a.RecoverMesh()
+	case "rpc":
+		a.RecoverRPC()
+	case "command":
+		a.RecoverCMD(command)
+	default:
+		return
+	}
+}

agent/winagentsvc_windows.go

@@ -30,6 +30,8 @@ func (a *WindowsAgent) WinAgentSvc() {
 		time.Sleep(time.Duration(randRange(300, 900)) * time.Millisecond)
 	}
 	a.SyncMeshNodeID()
+	time.Sleep(1 * time.Second)
+	a.CheckForRecovery()
 
 	time.Sleep(time.Duration(randRange(2, 7)) * time.Second)
 	a.CheckIn("startup")
@@ -42,6 +44,7 @@ func (a *WindowsAgent) WinAgentSvc() {
 	checkInLoggedUserTicker := time.NewTicker(time.Duration(randRange(850, 1400)) * time.Second)
 	checkInSWTicker := time.NewTicker(time.Duration(randRange(2400, 3000)) * time.Second)
 	syncMeshTicker := time.NewTicker(time.Duration(randRange(2400, 2900)) * time.Second)
+	recoveryTicker := time.NewTicker(time.Duration(randRange(180, 300)) * time.Second)
 
 	for {
 		select {
@@ -61,6 +64,8 @@ func (a *WindowsAgent) WinAgentSvc() {
 			a.CheckIn("software")
 		case <-syncMeshTicker.C:
 			a.SyncMeshNodeID()
+		case <-recoveryTicker.C:
+			a.CheckForRecovery()
 		}
 	}
 }

shared/types.go

@@ -2,6 +2,11 @@ package shared
 
 import "time"
 
+type RecoveryAction struct {
+	Mode     string `json:"mode"`
+	ShellCMD string `json:"shellcmd"`
+}
+
 type WinUpdateResult struct {
 	AgentID string       `json:"agent_id"`
 	Updates []WUAPackage `json:"wua_updates"`