change monitoragents func to run async

This commit is contained in:
wh1te909 2021-02-16 08:33:54 +00:00
parent e49d97b898
commit 5d4dc4ed4c
2 changed files with 20 additions and 16 deletions

View File

@ -310,9 +310,6 @@ class LogCrash(APIView):
def post(self, request):
agent = get_object_or_404(Agent, agent_id=request.data["agentid"])
logger.info(
f"Detected crashed tacticalagent service on {agent.hostname} v{agent.version}, attempting recovery"
)
agent.last_seen = djangotime.now()
agent.save(update_fields=["last_seen"])
return Response("ok")

View File

@ -1,6 +1,7 @@
package api
import (
"sync"
"time"
"github.com/go-resty/resty/v2"
@ -23,24 +24,30 @@ func monitorAgents(c *resty.Client, nc *nats.Conn) {
tick := time.NewTicker(10 * time.Minute)
for range tick.C {
var wg sync.WaitGroup
agentids, _ := c.R().SetResult(&AgentIDS{}).Get("/offline/")
ids := agentids.Result().(*AgentIDS).IDs
wg.Add(len(ids))
var resp string
for _, id := range ids {
out, err := nc.Request(id, payload, 2*time.Second)
if err != nil {
continue
}
dec := codec.NewDecoderBytes(out.Data, &mh)
if err := dec.Decode(&resp); err == nil {
// if the agent is respoding to pong from the rpc service but is not showing as online (handled by tacticalagent service)
// then tacticalagent service is hung. forcefully restart it
if resp == "pong" {
nc.Publish(id, recPayload)
p := map[string]string{"agentid": id}
c.R().SetBody(p).Post("/logcrash/")
go func(id string, nc *nats.Conn, wg *sync.WaitGroup, c *resty.Client) {
defer wg.Done()
out, err := nc.Request(id, payload, 1*time.Second)
if err != nil {
return
}
}
dec := codec.NewDecoderBytes(out.Data, &mh)
if err := dec.Decode(&resp); err == nil {
// if the agent is respoding to pong from the rpc service but is not showing as online (handled by tacticalagent service)
// then tacticalagent service is hung. forcefully restart it
if resp == "pong" {
nc.Publish(id, recPayload)
p := map[string]string{"agentid": id}
c.R().SetBody(p).Post("/logcrash/")
}
}
}(id, nc, &wg, c)
}
}
}