compress eventlogs to significantly reduce size of data sent back to the rmm, which was causing salt to use insane amounts of memory

_modules/get_eventlog.py

@@ -4,8 +4,17 @@ import win32con
 import win32evtlogutil
 import winerror
 import datetime
-import random
-import string
+import zlib
+import json
+import base64
+
+
+def _compress_json(j):
+    return {
+        "wineventlog": base64.b64encode(
+            zlib.compress(json.dumps(j).encode("utf-8", errors="ignore"))
+        ).decode("ascii", errors="ignore")
+    }
 
 
 def get_eventlog(logtype, last_n_days):
@@ -28,6 +37,7 @@ def get_eventlog(logtype, last_n_days):
 
     try:
         events = 1
+        uid = 0
         while events:
             events = win32evtlog.ReadEventLog(hand, flags, 0)
             for ev_obj in events:
@@ -43,6 +53,7 @@ def get_eventlog(logtype, last_n_days):
                 evt_category = str(ev_obj.EventCategory)
                 record = str(ev_obj.RecordNumber)
                 msg = str(win32evtlogutil.SafeFormatMessage(ev_obj, logtype))
+                uid += 1
 
                 event_dict = {
                     "computer": computer,
@@ -53,12 +64,7 @@ def get_eventlog(logtype, last_n_days):
                     "message": msg,
                     "time": the_time,
                     "record": record,
-                    "uid": "".join(
-                        [
-                            random.choice(string.ascii_letters + string.digits)
-                            for n in range(60)
-                        ]
-                    ),
+                    "uid": uid,
                 }
 
                 log.append(event_dict)
@@ -66,8 +72,8 @@ def get_eventlog(logtype, last_n_days):
             if time_obj < start_time:
                 break
 
-    except Exception as e:
+    except Exception:
         pass
 
     win32evtlog.CloseEventLog(hand)
-    return log
+    return _compress_json(log)

api/tacticalrmm/agents/views.py

@@ -1,6 +1,9 @@
 from loguru import logger
 import subprocess
 from packaging import version as pyver
+import zlib
+import json
+import base64
 
 from django.conf import settings
 from django.shortcuts import get_object_or_404
@@ -236,13 +239,18 @@ def get_event_log(request, pk, logtype, days):
             func="get_eventlog.get_eventlog",
             arg=[logtype, int(days)],
         )
-        data = resp.json()
     except Exception:
         return Response(
             {"error": "unable to contact the agent"}, status=status.HTTP_400_BAD_REQUEST
         )
 
-    return Response(data["return"][0][agent.salt_id])
+    return Response(
+        json.loads(
+            zlib.decompress(
+                base64.b64decode(resp.json()["return"][0][agent.salt_id]["wineventlog"])
+            )
+        )
+    )
 
 
 @api_view(["POST"])

web/src/components/EventLog.vue

@@ -19,6 +19,7 @@
       row-key="uid"
       binary-state-sort
       hide-bottom
+      virtual-scroll
     >
       <template v-slot:top>
         <q-btn dense flat push @click="getEventLog" icon="refresh" />
@@ -123,7 +124,7 @@ export default {
       this.events = [];
       this.$q.loading.show({ message: `Loading ${this.logType} event log...please wait` });
       axios.get(`/agents/${this.pk}/geteventlog/${this.logType}/${this.days}/`).then(r => {
-        this.events = r.data;
+        this.events = Object.freeze(r.data);
         this.$q.loading.hide();
       })
       .catch(e => {