From e9d71f169cbea3e5b3261d06e86a64325abdb66b Mon Sep 17 00:00:00 2001 From: sadnub Date: Sat, 16 Apr 2022 14:36:31 -0400 Subject: [PATCH] optimize the cache_db_values task --- ...move_agent_has_patches_pending_and_more.py | 21 +++++++ api/tacticalrmm/agents/models.py | 19 +----- api/tacticalrmm/agents/serializers.py | 4 +- api/tacticalrmm/agents/views.py | 9 ++- api/tacticalrmm/core/tasks.py | 61 +++++++++++++------ web/src/components/AgentTable.vue | 2 +- 6 files changed, 79 insertions(+), 37 deletions(-) create mode 100644 api/tacticalrmm/agents/migrations/0048_remove_agent_has_patches_pending_and_more.py diff --git a/api/tacticalrmm/agents/migrations/0048_remove_agent_has_patches_pending_and_more.py b/api/tacticalrmm/agents/migrations/0048_remove_agent_has_patches_pending_and_more.py new file mode 100644 index 00000000..06b0bf78 --- /dev/null +++ b/api/tacticalrmm/agents/migrations/0048_remove_agent_has_patches_pending_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.0.3 on 2022-04-16 17:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('agents', '0047_alter_agent_plat_alter_agent_site'), + ] + + operations = [ + migrations.RemoveField( + model_name='agent', + name='has_patches_pending', + ), + migrations.RemoveField( + model_name='agent', + name='pending_actions_count', + ), + ] diff --git a/api/tacticalrmm/agents/models.py b/api/tacticalrmm/agents/models.py index 144b1e48..1311aaae 100644 --- a/api/tacticalrmm/agents/models.py +++ b/api/tacticalrmm/agents/models.py @@ -68,8 +68,6 @@ class Agent(BaseAuditModel): ) maintenance_mode = models.BooleanField(default=False) block_policy_inheritance = models.BooleanField(default=False) - pending_actions_count = models.PositiveIntegerField(default=0) - has_patches_pending = models.BooleanField(default=False) alert_template = models.ForeignKey( "alerts.AlertTemplate", related_name="agents", @@ -351,23 +349,10 @@ class Agent(BaseAuditModel): checks = list(self.agentchecks.all()) + self.get_checks_from_policies() return self.add_check_results(checks) - def get_tasks_with_policies( - self, exclude_synced: bool = False - ) -> "List[AutomatedTask]": - from autotasks.models import TaskResult + def get_tasks_with_policies(self) -> "List[AutomatedTask]": tasks = list(self.autotasks.all()) + self.get_tasks_from_policies() - - if exclude_synced: - return [ - task - for task in tasks - if not task.task_result - or isinstance(task.task_result, TaskResult) - and task.task_result.sync_status != "synced" - ] - else: - return self.add_task_results(tasks) + return self.add_task_results(tasks) def add_task_results(self, tasks: "List[AutomatedTask]") -> "List[AutomatedTask]": diff --git a/api/tacticalrmm/agents/serializers.py b/api/tacticalrmm/agents/serializers.py index 95a16094..83520ba5 100644 --- a/api/tacticalrmm/agents/serializers.py +++ b/api/tacticalrmm/agents/serializers.py @@ -86,6 +86,8 @@ class AgentTableSerializer(serializers.ModelSerializer): policy = serializers.ReadOnlyField(source="policy.id") alert_template = serializers.SerializerMethodField() last_seen = serializers.ReadOnlyField() + pending_actions_count = serializers.ReadOnlyField() + has_pending_patches = serializers.ReadOnlyField() def get_alert_template(self, obj): @@ -121,7 +123,6 @@ class AgentTableSerializer(serializers.ModelSerializer): "monitoring_type", "description", "needs_reboot", - "has_patches_pending", "pending_actions_count", "status", "overdue_text_alert", @@ -137,6 +138,7 @@ class AgentTableSerializer(serializers.ModelSerializer): "block_policy_inheritance", "plat", "goarch", + "has_pending_patches", ] depth = 2 diff --git a/api/tacticalrmm/agents/views.py b/api/tacticalrmm/agents/views.py index 81f34619..04026682 100644 --- a/api/tacticalrmm/agents/views.py +++ b/api/tacticalrmm/agents/views.py @@ -14,7 +14,7 @@ from core.utils import ( get_core_settings, ) from django.conf import settings -from django.db.models import Q, Prefetch, F +from django.db.models import Q, Prefetch, Exists, Count from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.utils import timezone as djangotime @@ -29,6 +29,7 @@ from scripts.models import Script from scripts.tasks import handle_bulk_command_task, handle_bulk_script_task from winupdate.serializers import WinUpdatePolicySerializer from winupdate.tasks import bulk_check_for_updates_task, bulk_install_updates_task +from winupdate.models import WinUpdate from tacticalrmm.constants import AGENT_DEFER from tacticalrmm.permissions import ( @@ -107,6 +108,12 @@ class GetAgents(APIView): queryset=CheckResult.objects.select_related("assigned_check"), ), ) + .annotate(pending_actions_count=Count("pendingactions")) + .annotate( + has_pending_patches=Exists( + WinUpdate.objects.filter(action="approve", installed=False) + ) + ) ) ctx = {"default_tz": get_default_timezone()} serializer = AgentTableSerializer(agents, many=True, context=ctx) diff --git a/api/tacticalrmm/core/tasks.py b/api/tacticalrmm/core/tasks.py index 327c8376..3f9bd084 100644 --- a/api/tacticalrmm/core/tasks.py +++ b/api/tacticalrmm/core/tasks.py @@ -7,8 +7,11 @@ from alerts.tasks import prune_resolved_alerts from autotasks.models import TaskResult from checks.tasks import prune_check_history from clients.models import Client, Site +from checks.models import Check, CheckResult from core.utils import get_core_settings from django.conf import settings +from django.db.models import Prefetch, Exists, OuterRef +from logs.models import PendingAction from logs.tasks import prune_audit_log, prune_debug_log from packaging import version as pyver @@ -96,18 +99,51 @@ def _get_failing_data(agents: "QuerySet[Any]") -> Dict[str, bool]: @app.task def cache_db_fields_task() -> None: + + agent_queryset = ( + Agent.objects.defer(*AGENT_DEFER) + .select_related( + "site__server_policy", + "site__workstation_policy", + "site__client__server_policy", + "site__client__workstation_policy", + "policy", + "alert_template", + ) + .prefetch_related( + Prefetch( + "agentchecks", + queryset=Check.objects.select_related("script"), + ), + Prefetch( + "checkresults", + queryset=CheckResult.objects.select_related("assigned_check"), + ), + "autotasks", + Prefetch( + "taskresults", + queryset=TaskResult.objects.select_related("task"), + ), + ) + ) # update client/site failing check fields and agent counts for site in Site.objects.all(): - agents = site.agents.defer(*AGENT_DEFER) + agents = agent_queryset.filter(site=site) site.failing_checks = _get_failing_data(agents) site.save(update_fields=["failing_checks"]) for client in Client.objects.all(): - agents = Agent.objects.defer(*AGENT_DEFER).filter(site__client=client) + agents = agent_queryset.filter(site__client=client) client.failing_checks = _get_failing_data(agents) client.save(update_fields=["failing_checks"]) - for agent in Agent.objects.defer(*AGENT_DEFER): + for agent in agent_queryset.annotate( + has_pending_actions=Exists( + PendingAction.objects.filter( + pk=OuterRef("pk"), action_type="agent_update", status="pending" + ) + ) + ): if ( pyver.parse(agent.version) >= pyver.parse("1.6.0") and agent.status == "online" @@ -115,34 +151,25 @@ def cache_db_fields_task() -> None: # change agent update pending status to completed if agent has just updated if ( pyver.parse(agent.version) == pyver.parse(settings.LATEST_AGENT_VER) - and agent.pendingactions.filter( - action_type="agentupdate", status="pending" - ).exists() + and agent.has_pending_actions ): agent.pendingactions.filter( - action_type="agentupdate", status="pending" + action_type="agent_update", status="pending" ).update(status="completed") # sync scheduled tasks - for task in agent.get_tasks_with_policies(exclude_synced=True): + for task in agent.get_tasks_with_policies(): if not task.task_result or task.task_result.sync_status == "initial": task.create_task_on_agent(agent=agent if task.policy else None) elif task.task_result.sync_status == "pendingdeletion": task.delete_task_on_agent(agent=agent if task.policy else None) elif task.task_result.sync_status == "notsynced": task.modify_task_on_agent(agent=agent if task.policy else None) + elif task.task_result.sync_status == "synced": + continue # handles any alerting actions if Alert.objects.filter( alert_type="availability", agent=agent, resolved=False ).exists(): Alert.handle_alert_resolve(agent) - - # update pending patches and pending action counts - agent.pending_actions_count = agent.pendingactions.filter( - status="pending" - ).count() - agent.has_patches_pending = ( - agent.winupdates.filter(action="approve").filter(installed=False).exists() - ) - agent.save(update_fields=["pending_actions_count", "has_patches_pending"]) diff --git a/web/src/components/AgentTable.vue b/web/src/components/AgentTable.vue index 16259744..a7f65c1b 100644 --- a/web/src/components/AgentTable.vue +++ b/web/src/components/AgentTable.vue @@ -184,7 +184,7 @@