catch an exception that a celery task could potentially throw and configure automation task retries

api/tacticalrmm/automation/tasks.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Union
 from tacticalrmm.celery import app
 
 
-@app.task
+@app.task(retry_backoff=5, retry_jitter=True, retry_kwargs={"max_retries": 5})
 def generate_agent_checks_task(
     policy: int = None,
     site: int = None,
@@ -57,7 +57,9 @@ def generate_agent_checks_task(
     return "ok"
 
 
-@app.task
+@app.task(
+    acks_late=True, retry_backoff=5, retry_jitter=True, retry_kwargs={"max_retries": 5}
+)
 # updates policy managed check fields on agents
 def update_policy_check_fields_task(check: int) -> str:
     from checks.models import Check
@@ -73,7 +75,7 @@ def update_policy_check_fields_task(check: int) -> str:
     return "ok"
 
 
-@app.task
+@app.task(retry_backoff=5, retry_jitter=True, retry_kwargs={"max_retries": 5})
 # generates policy tasks on agents affected by a policy
 def generate_agent_autotasks_task(policy: int = None) -> str:
     from agents.models import Agent
@@ -100,7 +102,12 @@ def generate_agent_autotasks_task(policy: int = None) -> str:
     return "ok"
 
 
-@app.task
+@app.task(
+    acks_late=True,
+    retry_backoff=5,
+    retry_jitter=True,
+    retry_kwargs={"max_retries": 5},
+)
 def delete_policy_autotasks_task(task: int) -> str:
     from autotasks.models import AutomatedTask
 
@@ -120,7 +127,12 @@ def run_win_policy_autotasks_task(task: int) -> str:
     return "ok"
 
 
-@app.task
+@app.task(
+    acks_late=True,
+    retry_backoff=5,
+    retry_jitter=True,
+    retry_kwargs={"max_retries": 5},
+)
 def update_policy_autotasks_fields_task(task: int, update_agent: bool = False) -> str:
     from autotasks.models import AutomatedTask
 

api/tacticalrmm/autotasks/models.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.db.models.fields import DateTimeField
+from django.db.utils import DatabaseError
 from django.utils import timezone as djangotime
 from logs.models import BaseAuditModel
 from loguru import logger
@@ -365,9 +366,14 @@ class AutomatedTask(BaseAuditModel):
 
         if r != "ok" and "The system cannot find the file specified" not in r:
             self.sync_status = "pendingdeletion"
-            self.save(update_fields=["sync_status"])
+
+            try:
+                self.save(update_fields=["sync_status"])
+            except DatabaseError:
+                pass
+
             logger.warning(
-                f"{agent.hostname} task {self.name} was successfully modified"
+                f"{agent.hostname} task {self.name} will be deleted on next checkin"
             )
             return "timeout"
         else: