implement alert periodic notifications for agent, task, and check. implement sms/email functionality for autotasks

api/tacticalrmm/agents/migrations/0028_auto_20210206_1534.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.4 on 2021-02-06 15:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agents', '0027_agent_overdue_dashboard_alert'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='agentoutage',
+            name='outage_email_sent_time',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='agentoutage',
+            name='outage_sms_sent_time',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

api/tacticalrmm/agents/models.py

@@ -11,6 +11,7 @@ from collections import Counter
 from typing import List
 from typing import Union
 from loguru import logger
+import datetime as dt
 from packaging import version as pyver
 from distutils.version import LooseVersion
 from nats.aio.client import Client as NATS
@@ -680,16 +681,20 @@ class Agent(BaseAuditModel):
 
         # called when agent is offline
         else:
-            # return if outage has already been created
-            if self.agentoutages.exists() and self.agentoutages.last().is_active:
-                return
+            # outage hasn't been created yet so create it
+            if (
+                not self.agentoutages.exists()
+                and not self.agentoutages.last().is_active
+            ):
 
-            outage = AgentOutage(agent=self)
-            outage.save()
+                outage = AgentOutage(agent=self)
+                outage.save()
 
-            # add a null check history to allow gaps in graph
-            for check in self.agentchecks.all():
-                check.add_check_history(None)
+                # add a null check history to allow gaps in graph
+                for check in self.agentchecks.all():
+                    check.add_check_history(None)
+            else:
+                outage = self.agentoutages.last()
 
             # create dashboard alert if enabled
             if (
@@ -705,14 +710,20 @@ class Agent(BaseAuditModel):
                 and alert_template.agent_always_email
                 or self.overdue_email_alert
             ):
-                agent_outage_email_task.delay(pk=outage.pk)
+                agent_outage_email_task.delay(
+                    pk=outage.pk,
+                    alert_interval=alert_template.agent_periodic_alert_days,
+                )
 
             if (
                 alert_template
                 and alert_template.agent_always_text
                 or self.overdue_text_alert
             ):
-                agent_outage_sms_task.delay(pk=outage.pk)
+                agent_outage_sms_task.delay(
+                    pk=outage.pk,
+                    alert_interval=alert_template.agent_periodic_alert_days,
+                )
 
 
 class AgentOutage(models.Model):
@@ -727,6 +738,8 @@ class AgentOutage(models.Model):
     recovery_time = models.DateTimeField(null=True, blank=True)
     outage_email_sent = models.BooleanField(default=False)
     outage_sms_sent = models.BooleanField(default=False)
+    outage_email_sent_time = models.DateTimeField(null=True, blank=True)
+    outage_sms_sent_time = models.DateTimeField(null=True, blank=True)
     recovery_email_sent = models.BooleanField(default=False)
     recovery_sms_sent = models.BooleanField(default=False)
 

api/tacticalrmm/agents/tasks.py

@@ -3,8 +3,10 @@ from loguru import logger
 from time import sleep
 import random
 from packaging import version as pyver
-from typing import List
+from typing import List, Union
+import datetime as dt
 
+from django.utils import timezone as djangotime
 from django.conf import settings
 from scripts.models import Script
 
@@ -106,40 +108,74 @@ def auto_self_agent_update_task() -> None:
 
 
 @app.task
-def agent_outage_email_task(pk) -> None:
-    sleep(random.randint(1, 15))
+def agent_outage_email_task(pk: int, alert_interval: Union[float, None]) -> str:
+
     outage = AgentOutage.objects.get(pk=pk)
-    outage.send_outage_email()
-    outage.outage_email_sent = True
-    outage.save(update_fields=["outage_email_sent"])
+
+    if not outage.outage_email_sent:
+        sleep(random.randint(1, 15))
+        outage.send_outage_email()
+        outage.outage_email_sent = True
+        outage.outage_email_sent_time = djangotime.now()
+        outage.save(update_fields=["outage_email_sent"])
+    else:
+        if alert_interval:
+            # send an email only if the last email sent is older than alert interval
+            delta = djangotime.now() - dt.timedelta(days=alert_interval)
+            if outage.outage_email_sent_time < delta:
+                sleep(random.randint(1, 10))
+                outage.send_outage_email()
+                outage.outage_email_sent_time = djangotime.now()
+                outage.save(update_fields=["outage_email_sent_time"])
+
+    return "ok"
 
 
 @app.task
-def agent_recovery_email_task(pk) -> None:
+def agent_recovery_email_task(pk: int) -> str:
     sleep(random.randint(1, 15))
     outage = AgentOutage.objects.get(pk=pk)
     outage.send_recovery_email()
     outage.recovery_email_sent = True
-    outage.save(update_fields=["recovery_email_sent"])
+    outage.outage_email_sent_time = djangotime.now()
+    outage.save(update_fields=["recovery_email_sent", "outage_email_sent_time"])
+
+    return "ok"
 
 
 @app.task
-def agent_outage_sms_task(pk) -> None:
-    sleep(random.randint(1, 3))
+def agent_outage_sms_task(pk: int, alert_interval: Union[float, None]) -> str:
     outage = AgentOutage.objects.get(pk=pk)
-    outage.send_outage_sms()
-    outage.outage_sms_sent = True
-    outage.save(update_fields=["outage_sms_sent"])
+
+    if not outage.outage_sms_sent:
+        sleep(random.randint(1, 15))
+        outage.send_outage_sms()
+        outage.outage_sms_sent = True
+        outage.outage_sms_sent_time = djangotime.now()
+        outage.save(update_fields=["outage_sms_sent", "outage_sms_sent_time"])
+    else:
+        if alert_interval:
+            # send an sms only if the last sms sent is older than alert interval
+            delta = djangotime.now() - dt.timedelta(days=alert_interval)
+            if outage.outage_sms_sent_time < delta:
+                sleep(random.randint(1, 10))
+                outage.send_outage_sms()
+                outage.outage_sms_sent_time = djangotime.now()
+                outage.save(update_fields=["outage_sms_sent_time"])
+
+    return "ok"
 
 
 @app.task
-def agent_recovery_sms_task(pk) -> None:
+def agent_recovery_sms_task(pk: int) -> str:
     sleep(random.randint(1, 3))
     outage = AgentOutage.objects.get(pk=pk)
     outage.send_recovery_sms()
     outage.recovery_sms_sent = True
     outage.save(update_fields=["recovery_sms_sent"])
 
+    return "ok"
+
 
 @app.task
 def agent_outages_task() -> None:

api/tacticalrmm/autotasks/models.py

@@ -225,6 +225,12 @@ class AutomatedTask(BaseAuditModel):
 
     def handle_alert(self) -> None:
         from alerts.models import Alert, AlertTemplate
+        from autotasks.tasks import (
+            handle_task_email_alert,
+            handle_task_sms_alert,
+            handle_resolved_task_sms_alert,
+            handle_resolved_task_email_alert,
+        )
 
         self.status = "failing" if self.retcode != 0 else "passing"
 
@@ -247,11 +253,9 @@ class AutomatedTask(BaseAuditModel):
                     not self.resolved_email_sent
                     and alert_template.task_email_on_resolved
                 ):
-                    # TODO: send email on resolved
-                    pass
+                    handle_resolved_task_email_alert.delay(self.pk)
                 if not self.resolved_text_sent and alert_template.task_text_on_resolved:
-                    # TODO: send text on resolved
-                    pass
+                    handle_resolved_task_sms_alert.delay(self.pk)
         else:
             # create alert in dashboard if enabled
             if (
@@ -263,10 +267,72 @@ class AutomatedTask(BaseAuditModel):
 
             # send email if enabled
             if self.email_alert or alert_template and alert_template.check_always_email:
-                handle_task_email_alert_task.delay(self.pk)
+                handle_task_email_alert.delay(
+                    self.pk, alert_template.task_periodic_alert_days
+                )
 
             # send text if enabled
             if self.text_alert or alert_template and alert_template.check_always_text:
-                handle_task_sms_alert_task.delay(self.pk)
+                handle_task_sms_alert.delay(
+                    self.pk, alert_template.task_periodic_alert_days
+                )
 
         self.save()
+
+    def send_email(self):
+        from core.models import CoreSettings
+
+        CORE = CoreSettings.objects.first()
+
+        if self.agent:
+            subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Failed"
+        else:
+            subject = f"{self} Failed"
+
+        body = (
+            subject
+            + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}"
+        )
+
+        CORE.send_mail(subject, body)
+
+    def send_sms(self):
+
+        from core.models import CoreSettings
+
+        CORE = CoreSettings.objects.first()
+
+        if self.agent:
+            subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Failed"
+        else:
+            subject = f"{self} Failed"
+
+        body = (
+            subject
+            + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}"
+        )
+
+        CORE.send_sms(body)
+
+    def send_resolved_email(self):
+        from core.models import CoreSettings
+
+        CORE = CoreSettings.objects.first()
+        subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved"
+        body = (
+            subject
+            + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}"
+        )
+
+        CORE.send_mail(subject, body)
+
+    def send_resolved_text(self):
+        from core.models import CoreSettings
+
+        CORE = CoreSettings.objects.first()
+        subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved"
+        body = (
+            subject
+            + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}"
+        )
+        CORE.send_sms(body)

api/tacticalrmm/autotasks/tasks.py

@@ -6,6 +6,9 @@ from django.conf import settings
 import pytz
 from django.utils import timezone as djangotime
 from packaging import version as pyver
+from typing import Union
+import random
+from time import sleep
 
 from .models import AutomatedTask
 from logs.models import PendingAction
@@ -243,3 +246,89 @@ def remove_orphaned_win_tasks(agentpk):
                 logger.info(f"Removed orphaned task {task} from {agent.hostname}")
 
     logger.info(f"Orphaned task cleanup finished on {agent.hostname}")
+
+
+@app.task
+def handle_task_email_alert(pk: int, alert_interval: Union[float, None]) -> str:
+    from .models import AutomatedTask
+
+    check = AutomatedTask.objects.get(pk=pk)
+
+    if not check.agent.maintenance_mode:
+        # first time sending email
+        if not check.email_sent:
+            sleep(random.randint(1, 10))
+            check.send_email()
+            check.email_sent = djangotime.now()
+            check.save(update_fields=["email_sent"])
+        else:
+            if alert_interval:
+                # send an email only if the last email sent is older than alert interval
+                delta = djangotime.now() - dt.timedelta(days=alert_interval)
+                if check.email_sent < delta:
+                    sleep(random.randint(1, 10))
+                    check.send_email()
+                    check.email_sent = djangotime.now()
+                    check.save(update_fields=["email_sent"])
+
+    return "ok"
+
+
+@app.task
+def handle_task_sms_alert(pk: int, alert_interval: Union[float, None]) -> str:
+    from .models import AutomatedTask
+
+    task = AutomatedTask.objects.get(pk=pk)
+
+    if not task.agent.maintenance_mode:
+        # first time sending text
+        if not task.text_sent:
+            sleep(random.randint(1, 3))
+            task.send_sms()
+            task.text_sent = djangotime.now()
+            task.save(update_fields=["text_sent"])
+        else:
+            if alert_interval:
+                # send a text only if the last text sent is older than alert interval
+                delta = djangotime.now() - dt.timedelta(days=alert_interval)
+                if task.text_sent < delta:
+                    sleep(random.randint(1, 3))
+                    task.send_sms()
+                    task.text_sent = djangotime.now()
+                    task.save(update_fields=["text_sent"])
+
+    return "ok"
+
+
+@app.task
+def handle_resolved_task_sms_alert(pk: int) -> str:
+    from .models import AutomatedTask
+
+    task = AutomatedTask.objects.get(pk=pk)
+
+    if not task.agent.maintenance_mode:
+        # first time sending text
+        if not task.resolved_text_sent:
+            sleep(random.randint(1, 3))
+            task.send_resolved_sms()
+            task.resolved_text_sent = djangotime.now()
+            task.save(update_fields=["resolved_text_sent"])
+
+    return "ok"
+
+
+@app.task
+def handle_resolved_task_email_alert(pk: int) -> str:
+    from .models import AutomatedTask
+
+    task = AutomatedTask.objects.get(pk=pk)
+
+    if not task.agent.maintenance_mode:
+        # first time sending email
+        if not task.resolved_email_sent:
+            sleep(random.randint(1, 10))
+            task.send_resolved_email()
+            task.resolved_email_sent = djangotime.now()
+            task.save(update_fields=["resolved_email_sent"])
+
+    return "ok"

api/tacticalrmm/checks/models.py

@@ -307,7 +307,9 @@ class Check(BaseAuditModel):
                     and self.email_alert
                     or alert_template.check_always_email
                 ):
-                    handle_check_email_alert_task.delay(self.pk)
+                    handle_check_email_alert_task.delay(
+                        self.pk, alert_template.check_periodic_alert_days
+                    )
 
                 # send text if enabled
                 if (
@@ -315,7 +317,9 @@ class Check(BaseAuditModel):
                     and self.text_alert
                     or alert_template.check_always_text
                 ):
-                    handle_check_sms_alert_task.delay(self.pk)
+                    handle_check_sms_alert_task.delay(
+                        self.pk, alert_template.check_periodic_alert_days
+                    )
 
                 if alert_template.actions:
                     # TODO: run scripts on agent
@@ -803,10 +807,16 @@ class Check(BaseAuditModel):
         CORE.send_sms(body)
 
     def send_resolved_email(self):
-        pass
+        CORE = CoreSettings.objects.first()
+        subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved"
+        body = f"{self} is now back to normal"
+
+        CORE.send_mail(subject, body)
 
     def send_resolved_text(self):
-        pass
+        CORE = CoreSettings.objects.first()
+        subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved"
+        CORE.send_sms(subject)
 
 
 class CheckHistory(models.Model):

api/tacticalrmm/checks/tasks.py

@@ -1,13 +1,14 @@
 import datetime as dt
 import random
 from time import sleep
+from typing import Union
 
 from tacticalrmm.celery import app
 from django.utils import timezone as djangotime
 
 
 @app.task
-def handle_check_email_alert_task(pk):
+def handle_check_email_alert_task(pk, alert_interval: Union[float, None]) -> str:
     from .models import Check
 
     check = Check.objects.get(pk=pk)
@@ -20,19 +21,20 @@ def handle_check_email_alert_task(pk):
             check.email_sent = djangotime.now()
             check.save(update_fields=["email_sent"])
         else:
-            # send an email only if the last email sent is older than 24 hours
-            delta = djangotime.now() - dt.timedelta(hours=24)
-            if check.email_sent < delta:
-                sleep(random.randint(1, 10))
-                check.send_email()
-                check.email_sent = djangotime.now()
-                check.save(update_fields=["email_sent"])
+            if alert_interval:
+                # send an email only if the last email sent is older than alert interval
+                delta = djangotime.now() - dt.timedelta(days=alert_interval)
+                if check.email_sent < delta:
+                    sleep(random.randint(1, 10))
+                    check.send_email()
+                    check.email_sent = djangotime.now()
+                    check.save(update_fields=["email_sent"])
 
     return "ok"
 
 
 @app.task
-def handle_check_sms_alert_task(pk):
+def handle_check_sms_alert_task(pk, alert_interval: Union[float, None]) -> str:
     from .models import Check
 
     check = Check.objects.get(pk=pk)
@@ -45,19 +47,20 @@ def handle_check_sms_alert_task(pk):
             check.text_sent = djangotime.now()
             check.save(update_fields=["text_sent"])
         else:
-            # send a text only if the last text sent is older than 24 hours
-            delta = djangotime.now() - dt.timedelta(hours=24)
-            if check.text_sent < delta:
-                sleep(random.randint(1, 3))
-                check.send_sms()
-                check.text_sent = djangotime.now()
-                check.save(update_fields=["text_sent"])
+            if alert_interval:
+                # send a text only if the last text sent is older than 24 hours
+                delta = djangotime.now() - dt.timedelta(days=alert_interval)
+                if check.text_sent < delta:
+                    sleep(random.randint(1, 3))
+                    check.send_sms()
+                    check.text_sent = djangotime.now()
+                    check.save(update_fields=["text_sent"])
 
     return "ok"
 
 
 @app.task
-def handle_resolved_check_sms_alert_task(pk):
+def handle_resolved_check_sms_alert_task(pk: int) -> str:
     from .models import Check
 
     check = Check.objects.get(pk=pk)
@@ -74,7 +77,7 @@ def handle_resolved_check_sms_alert_task(pk):
 
 
 @app.task
-def handle_resolved_check_email_alert_task(pk):
+def handle_resolved_check_email_alert_task(pk: int) -> str:
     from .models import Check
 
     check = Check.objects.get(pk=pk)