diff --git a/api/tacticalrmm/agents/migrations/0028_auto_20210206_1534.py b/api/tacticalrmm/agents/migrations/0028_auto_20210206_1534.py new file mode 100644 index 00000000..a48d5d37 --- /dev/null +++ b/api/tacticalrmm/agents/migrations/0028_auto_20210206_1534.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.4 on 2021-02-06 15:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('agents', '0027_agent_overdue_dashboard_alert'), + ] + + operations = [ + migrations.AddField( + model_name='agentoutage', + name='outage_email_sent_time', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='agentoutage', + name='outage_sms_sent_time', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/api/tacticalrmm/agents/models.py b/api/tacticalrmm/agents/models.py index e7b0308b..714f108e 100644 --- a/api/tacticalrmm/agents/models.py +++ b/api/tacticalrmm/agents/models.py @@ -11,6 +11,7 @@ from collections import Counter from typing import List from typing import Union from loguru import logger +import datetime as dt from packaging import version as pyver from distutils.version import LooseVersion from nats.aio.client import Client as NATS @@ -680,16 +681,20 @@ class Agent(BaseAuditModel): # called when agent is offline else: - # return if outage has already been created - if self.agentoutages.exists() and self.agentoutages.last().is_active: - return + # outage hasn't been created yet so create it + if ( + not self.agentoutages.exists() + and not self.agentoutages.last().is_active + ): - outage = AgentOutage(agent=self) - outage.save() + outage = AgentOutage(agent=self) + outage.save() - # add a null check history to allow gaps in graph - for check in self.agentchecks.all(): - check.add_check_history(None) + # add a null check history to allow gaps in graph + for check in self.agentchecks.all(): + check.add_check_history(None) + else: + outage = self.agentoutages.last() # create dashboard alert if enabled if ( @@ -705,14 +710,20 @@ class Agent(BaseAuditModel): and alert_template.agent_always_email or self.overdue_email_alert ): - agent_outage_email_task.delay(pk=outage.pk) + agent_outage_email_task.delay( + pk=outage.pk, + alert_interval=alert_template.agent_periodic_alert_days, + ) if ( alert_template and alert_template.agent_always_text or self.overdue_text_alert ): - agent_outage_sms_task.delay(pk=outage.pk) + agent_outage_sms_task.delay( + pk=outage.pk, + alert_interval=alert_template.agent_periodic_alert_days, + ) class AgentOutage(models.Model): @@ -727,6 +738,8 @@ class AgentOutage(models.Model): recovery_time = models.DateTimeField(null=True, blank=True) outage_email_sent = models.BooleanField(default=False) outage_sms_sent = models.BooleanField(default=False) + outage_email_sent_time = models.DateTimeField(null=True, blank=True) + outage_sms_sent_time = models.DateTimeField(null=True, blank=True) recovery_email_sent = models.BooleanField(default=False) recovery_sms_sent = models.BooleanField(default=False) diff --git a/api/tacticalrmm/agents/tasks.py b/api/tacticalrmm/agents/tasks.py index ba485b8b..3fa420e0 100644 --- a/api/tacticalrmm/agents/tasks.py +++ b/api/tacticalrmm/agents/tasks.py @@ -3,8 +3,10 @@ from loguru import logger from time import sleep import random from packaging import version as pyver -from typing import List +from typing import List, Union +import datetime as dt +from django.utils import timezone as djangotime from django.conf import settings from scripts.models import Script @@ -106,40 +108,74 @@ def auto_self_agent_update_task() -> None: @app.task -def agent_outage_email_task(pk) -> None: - sleep(random.randint(1, 15)) +def agent_outage_email_task(pk: int, alert_interval: Union[float, None]) -> str: + outage = AgentOutage.objects.get(pk=pk) - outage.send_outage_email() - outage.outage_email_sent = True - outage.save(update_fields=["outage_email_sent"]) + + if not outage.outage_email_sent: + sleep(random.randint(1, 15)) + outage.send_outage_email() + outage.outage_email_sent = True + outage.outage_email_sent_time = djangotime.now() + outage.save(update_fields=["outage_email_sent"]) + else: + if alert_interval: + # send an email only if the last email sent is older than alert interval + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if outage.outage_email_sent_time < delta: + sleep(random.randint(1, 10)) + outage.send_outage_email() + outage.outage_email_sent_time = djangotime.now() + outage.save(update_fields=["outage_email_sent_time"]) + + return "ok" @app.task -def agent_recovery_email_task(pk) -> None: +def agent_recovery_email_task(pk: int) -> str: sleep(random.randint(1, 15)) outage = AgentOutage.objects.get(pk=pk) outage.send_recovery_email() outage.recovery_email_sent = True - outage.save(update_fields=["recovery_email_sent"]) + outage.outage_email_sent_time = djangotime.now() + outage.save(update_fields=["recovery_email_sent", "outage_email_sent_time"]) + + return "ok" @app.task -def agent_outage_sms_task(pk) -> None: - sleep(random.randint(1, 3)) +def agent_outage_sms_task(pk: int, alert_interval: Union[float, None]) -> str: outage = AgentOutage.objects.get(pk=pk) - outage.send_outage_sms() - outage.outage_sms_sent = True - outage.save(update_fields=["outage_sms_sent"]) + + if not outage.outage_sms_sent: + sleep(random.randint(1, 15)) + outage.send_outage_sms() + outage.outage_sms_sent = True + outage.outage_sms_sent_time = djangotime.now() + outage.save(update_fields=["outage_sms_sent", "outage_sms_sent_time"]) + else: + if alert_interval: + # send an sms only if the last sms sent is older than alert interval + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if outage.outage_sms_sent_time < delta: + sleep(random.randint(1, 10)) + outage.send_outage_sms() + outage.outage_sms_sent_time = djangotime.now() + outage.save(update_fields=["outage_sms_sent_time"]) + + return "ok" @app.task -def agent_recovery_sms_task(pk) -> None: +def agent_recovery_sms_task(pk: int) -> str: sleep(random.randint(1, 3)) outage = AgentOutage.objects.get(pk=pk) outage.send_recovery_sms() outage.recovery_sms_sent = True outage.save(update_fields=["recovery_sms_sent"]) + return "ok" + @app.task def agent_outages_task() -> None: diff --git a/api/tacticalrmm/autotasks/models.py b/api/tacticalrmm/autotasks/models.py index 53a91720..cdc5d6c1 100644 --- a/api/tacticalrmm/autotasks/models.py +++ b/api/tacticalrmm/autotasks/models.py @@ -225,6 +225,12 @@ class AutomatedTask(BaseAuditModel): def handle_alert(self) -> None: from alerts.models import Alert, AlertTemplate + from autotasks.tasks import ( + handle_task_email_alert, + handle_task_sms_alert, + handle_resolved_task_sms_alert, + handle_resolved_task_email_alert, + ) self.status = "failing" if self.retcode != 0 else "passing" @@ -247,11 +253,9 @@ class AutomatedTask(BaseAuditModel): not self.resolved_email_sent and alert_template.task_email_on_resolved ): - # TODO: send email on resolved - pass + handle_resolved_task_email_alert.delay(self.pk) if not self.resolved_text_sent and alert_template.task_text_on_resolved: - # TODO: send text on resolved - pass + handle_resolved_task_sms_alert.delay(self.pk) else: # create alert in dashboard if enabled if ( @@ -263,10 +267,72 @@ class AutomatedTask(BaseAuditModel): # send email if enabled if self.email_alert or alert_template and alert_template.check_always_email: - handle_task_email_alert_task.delay(self.pk) + handle_task_email_alert.delay( + self.pk, alert_template.task_periodic_alert_days + ) # send text if enabled if self.text_alert or alert_template and alert_template.check_always_text: - handle_task_sms_alert_task.delay(self.pk) + handle_task_sms_alert.delay( + self.pk, alert_template.task_periodic_alert_days + ) self.save() + + def send_email(self): + from core.models import CoreSettings + + CORE = CoreSettings.objects.first() + + if self.agent: + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Failed" + else: + subject = f"{self} Failed" + + body = ( + subject + + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}" + ) + + CORE.send_mail(subject, body) + + def send_sms(self): + + from core.models import CoreSettings + + CORE = CoreSettings.objects.first() + + if self.agent: + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Failed" + else: + subject = f"{self} Failed" + + body = ( + subject + + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}" + ) + + CORE.send_sms(body) + + def send_resolved_email(self): + from core.models import CoreSettings + + CORE = CoreSettings.objects.first() + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved" + body = ( + subject + + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}" + ) + + CORE.send_mail(subject, body) + + def send_resolved_text(self): + from core.models import CoreSettings + + CORE = CoreSettings.objects.first() + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved" + body = ( + subject + + f" - Return code: {self.retcode}\nStdout:{self.stdout}\nStderr: {self.stderr}" + ) + CORE.send_sms(body) \ No newline at end of file diff --git a/api/tacticalrmm/autotasks/tasks.py b/api/tacticalrmm/autotasks/tasks.py index 0e84dae3..f5bace61 100644 --- a/api/tacticalrmm/autotasks/tasks.py +++ b/api/tacticalrmm/autotasks/tasks.py @@ -6,6 +6,9 @@ from django.conf import settings import pytz from django.utils import timezone as djangotime from packaging import version as pyver +from typing import Union +import random +from time import sleep from .models import AutomatedTask from logs.models import PendingAction @@ -243,3 +246,89 @@ def remove_orphaned_win_tasks(agentpk): logger.info(f"Removed orphaned task {task} from {agent.hostname}") logger.info(f"Orphaned task cleanup finished on {agent.hostname}") + + +@app.task +def handle_task_email_alert(pk: int, alert_interval: Union[float, None]) -> str: + from .models import AutomatedTask + + check = AutomatedTask.objects.get(pk=pk) + + if not check.agent.maintenance_mode: + # first time sending email + if not check.email_sent: + sleep(random.randint(1, 10)) + check.send_email() + check.email_sent = djangotime.now() + check.save(update_fields=["email_sent"]) + else: + if alert_interval: + # send an email only if the last email sent is older than alert interval + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if check.email_sent < delta: + sleep(random.randint(1, 10)) + check.send_email() + check.email_sent = djangotime.now() + check.save(update_fields=["email_sent"]) + + return "ok" + + +@app.task +def handle_task_sms_alert(pk: int, alert_interval: Union[float, None]) -> str: + from .models import AutomatedTask + + task = AutomatedTask.objects.get(pk=pk) + + if not task.agent.maintenance_mode: + # first time sending text + if not task.text_sent: + sleep(random.randint(1, 3)) + task.send_sms() + task.text_sent = djangotime.now() + task.save(update_fields=["text_sent"]) + else: + if alert_interval: + # send a text only if the last text sent is older than alert interval + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if task.text_sent < delta: + sleep(random.randint(1, 3)) + task.send_sms() + task.text_sent = djangotime.now() + task.save(update_fields=["text_sent"]) + + return "ok" + + +@app.task +def handle_resolved_task_sms_alert(pk: int) -> str: + from .models import AutomatedTask + + task = AutomatedTask.objects.get(pk=pk) + + if not task.agent.maintenance_mode: + # first time sending text + if not task.resolved_text_sent: + sleep(random.randint(1, 3)) + task.send_resolved_sms() + task.resolved_text_sent = djangotime.now() + task.save(update_fields=["resolved_text_sent"]) + + return "ok" + + +@app.task +def handle_resolved_task_email_alert(pk: int) -> str: + from .models import AutomatedTask + + task = AutomatedTask.objects.get(pk=pk) + + if not task.agent.maintenance_mode: + # first time sending email + if not task.resolved_email_sent: + sleep(random.randint(1, 10)) + task.send_resolved_email() + task.resolved_email_sent = djangotime.now() + task.save(update_fields=["resolved_email_sent"]) + + return "ok" \ No newline at end of file diff --git a/api/tacticalrmm/checks/models.py b/api/tacticalrmm/checks/models.py index 1eba9882..84128eab 100644 --- a/api/tacticalrmm/checks/models.py +++ b/api/tacticalrmm/checks/models.py @@ -307,7 +307,9 @@ class Check(BaseAuditModel): and self.email_alert or alert_template.check_always_email ): - handle_check_email_alert_task.delay(self.pk) + handle_check_email_alert_task.delay( + self.pk, alert_template.check_periodic_alert_days + ) # send text if enabled if ( @@ -315,7 +317,9 @@ class Check(BaseAuditModel): and self.text_alert or alert_template.check_always_text ): - handle_check_sms_alert_task.delay(self.pk) + handle_check_sms_alert_task.delay( + self.pk, alert_template.check_periodic_alert_days + ) if alert_template.actions: # TODO: run scripts on agent @@ -803,10 +807,16 @@ class Check(BaseAuditModel): CORE.send_sms(body) def send_resolved_email(self): - pass + CORE = CoreSettings.objects.first() + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved" + body = f"{self} is now back to normal" + + CORE.send_mail(subject, body) def send_resolved_text(self): - pass + CORE = CoreSettings.objects.first() + subject = f"{self.agent.client.name}, {self.agent.site.name}, {self} Resolved" + CORE.send_sms(subject) class CheckHistory(models.Model): diff --git a/api/tacticalrmm/checks/tasks.py b/api/tacticalrmm/checks/tasks.py index d5b791bc..38246f55 100644 --- a/api/tacticalrmm/checks/tasks.py +++ b/api/tacticalrmm/checks/tasks.py @@ -1,13 +1,14 @@ import datetime as dt import random from time import sleep +from typing import Union from tacticalrmm.celery import app from django.utils import timezone as djangotime @app.task -def handle_check_email_alert_task(pk): +def handle_check_email_alert_task(pk, alert_interval: Union[float, None]) -> str: from .models import Check check = Check.objects.get(pk=pk) @@ -20,19 +21,20 @@ def handle_check_email_alert_task(pk): check.email_sent = djangotime.now() check.save(update_fields=["email_sent"]) else: - # send an email only if the last email sent is older than 24 hours - delta = djangotime.now() - dt.timedelta(hours=24) - if check.email_sent < delta: - sleep(random.randint(1, 10)) - check.send_email() - check.email_sent = djangotime.now() - check.save(update_fields=["email_sent"]) + if alert_interval: + # send an email only if the last email sent is older than alert interval + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if check.email_sent < delta: + sleep(random.randint(1, 10)) + check.send_email() + check.email_sent = djangotime.now() + check.save(update_fields=["email_sent"]) return "ok" @app.task -def handle_check_sms_alert_task(pk): +def handle_check_sms_alert_task(pk, alert_interval: Union[float, None]) -> str: from .models import Check check = Check.objects.get(pk=pk) @@ -45,19 +47,20 @@ def handle_check_sms_alert_task(pk): check.text_sent = djangotime.now() check.save(update_fields=["text_sent"]) else: - # send a text only if the last text sent is older than 24 hours - delta = djangotime.now() - dt.timedelta(hours=24) - if check.text_sent < delta: - sleep(random.randint(1, 3)) - check.send_sms() - check.text_sent = djangotime.now() - check.save(update_fields=["text_sent"]) + if alert_interval: + # send a text only if the last text sent is older than 24 hours + delta = djangotime.now() - dt.timedelta(days=alert_interval) + if check.text_sent < delta: + sleep(random.randint(1, 3)) + check.send_sms() + check.text_sent = djangotime.now() + check.save(update_fields=["text_sent"]) return "ok" @app.task -def handle_resolved_check_sms_alert_task(pk): +def handle_resolved_check_sms_alert_task(pk: int) -> str: from .models import Check check = Check.objects.get(pk=pk) @@ -74,7 +77,7 @@ def handle_resolved_check_sms_alert_task(pk): @app.task -def handle_resolved_check_email_alert_task(pk): +def handle_resolved_check_email_alert_task(pk: int) -> str: from .models import Check check = Check.objects.get(pk=pk)