switch to gunicorn due to issues with uwsgi and reporting

.gitignore

@@ -58,3 +58,4 @@ coverage.xml
 setup_dev.yml
 11env/
 query_schema.json
+gunicorn_config.py

api/tacticalrmm/core/management/commands/create_gunicorn_conf.py

@@ -0,0 +1,70 @@
+import multiprocessing
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Generate conf for gunicorn"
+
+    def handle(self, *args, **kwargs):
+        self.stdout.write("Creating gunicorn conf...")
+
+        cpu_count = multiprocessing.cpu_count()
+
+        # worker processes
+        workers = getattr(settings, "TRMM_GUNICORN_WORKERS", cpu_count * 2 + 1)
+        threads = getattr(settings, "TRMM_GUNICORN_THREADS", cpu_count * 2)
+        worker_class = getattr(settings, "TRMM_GUNICORN_WORKER_CLASS", "gthread")
+        max_requests = getattr(settings, "TRMM_GUNICORN_MAX_REQUESTS", 50)
+        max_requests_jitter = getattr(settings, "TRMM_GUNICORN_MAX_REQUESTS_JITTER", 8)
+        worker_connections = getattr(settings, "TRMM_GUNICORN_WORKER_CONNS", 1000)
+        timeout = getattr(settings, "TRMM_GUNICORN_TIMEOUT", 300)
+        graceful_timeout = getattr(settings, "TRMM_GUNICORN_GRACEFUL_TIMEOUT", 300)
+
+        # socket
+        backlog = getattr(settings, "TRMM_GUNICORN_BACKLOG", 2048)
+        if getattr(settings, "DOCKER_BUILD", False):
+            bind = "0.0.0.0:8080"
+        else:
+            bind = f"unix:{settings.BASE_DIR / 'tacticalrmm.sock'}"
+
+        # security
+        limit_request_line = getattr(settings, "TRMM_GUNICORN_LIMIT_REQUEST_LINE", 0)
+        limit_request_fields = getattr(
+            settings, "TRMM_GUNICORN_LIMIT_REQUEST_FIELDS", 500
+        )
+        limit_request_field_size = getattr(
+            settings, "TRMM_GUNICORN_LIMIT_REQUEST_FIELD_SIZE", 0
+        )
+
+        # server
+        preload_app = getattr(settings, "TRMM_GUNICORN_PRELOAD_APP", True)
+
+        # log
+        loglevel = getattr(settings, "TRMM_GUNICORN_LOGLEVEL", "info")
+
+        cfg = [
+            f"bind = '{bind}'",
+            f"workers = {workers}",
+            f"threads = {threads}",
+            f"worker_class = '{worker_class}'",
+            f"backlog = {backlog}",
+            f"worker_connections = {worker_connections}",
+            f"timeout = {timeout}",
+            f"graceful_timeout = {graceful_timeout}",
+            f"limit_request_line = {limit_request_line}",
+            f"limit_request_fields = {limit_request_fields}",
+            f"limit_request_field_size = {limit_request_field_size}",
+            f"max_requests = {max_requests}",
+            f"max_requests_jitter = {max_requests_jitter}",
+            f"loglevel = '{loglevel}'",
+            f"chdir = '{settings.BASE_DIR}'",
+            f"preload_app = {preload_app}",
+        ]
+
+        with open(settings.BASE_DIR / "gunicorn_config.py", "w") as fp:
+            for line in cfg:
+                fp.write(line + "\n")
+
+        self.stdout.write("Created gunicorn conf")

api/tacticalrmm/requirements.txt

@@ -14,6 +14,7 @@ django-ipware==5.0.0
 django-rest-knox==4.2.0
 djangorestframework==3.14.0
 drf-spectacular==0.26.5
+gunicorn==21.2.0
 hiredis==2.2.3
 meshctrl==0.1.15
 msgpack==1.0.7
@@ -33,12 +34,11 @@ six==1.16.0
 sqlparse==0.4.4
 twilio==8.10.0
 urllib3==2.0.7
-uWSGI==2.0.22
 validators==0.20.0
 vine==5.0.0
 websockets==11.0.3
 zipp==3.17.0
-pandas==2.1.1
+pandas==2.1.2
 kaleido==0.2.1
 jinja2==3.1.2
 markdown==3.3.6

docker/containers/tactical/entrypoint.sh

@@ -54,12 +54,12 @@ if [ "$1" = 'tactical-init' ]; then
   mkdir -p ${TACTICAL_DIR}/api/tacticalrmm/private/log
   touch ${TACTICAL_DIR}/api/tacticalrmm/private/log/django_debug.log
 
-  until (echo > /dev/tcp/"${POSTGRES_HOST}"/"${POSTGRES_PORT}") &> /dev/null; do
+  until (echo >/dev/tcp/"${POSTGRES_HOST}"/"${POSTGRES_PORT}") &>/dev/null; do
     echo "waiting for postgresql container to be ready..."
     sleep 5
   done
 
-  until (echo > /dev/tcp/"${MESH_SERVICE}"/4443) &> /dev/null; do
+  until (echo >/dev/tcp/"${MESH_SERVICE}"/4443) &>/dev/null; do
     echo "waiting for meshcentral container to be ready..."
     sleep 5
   done
@@ -69,7 +69,8 @@ if [ "$1" = 'tactical-init' ]; then
   ADMINURL=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 70 | head -n 1)
   DJANGO_SEKRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 80 | head -n 1)
 
-  localvars="$(cat << EOF
+  localvars="$(
+    cat <<EOF
 SECRET_KEY = '${DJANGO_SEKRET}'
 
 DEBUG = False
@@ -110,15 +111,15 @@ REDIS_HOST    = '${REDIS_HOST}'
 MESH_WS_URL = '${MESH_WS_URL}'
 ADMIN_ENABLED = False
 EOF
-)"
+  )"
 
-  echo "${localvars}" > ${TACTICAL_DIR}/api/tacticalrmm/local_settings.py
+  echo "${localvars}" >${TACTICAL_DIR}/api/tacticalrmm/local_settings.py
 
   # run migrations and init scripts
   python manage.py pre_update_tasks
   python manage.py migrate --no-input
   python manage.py generate_json_schemas
-  python manage.py get_webtar_url > ${TACTICAL_DIR}/tmp/web_tar_url
+  python manage.py get_webtar_url >${TACTICAL_DIR}/tmp/web_tar_url
   python manage.py collectstatic --no-input
   python manage.py initial_db_setup
   python manage.py initial_mesh_setup
@@ -126,7 +127,7 @@ EOF
   python manage.py load_community_scripts
   python manage.py reload_nats
   python manage.py create_natsapi_conf
-  python manage.py create_uwsgi_conf
+  python manage.py create_gunicorn_conf
   python manage.py create_installer_user
   python manage.py clear_redis_celery_locks
   python manage.py post_update_tasks
@@ -149,7 +150,7 @@ fi
 if [ "$1" = 'tactical-backend' ]; then
   check_tactical_ready
 
-  uwsgi ${TACTICAL_DIR}/api/app.ini
+  gunicorn -c ${TACTICAL_DIR}/api/gunicorn_config.py tacticalrmm.wsgi:application
 fi
 
 if [ "$1" = 'tactical-celery' ]; then

install.sh

@@ -505,7 +505,7 @@ python manage.py migrate
 python manage.py generate_json_schemas
 python manage.py collectstatic --no-input
 python manage.py create_natsapi_conf
-python manage.py create_uwsgi_conf
+python manage.py create_gunicorn_conf
 python manage.py load_chocos
 python manage.py load_community_scripts
 WEB_VERSION=$(python manage.py get_config webversion)
@@ -528,7 +528,7 @@ read -n 1 -s -r -p "Press any key to continue..."
 rmmservice="$(
   cat <<EOF
 [Unit]
-Description=tacticalrmm uwsgi daemon
+Description=tacticalrmm gunicorn daemon v1
 After=network.target postgresql.service
 
 [Service]
@@ -536,7 +536,10 @@ User=${USER}
 Group=www-data
 WorkingDirectory=/rmm/api/tacticalrmm
 Environment="PATH=/rmm/api/env/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ExecStart=/rmm/api/env/bin/uwsgi --ini app.ini
+ExecStart=/rmm/api/env/bin/gunicorn -c gunicorn_config.py tacticalrmm.wsgi:application
+ExecReload=/bin/kill -s HUP \$MAINPID
+ExecStartPre=rm -f /rmm/api/tacticalrmm/tacticalrmm.sock
+KillMode=mixed
 Restart=always
 RestartSec=10s
 
@@ -617,8 +620,8 @@ nginxrmm="$(
   cat <<EOF
 server_tokens off;
 
-upstream tacticalrmm {
+upstream tacticalgunicorn {
-    server unix:////rmm/api/tacticalrmm/tacticalrmm.sock;
+    server unix:/rmm/api/tacticalrmm/tacticalrmm.sock;
 }
 
 map \$http_user_agent \$ignore_ua {
@@ -696,10 +699,11 @@ server {
     }
 
     location / {
-        uwsgi_pass  tacticalrmm;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-        include     /etc/nginx/uwsgi_params;
+        proxy_set_header X-Forwarded-Proto \$scheme;
-        uwsgi_read_timeout 300s;
+        proxy_set_header Host \$http_host;
-        uwsgi_ignore_client_abort on;
+        proxy_redirect off;
+        proxy_pass http://tacticalgunicorn;
     }
 }
 EOF

restore.sh

@@ -235,6 +235,36 @@ sudo chown ${USER}:${USER} -R /etc/conf.d
 print_green 'Restoring systemd services'
 
 sudo cp $tmp_dir/systemd/* /etc/systemd/system/
+
+# for older systems still on uwsgi, migrate to gunicorn
+if ! grep -q gunicorn /etc/systemd/system/rmm.service; then
+  sudo rm -f /etc/systemd/system/rmm.service
+
+  gunicornsvc="$(
+    cat <<EOF
+[Unit]
+Description=tacticalrmm gunicorn daemon v1
+After=network.target postgresql.service
+
+[Service]
+User=${USER}
+Group=www-data
+WorkingDirectory=/rmm/api/tacticalrmm
+Environment="PATH=/rmm/api/env/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ExecStart=/rmm/api/env/bin/gunicorn -c gunicorn_config.py tacticalrmm.wsgi:application
+ExecReload=/bin/kill -s HUP \$MAINPID
+ExecStartPre=rm -f /rmm/api/tacticalrmm/tacticalrmm.sock
+KillMode=mixed
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+  )"
+  echo "${gunicornsvc}" | sudo tee /etc/systemd/system/rmm.service >/dev/null
+fi
+
 sudo systemctl daemon-reload
 
 print_green "Installing Python ${PYTHON_VER}"
@@ -428,7 +458,7 @@ python manage.py migrate
 python manage.py generate_json_schemas
 python manage.py collectstatic --no-input
 python manage.py create_natsapi_conf
-python manage.py create_uwsgi_conf
+python manage.py create_gunicorn_conf
 python manage.py reload_nats
 python manage.py post_update_tasks
 API=$(python manage.py get_config api)
@@ -456,7 +486,7 @@ for i in frontend meshcentral; do
   sudo ln -s /etc/nginx/sites-available/${i}.conf /etc/nginx/sites-enabled/${i}.conf
 done
 
-if ! grep -q "location /assets/" $tmp_dir/nginx/rmm.conf; then
+if ! grep -q gunicorn $tmp_dir/nginx/rmm.conf; then
   if [ -d "${tmp_dir}/certs/selfsigned" ]; then
     CERT_PUB_KEY="${certdir}/cert.pem"
     CERT_PRIV_KEY="${certdir}/key.pem"
@@ -465,8 +495,8 @@ if ! grep -q "location /assets/" $tmp_dir/nginx/rmm.conf; then
     cat <<EOF
 server_tokens off;
 
-upstream tacticalrmm {
+upstream tacticalgunicorn {
-    server unix:////rmm/api/tacticalrmm/tacticalrmm.sock;
+    server unix:/rmm/api/tacticalrmm/tacticalrmm.sock;
 }
 
 map \$http_user_agent \$ignore_ua {
@@ -544,10 +574,11 @@ server {
     }
 
     location / {
-        uwsgi_pass  tacticalrmm;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-        include     /etc/nginx/uwsgi_params;
+        proxy_set_header X-Forwarded-Proto \$scheme;
-        uwsgi_read_timeout 300s;
+        proxy_set_header Host \$http_host;
-        uwsgi_ignore_client_abort on;
+        proxy_redirect off;
+        proxy_pass http://tacticalgunicorn;
     }
 }
 EOF

update.sh

@@ -142,6 +142,35 @@ EOF
   sudo systemctl daemon-reload
 fi
 
+if ! grep -q gunicorn /etc/systemd/system/rmm.service; then
+  sudo rm -f /etc/systemd/system/rmm.service
+
+  gunicornsvc="$(
+    cat <<EOF
+[Unit]
+Description=tacticalrmm gunicorn daemon v1
+After=network.target postgresql.service
+
+[Service]
+User=${USER}
+Group=www-data
+WorkingDirectory=/rmm/api/tacticalrmm
+Environment="PATH=/rmm/api/env/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ExecStart=/rmm/api/env/bin/gunicorn -c gunicorn_config.py tacticalrmm.wsgi:application
+ExecReload=/bin/kill -s HUP \$MAINPID
+ExecStartPre=rm -f /rmm/api/tacticalrmm/tacticalrmm.sock
+KillMode=mixed
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+  )"
+  echo "${gunicornsvc}" | sudo tee /etc/systemd/system/rmm.service >/dev/null
+  sudo systemctl daemon-reload
+fi
+
 if [ ! -f /etc/apt/sources.list.d/nginx.list ]; then
   osname=$(lsb_release -si)
   osname=${osname^}
@@ -342,7 +371,7 @@ python manage.py reload_nats
 python manage.py load_chocos
 python manage.py create_installer_user
 python manage.py create_natsapi_conf
-python manage.py create_uwsgi_conf
+python manage.py create_gunicorn_conf
 python manage.py clear_redis_celery_locks
 python manage.py post_update_tasks
 API=$(python manage.py get_config api)
@@ -375,8 +404,8 @@ if ! grep -q "location /assets/" $rmmconf; then
     cat <<EOF
 server_tokens off;
 
-upstream tacticalrmm {
+upstream tacticalgunicorn {
-    server unix:////rmm/api/tacticalrmm/tacticalrmm.sock;
+    server unix:/rmm/api/tacticalrmm/tacticalrmm.sock;
 }
 
 map \$http_user_agent \$ignore_ua {
@@ -454,10 +483,11 @@ server {
     }
 
     location / {
-        uwsgi_pass  tacticalrmm;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-        include     /etc/nginx/uwsgi_params;
+        proxy_set_header X-Forwarded-Proto \$scheme;
-        uwsgi_read_timeout 300s;
+        proxy_set_header Host \$http_host;
-        uwsgi_ignore_client_abort on;
+        proxy_redirect off;
+        proxy_pass http://tacticalgunicorn;
     }
 }
 EOF