From f68b102ca8f99c5fe049e24cdb86eb382b90d018 Mon Sep 17 00:00:00 2001 From: sadnub Date: Sat, 2 Jan 2021 00:05:37 -0500 Subject: [PATCH] Add Dev Containers --- .devcontainer/api.dockerfile | 28 +++ .devcontainer/docker-compose.debug.yml | 19 ++ .devcontainer/docker-compose.yml | 232 ++++++++++++++++++ .devcontainer/entrypoint.sh | 182 ++++++++++++++ .devcontainer/requirements.txt | 44 ++++ .dockerignore | 27 +- .vscode/launch.json | 14 ++ .vscode/tasks.json | 23 ++ .../containers/tactical-nginx/entrypoint.sh | 7 +- 9 files changed, 570 insertions(+), 6 deletions(-) create mode 100644 .devcontainer/api.dockerfile create mode 100644 .devcontainer/docker-compose.debug.yml create mode 100644 .devcontainer/docker-compose.yml create mode 100644 .devcontainer/entrypoint.sh create mode 100644 .devcontainer/requirements.txt create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/api.dockerfile b/.devcontainer/api.dockerfile new file mode 100644 index 00000000..8d797a20 --- /dev/null +++ b/.devcontainer/api.dockerfile @@ -0,0 +1,28 @@ +FROM python:3.8-slim + +ENV TACTICAL_DIR /opt/tactical +ENV TACTICAL_GO_DIR /usr/local/rmmgo +ENV TACTICAL_READY_FILE ${TACTICAL_DIR}/tmp/tactical.ready +ENV WORKSPACE_DIR /workspace +ENV TACTICAL_USER tactical +ENV VIRTUAL_ENV ${WORKSPACE_DIR}/api/tacticalrmm/env +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8000 + +RUN groupadd -g 1000 tactical && \ + useradd -u 1000 -g 1000 tactical + +# Copy Go Files +COPY --from=golang:1.15 /usr/local/go ${TACTICAL_GO_DIR}/go + +# Copy Dev python reqs +COPY ./requirements.txt / + +# Copy Docker Entrypoint +COPY ./entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +WORKDIR ${WORKSPACE_DIR}/api/tacticalrmm diff --git a/.devcontainer/docker-compose.debug.yml b/.devcontainer/docker-compose.debug.yml new file mode 100644 index 00000000..fe7b819c --- /dev/null +++ b/.devcontainer/docker-compose.debug.yml @@ -0,0 +1,19 @@ +version: '3.4' + +services: + api-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000 --nothreading --noreload"] + ports: + - 8000:8000 + - 5678:5678 + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + networks: + dev: + aliases: + - tactical-backend diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..56a9d667 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,232 @@ +version: '3.4' + +services: + api-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + command: ["tactical-api"] + ports: + - 8000:8000 + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + networks: + dev: + aliases: + - tactical-backend + + app-dev: + image: node:12-alpine + ports: + - 8080:8080 + command: /bin/sh -c "npm install && npm run serve -- --host 0.0.0.0 --port 8080" + working_dir: /workspace/web + volumes: + - ..:/workspace:cached + networks: + dev: + aliases: + - tactical-frontend + + # salt master and api + salt-dev: + image: ${IMAGE_REPO}tactical-salt:${VERSION} + restart: always + volumes: + - tactical-data-dev:/opt/tactical + - salt-data-dev:/etc/salt + ports: + - "4505:4505" + - "4506:4506" + networks: + dev: + aliases: + - tactical-salt + + # nats + nats-dev: + image: ${IMAGE_REPO}tactical-nats:${VERSION} + restart: always + ports: + - "4222:4222" + volumes: + - tactical-data-dev:/opt/tactical + networks: + dev: + aliases: + - ${API_HOST} + - tactical-nats + + # meshcentral container + meshcentral-dev: + image: ${IMAGE_REPO}tactical-meshcentral:${VERSION} + restart: always + environment: + MESH_HOST: ${MESH_HOST} + MESH_USER: ${MESH_USER} + MESH_PASS: ${MESH_PASS} + MONGODB_USER: ${MONGODB_USER} + MONGODB_PASSWORD: ${MONGODB_PASSWORD} + NGINX_HOST_IP: 172.21.0.20 + networks: + dev: + aliases: + - tactical-meshcentral + - ${MESH_HOST} + volumes: + - tactical-data-dev:/opt/tactical + - mesh-data-dev:/home/node/app/meshcentral-data + depends_on: + - mongodb-dev + + # mongodb container for meshcentral + mongodb-dev: + image: mongo:4.4 + restart: always + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USER} + MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASSWORD} + MONGO_INITDB_DATABASE: meshcentral + networks: + dev: + aliases: + - tactical-mongodb + volumes: + - mongo-dev-data:/data/db + + # postgres database for api service + postgres-dev: + image: postgres:13-alpine + restart: always + environment: + POSTGRES_DB: tacticalrmm + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASS} + volumes: + - postgres-data-dev:/var/lib/postgresql/data + networks: + dev: + aliases: + - tactical-postgres + + # redis container for celery tasks + redis-dev: + restart: always + image: redis:6.0-alpine + networks: + dev: + aliases: + - tactical-redis + + init-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + restart: on-failure + command: ["tactical-init-dev"] + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASS: ${POSTGRES_PASS} + APP_HOST: ${APP_HOST} + API_HOST: ${API_HOST} + MESH_HOST: ${MESH_HOST} + TRMM_USER: ${TRMM_USER} + TRMM_PASS: ${TRMM_PASS} + depends_on: + - postgres-dev + - meshcentral-dev + networks: + - dev + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + + # container for celery worker service + celery-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + command: ["tactical-celery-dev"] + restart: always + networks: + - dev + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + depends_on: + - postgres-dev + - redis-dev + + # container for celery beat service + celerybeat-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + command: ["tactical-celerybeat-dev"] + restart: always + networks: + - dev + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + depends_on: + - postgres-dev + - redis-dev + + # container for celery winupdate tasks + celerywinupdate-dev: + image: api-dev + build: + context: . + dockerfile: ./api.dockerfile + command: ["tactical-celerywinupdate-dev"] + restart: always + networks: + - dev + volumes: + - tactical-data-dev:/opt/tactical + - ..:/workspace:cached + depends_on: + - postgres-dev + - redis-dev + + nginx-dev: + # container for tactical reverse proxy + image: ${IMAGE_REPO}tactical-nginx:${VERSION} + restart: always + environment: + APP_HOST: ${APP_HOST} + API_HOST: ${API_HOST} + MESH_HOST: ${MESH_HOST} + CERT_PUB_KEY: ${CERT_PUB_KEY} + CERT_PRIV_KEY: ${CERT_PRIV_KEY} + APP_PORT: 8080 + API_PORT: 8000 + networks: + dev: + ipv4_address: 172.21.0.20 + ports: + - "80:80" + - "443:443" + volumes: + - tactical-data-dev:/opt/tactical + +volumes: + tactical-data-dev: + postgres-data-dev: + mongo-dev-data: + mesh-data-dev: + salt-data-dev: + +networks: + dev: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.21.0.0/24 diff --git a/.devcontainer/entrypoint.sh b/.devcontainer/entrypoint.sh new file mode 100644 index 00000000..4f622fd4 --- /dev/null +++ b/.devcontainer/entrypoint.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -e + +: "${TRMM_USER:=tactical}" +: "${TRMM_PASS:=tactical}" +: "${POSTGRES_HOST:=tactical-postgres}" +: "${POSTGRES_PORT:=5432}" +: "${POSTGRES_USER:=tactical}" +: "${POSTGRES_PASS:=tactical}" +: "${POSTGRES_DB:=tacticalrmm}" +: "${SALT_HOST:=tactical-salt}" +: "${SALT_USER:=saltapi}" +: "${MESH_CONTAINER:=tactical-meshcentral}" +: "${MESH_USER:=meshcentral}" +: "${MESH_PASS:=meshcentralpass}" +: "${MESH_HOST:=tactical-meshcentral}" +: "${API_HOST:=tactical-backend}" +: "${APP_HOST:=tactical-frontend}" +: "${REDIS_HOST:=tactical-redis}" + +# Add python venv to path +export PATH="${VIRTUAL_ENV}/bin:$PATH" + +function check_tactical_ready { + sleep 15 + until [ -f "${TACTICAL_READY_FILE}" ]; do + echo "waiting for init container to finish install or update..." + sleep 10 + done +} + +function django_setup { + until (echo > /dev/tcp/"${POSTGRES_HOST}"/"${POSTGRES_PORT}") &> /dev/null; do + echo "waiting for postgresql container to be ready..." + sleep 5 + done + + until (echo > /dev/tcp/"${MESH_CONTAINER}"/443) &> /dev/null; do + echo "waiting for meshcentral container to be ready..." + sleep 5 + done + + echo "setting up django environment" + + # configure django settings + MESH_TOKEN=$(cat ${TACTICAL_DIR}/tmp/mesh_token) + + DJANGO_SEKRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 80 | head -n 1) + + # write salt pass to tmp dir + if [ ! -f "${TACTICAL__DIR}/tmp/salt_pass" ]; then + SALT_PASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) + echo "${SALT_PASS}" > ${TACTICAL_DIR}/tmp/salt_pass + else + SALT_PASS=$(cat ${TACTICAL_DIR}/tmp/salt_pass) + fi + + localvars="$(cat << EOF +SECRET_KEY = '${DJANGO_SEKRET}' + +DEBUG = True + +DOCKER_BUILD = True + +CERT_FILE = '/opt/tactical/certs/fullchain.pem' +KEY_FILE = '/opt/tactical/certs/privkey.pem' + +SCRIPTS_DIR = '${WORKSPACE_DIR}/scripts' + +ALLOWED_HOSTS = ['${API_HOST}'] + +ADMIN_URL = 'admin/' + +CORS_ORIGIN_ALLOW_ALL = True + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': '${POSTGRES_DB}', + 'USER': '${POSTGRES_USER}', + 'PASSWORD': '${POSTGRES_PASS}', + 'HOST': '${POSTGRES_HOST}', + 'PORT': '${POSTGRES_PORT}', + } +} + +REST_FRAMEWORK = { + 'DATETIME_FORMAT': '%b-%d-%Y - %H:%M', + + 'DEFAULT_PERMISSION_CLASSES': ( + 'rest_framework.permissions.IsAuthenticated', + ), + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'knox.auth.TokenAuthentication', + ), +} + +if not DEBUG: + REST_FRAMEWORK.update({ + 'DEFAULT_RENDERER_CLASSES': ( + 'rest_framework.renderers.JSONRenderer', + ) + }) + +SALT_USERNAME = '${SALT_USER}' +SALT_PASSWORD = '${SALT_PASS}' +SALT_HOST = '${SALT_HOST}' +MESH_USERNAME = '${MESH_USER}' +MESH_SITE = 'https://${MESH_HOST}' +MESH_TOKEN_KEY = '${MESH_TOKEN}' +REDIS_HOST = '${REDIS_HOST}' +EOF +)" + + echo "${localvars}" > ${WORKSPACE_DIR}/api/tacticalrmm/tacticalrmm/local_settings.py + + # run migrations and init scripts + python manage.py migrate --no-input + python manage.py collectstatic --no-input + python manage.py initial_db_setup + python manage.py initial_mesh_setup + python manage.py load_chocos + python manage.py load_community_scripts + python manage.py reload_nats + + # create super user + echo "from accounts.models import User; User.objects.create_superuser('${TRMM_USER}', 'admin@example.com', '${TRMM_PASS}') if not User.objects.filter(username='${TRMM_USER}').exists() else 0;" | python manage.py shell + +} + +if [ "$1" = 'tactical-init-dev' ]; then + + # make directories if they don't exist + mkdir -p ${TACTICAL_DIR}/tmp + + test -f "${TACTICAL_READY_FILE}" && rm "${TACTICAL_READY_FILE}" + + # setup Python virtual env and install dependencies + python -m venv ${VIRTUAL_ENV} + env/bin/pip install --no-cache-dir -r /requirements.txt + + django_setup + + # create .env file for frontend + webenv="$(cat << EOF +PROD_URL = "http://${API_HOST}:8000" +DEV_URL = "http://${API_HOST}:8000" +DEV_HOST = 0.0.0.0 +DEV_PORT = 8080 +EOF +)" + echo "${webenv}" | tee ${WORKSPACE_DIR}/web/.env > /dev/null + + # chown everything to tactical user + chown -R "${TACTICAL_USER}":"${TACTICAL_USER}" "${WORKSPACE_DIR}" + chown -R "${TACTICAL_USER}":"${TACTICAL_USER}" "${TACTICAL_DIR}" + + # create install ready file + su -c "echo 'tactical-init' > ${TACTICAL_READY_FILE}" "${TACTICAL_USER}" +fi + +if [ "$1" = 'tactical-api' ]; then + check_tactical_ready + python manage.py runserver 0.0.0.0:8000 +fi + +if [ "$1" = 'tactical-celery-dev' ]; then + check_tactical_ready + celery -A tacticalrmm worker -l debug +fi + +if [ "$1" = 'tactical-celerybeat-dev' ]; then + check_tactical_ready + test -f "${WORKSPACE_DIR}/api/tacticalrmm/celerybeat.pid" && rm "${WORKSPACE_DIR}/api/tacticalrmm/celerybeat.pid" + celery -A tacticalrmm beat -l debug +fi + +if [ "$1" = 'tactical-celerywinupdate-dev' ]; then + check_tactical_ready + celery -A tacticalrmm worker -Q wupdate -l debug +fi diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt new file mode 100644 index 00000000..64501412 --- /dev/null +++ b/.devcontainer/requirements.txt @@ -0,0 +1,44 @@ +# To ensure app dependencies are ported from your virtual environment/host machine into your container, run 'pip freeze > requirements.txt' in the terminal to overwrite this file +amqp==2.6.1 +asgiref==3.3.1 +asyncio-nats-client==0.11.4 +billiard==3.6.3.0 +celery==4.4.6 +certifi==2020.12.5 +cffi==1.14.3 +chardet==3.0.4 +cryptography==3.2.1 +decorator==4.4.2 +Django==3.1.4 +django-cors-headers==3.5.0 +django-rest-knox==4.1.0 +djangorestframework==3.12.2 +future==0.18.2 +idna==2.10 +kombu==4.6.11 +loguru==0.5.3 +msgpack==1.0.0 +packaging==20.4 +psycopg2-binary==2.8.6 +pycparser==2.20 +pycryptodome==3.9.9 +pyotp==2.4.1 +pyparsing==2.4.7 +pytz==2020.4 +qrcode==6.1 +redis==3.5.3 +requests==2.25.0 +six==1.15.0 +sqlparse==0.4.1 +twilio==6.49.0 +urllib3==1.26.2 +validators==0.18.1 +vine==1.3.0 +websockets==8.1 +zipp==3.4.0 +black +Werkzeug +django-extensions +coverage +coveralls +model_bakery diff --git a/.dockerignore b/.dockerignore index 041f7d5a..8fc93f28 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,24 @@ -.git -.cache -**/*.env -**/env +**/__pycache__ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/charts +**/docker-compose* +**/Dockerfile* **/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +README.md diff --git a/.vscode/launch.json b/.vscode/launch.json index 3646a0f3..37ffa7a7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,6 +14,20 @@ "0.0.0.0:8000" ], "django": true + }, + { + "name": "Django: Docker Remote Attach", + "type": "python", + "request": "attach", + "port": 5678, + "host": "localhost", + "preLaunchTask": "docker debug", + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/api/tacticalrmm", + "remoteRoot": "/workspace/api/tacticalrmm" + } + ] } ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..10e4059d --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "docker debug", + "type": "shell", + "command": "docker-compose", + "args": [ + "-p", + "trmm", + "-f", + ".devcontainer/docker-compose.yml", + "-f", + ".devcontainer/docker-compose.debug.yml", + "up", + "-d", + "--build" + ] + } + ] +} \ No newline at end of file diff --git a/docker/containers/tactical-nginx/entrypoint.sh b/docker/containers/tactical-nginx/entrypoint.sh index 1941bcea..a047c8ae 100644 --- a/docker/containers/tactical-nginx/entrypoint.sh +++ b/docker/containers/tactical-nginx/entrypoint.sh @@ -2,6 +2,9 @@ set -e +: "${APP_PORT:=80}" +: "${API_PORT:=80}" + CERT_PRIV_PATH=${TACTICAL_DIR}/certs/privkey.pem CERT_PUB_PATH=${TACTICAL_DIR}/certs/fullchain.pem @@ -31,7 +34,7 @@ server { location / { #Using variable to disable start checks - set \$api http://tactical-backend; + set \$api http://tactical-backend:${API_PORT}; proxy_pass \$api; proxy_http_version 1.1; @@ -95,7 +98,7 @@ server { location / { #Using variable to disable start checks - set \$app http://tactical-frontend; + set \$app http://tactical-frontend:${APP_PORT}; proxy_pass \$app; proxy_http_version 1.1;