Merge pull request #2013 from jerryfletcher21/traditional-environment-production

Traditional non docker environment for production

requirements.txt

@@ -14,7 +14,7 @@ grpcio-tools==1.67.0
 numpy==1.26.4
 Pillow==11.0.0
 python-decouple==3.8
-requests==2.32.0
+requests==2.32.3
 ring==0.10.1
 gunicorn==23.0.0
 psycopg2==2.9.10
@@ -31,4 +31,4 @@ base91==1.0.1
 nostr-sdk==0.35.1
 pygeohash==3.0.1
 asgiref == 3.8.1
-secp256k1
+secp256k1

scripts/traditional/README.md

@@ -262,6 +262,32 @@ $ ra_reg escrow-pay RobotName
 ...
 ```
 
+Production:
+```
+# edit .env setting LNVENDOR to either "CLN" or "LND"
+DEVELOPMENT = False
+TESTING = False
+USE_TOR = True
+LOG_TO_CONSOLE = False
+LOGGER_LEVEL = "WARNING"
+HOST_NAME = "coordinator_onion_address.onion"
+HOST_NAME2 = "*"
+ONION_LOCATION = "coordinator_onion_address.onion"
+
+$ . venv/bin/activate
+
+$ scripts/traditional/traditional-services postgres-setup
+
+$ scripts/traditional/traditional-services postgres-database-production
+
+$ scripts/traditional/traditional-services strfry-setup
+$ scripts/traditional/traditional-services nginx-setup
+
+# change commit_sha
+
+$ scripts/traditional/traditional-services production
+```
+
 Update:
 ```
 $ . venv/bin/activate
@@ -269,9 +295,19 @@ $ . venv/bin/activate
 $ pip install -r requirements_dev.txt
 $ pip install -r requirements.txt
 
+$ scripts/generate_grpc.sh
+
+# for testing:
 # start just postgres and redis in an other window
 $ scripts/traditional/traditional-services test
-
 # in the main window
+$ . scripts/traditional/regtest-nodes test
+
+# for production:
+# backup traditional folder
+# update commit_sha
+$ scripts/traditional/traditional-services production
+
+# then for both testing and production, in the main window
 $ python3 manage.py migrate
 ```

scripts/traditional/strfry-sync-external

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+_get_env_var() {
+    if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then
+        echo "error: getting $1 from .env" >&2
+        return 1
+    fi
+    printf "%s\n" "$env_var"
+
+    return 0
+}
+
+# transform relative path into absolute and remove trailing slashes
+_get_env_var_path() {
+    env_var="$(_get_env_var "$1")" || return "$?"
+    real_path="$(realpath -m "$env_var")" || return "$?"
+    printf "%s\n" "$real_path"
+
+    return 0
+}
+
+TRADITIONAL_SERVICES_DIR="$(_get_env_var_path "TRADITIONAL_SERVICES_DIR")" || return "$?"
+
+STRFRY_DIR="$TRADITIONAL_SERVICES_DIR/strfry"
+
+STRFRY_GIT_DIR="$(_get_env_var_path "STRFRY_GIT_DIR")" || return "$?"
+
+strfry_bin="$STRFRY_GIT_DIR/strfry"
+
+onion_url_list=$(cat << EOF
+wss://nostr.satstralia.com
+wss://relay.damus.io
+wss://freelay.sovbit.host
+EOF
+)
+
+while true; do
+    for onion_url in $onion_url_list; do
+        torsocks timeout 15s "$strfry_bin" \
+            --config "$STRFRY_DIR/strfry.conf" \
+            sync "$onion_url" \
+            --filter '{"kinds":[38383]}' \
+            --dir both
+    done
+
+    sleep 300
+done

scripts/traditional/strfry-sync-federation

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+_get_env_var() {
+    if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then
+        echo "error: getting $1 from .env" >&2
+        return 1
+    fi
+    printf "%s\n" "$env_var"
+
+    return 0
+}
+
+# transform relative path into absolute and remove trailing slashes
+_get_env_var_path() {
+    env_var="$(_get_env_var "$1")" || return "$?"
+    real_path="$(realpath -m "$env_var")" || return "$?"
+    printf "%s\n" "$real_path"
+
+    return 0
+}
+
+TRADITIONAL_SERVICES_DIR="$(_get_env_var_path "TRADITIONAL_SERVICES_DIR")" || return "$?"
+
+STRFRY_DIR="$TRADITIONAL_SERVICES_DIR/strfry"
+
+STRFRY_GIT_DIR="$(_get_env_var_path "STRFRY_GIT_DIR")" || return "$?"
+
+strfry_bin="$STRFRY_GIT_DIR/strfry"
+
+onion_url_list=$(cat << EOF
+ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/relay
+ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/relay
+ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/relay
+ws://otmoonrndnrddqdlhu6b36heunmbyw3cgvadqo2oqeau3656wfv7fwad.onion/relay
+ws://librebazovfmmkyi2jekraxsuso3mh622avuuzqpejixdl5dhuhb4tid.onion/relay
+EOF
+)
+
+while true; do
+    for onion_url in $onion_url_list; do
+        torsocks timeout 15s "$strfry_bin" \
+            --config "$STRFRY_DIR/strfry.conf" \
+            sync "$onion_url" \
+            --filter '{"kinds":[38383, 31986, 1059]}' \
+            --dir both
+    done
+
+    sleep 120
+done

scripts/traditional/templates/index.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+        <meta charset="utf-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+        <title>
+robosats
+        </title>
+        <style type="text/css">
+            body {
+                font-size: 16px;
+                margin: 40px auto;
+                padding: 0 10px;
+                max-width: 650px;
+            }
+        </style>
+    </head>
+    <body>
+        <header>
+            <h1>
+Robosats
+            </h1>
+        </header>
+        <p>
+robosats coordinator
+        </p>
+        <p>
+available at /api
+        </p>
+    </body>
+</html>

scripts/traditional/templates/nginx.conf

@@ -0,0 +1,236 @@
+# adapted from:
+# * debian /etc/nginx/nginx.conf
+# * robosats-deploy/compose/nginx/mn.conf.d/local.conf
+
+# user robosats;
+worker_processes auto;
+pid $NGINX_DIR/nginx.pid;
+error_log $NGINX_DIR/error.log;
+
+events {
+    worker_connections 1024;
+    # multi_accept on;
+}
+
+http {
+    client_body_temp_path $NGINX_DIR/temp/client_body;
+    proxy_temp_path       $NGINX_DIR/temp/proxy;
+    fastcgi_temp_path     $NGINX_DIR/temp/fastcgi;
+    uwsgi_temp_path       $NGINX_DIR/temp/uwsgi;
+    scgi_temp_path        $NGINX_DIR/temp/scgi;
+
+    sendfile on;
+    tcp_nopush on;
+    types_hash_max_size 2048;
+    server_tokens off;
+
+    # nginx: [emerg] could not build map_hash, you should increase map_hash_bucket_size: 64
+    map_hash_bucket_size 4096;
+
+    # server_names_hash_bucket_size 64;
+    # server_name_in_redirect off;
+
+    types {
+        text/html                             html htm shtml;
+        text/css                              css;
+        text/xml                              xml;
+        image/gif                             gif;
+        image/jpeg                            jpeg jpg;
+        application/javascript                js;
+        application/atom+xml                  atom;
+        application/rss+xml                   rss;
+
+        text/mathml                           mml;
+        text/plain                            txt;
+        text/vnd.sun.j2me.app-descriptor      jad;
+        text/vnd.wap.wml                      wml;
+        text/x-component                      htc;
+
+        image/avif                            avif;
+        image/png                             png;
+        image/svg+xml                         svg svgz;
+        image/tiff                            tif tiff;
+        image/vnd.wap.wbmp                    wbmp;
+        image/webp                            webp;
+        image/x-icon                          ico;
+        image/x-jng                           jng;
+        image/x-ms-bmp                        bmp;
+
+        font/woff                             woff;
+        font/woff2                            woff2;
+
+        application/java-archive              jar war ear;
+        application/json                      json;
+        application/mac-binhex40              hqx;
+        application/msword                    doc;
+        application/pdf                       pdf;
+        application/postscript                ps eps ai;
+        application/rtf                       rtf;
+        application/vnd.apple.mpegurl         m3u8;
+        application/vnd.google-earth.kml+xml  kml;
+        application/vnd.google-earth.kmz      kmz;
+        application/vnd.ms-excel              xls;
+        application/vnd.ms-fontobject         eot;
+        application/vnd.ms-powerpoint         ppt;
+        application/vnd.oasis.opendocument.graphics        odg;
+        application/vnd.oasis.opendocument.presentation    odp;
+        application/vnd.oasis.opendocument.spreadsheet     ods;
+        application/vnd.oasis.opendocument.text            odt;
+        application/vnd.openxmlformats-officedocument.presentationml.presentation    pptx;
+        application/vnd.openxmlformats-officedocument.spreadsheetml.sheet    xlsx;
+        application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;
+        application/vnd.wap.wmlc              wmlc;
+        application/wasm                      wasm;
+        application/x-7z-compressed           7z;
+        application/x-cocoa                   cco;
+        application/x-java-archive-diff       jardiff;
+        application/x-java-jnlp-file          jnlp;
+        application/x-makeself                run;
+        application/x-perl                    pl pm;
+        application/x-pilot                   prc pdb;
+        application/x-rar-compressed          rar;
+        application/x-redhat-package-manager  rpm;
+        application/x-sea                     sea;
+        application/x-shockwave-flash         swf;
+        application/x-stuffit                 sit;
+        application/x-tcl                     tcl tk;
+        application/x-x509-ca-cert            der pem crt;
+        application/x-xpinstall               xpi;
+        application/xhtml+xml                 xhtml;
+        application/xspf+xml                  xspf;
+        application/zip                       zip;
+
+        application/octet-stream              bin exe dll;
+        application/octet-stream              deb;
+        application/octet-stream              dmg;
+        application/octet-stream              iso img;
+        application/octet-stream              msi msp msm;
+
+        audio/midi                            mid midi kar;
+        audio/mpeg                            mp3;
+        audio/ogg                             ogg;
+        audio/x-m4a                           m4a;
+        audio/x-realaudio                     ra;
+
+        video/3gpp                            3gpp 3gp;
+        video/mp2t                            ts;
+        video/mp4                             mp4;
+        video/mpeg                            mpeg mpg;
+        video/quicktime                       mov;
+        video/webm                            webm;
+        video/x-flv                           flv;
+        video/x-m4v                           m4v;
+        video/x-mng                           mng;
+        video/x-ms-asf                        asx asf;
+        video/x-ms-wmv                        wmv;
+        video/x-msvideo                       avi;
+    }
+
+    default_type application/octet-stream;
+
+    access_log $NGINX_DIR/access.log;
+
+    limit_req_zone $binary_remote_addr zone=tenpersec:10m rate=100r/s;
+
+    upstream robosats_gunicorn_rest {
+        server 127.0.0.1:$GUNICORN_PORT;
+    }
+
+    upstream robosats_daphne_websocket {
+        server 127.0.0.1:$DAPHNE_PORT;
+    }
+
+    geo $allowed_localIP {
+        default 0;
+        # 127.0.0.1 1;
+        192.168.0.0/16 1;
+        # 192.168.x.x 1;
+    }
+
+    map $host $allowed_onion {
+        default 0;
+        # "$ONION_LOCATION" 1;
+    }
+
+    server {
+        listen $RUNSERVER_PORT;
+        server_name robosats.com;
+        large_client_header_buffers 4 64k;
+
+        location / {
+            root $NGINX_DIR/root;
+            try_files $uri $uri/ /index.html =404;
+            # try_files /index.html =404;
+        }
+
+        location /static {
+            set $allow_access 0;
+            if ($allowed_localIP = 1) {
+                set $allow_access 1;
+            }
+            if ($allow_access = 0){
+                return 403;
+            }
+
+            alias $NGINX_DIR/static;
+        }
+
+        # tor to web providers (identification files)
+        location /.well-known {
+            alias $NGINX_DIR/well-known;
+        }
+
+        location /api/ {
+            # requests are passed to gunicorn
+            proxy_pass http://robosats_gunicorn_rest;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $host;
+            proxy_redirect off;
+            add_header Onion-Location $ONION_LOCATION$request_uri;
+            limit_req zone=tenpersec burst=10;
+        }
+
+        location /coordinator {
+            set $allow_access 0;
+            if ($allowed_localIP = 1) {
+                set $allow_access 1;
+            }
+            if ($allowed_onion = 1) {
+                set $allow_access 1;
+            }
+            if ($allow_access = 0){
+                return 403;
+            }
+
+            proxy_pass http://robosats_gunicorn_rest;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $host;
+            proxy_redirect off;
+            add_header Onion-Location $ONION_LOCATION$request_uri;
+        }
+
+        location /ws/ {
+            # websockets are passed to daphne
+            proxy_pass http://robosats_daphne_websocket;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "Upgrade";
+            proxy_set_header Host $host;
+            limit_req zone=tenpersec burst=10;
+        }
+
+        location /relay {
+            proxy_pass http://127.0.0.1:$STRFRY_PORT;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "Upgrade";
+            proxy_set_header Host $host;
+        }
+
+        location = /favicon.ico {
+            alias $NGINX_DIR/favicon.ico;
+            access_log off;
+            log_not_found off;
+        }
+    }
+}

scripts/traditional/traditional-services

@@ -57,6 +57,7 @@ _services_environment_set() {
 
     POSTGRES_DIR="$TRADITIONAL_SERVICES_DIR/postgres"
     REDIS_DIR="$TRADITIONAL_SERVICES_DIR/redis"
+    NGINX_DIR="$TRADITIONAL_SERVICES_DIR/nginx"
     STRFRY_DIR="$TRADITIONAL_SERVICES_DIR/strfry"
     GNUPG_DIR="$(_get_env_var_path "GNUPG_DIR")" || return "$?"
 
@@ -76,6 +77,8 @@ _services_environment_set() {
 
     STRFRY_GIT_DIR="$(_get_env_var_path "STRFRY_GIT_DIR")" || return "$?"
 
+    DAPHNE_PORT="$(_get_env_var "DAPHNE_PORT")" || return "$?"
+    GUNICORN_PORT="$(_get_env_var "GUNICORN_PORT")" || return "$?"
     RUNSERVER_PORT="$(_get_env_var "RUNSERVER_PORT")" || return "$?"
     STRFRY_PORT="$(_get_env_var "STRFRY_PORT")" || return "$?"
 
@@ -90,8 +93,11 @@ _services_environment_set() {
 
     TRADITIONAL_DIR="$(realpath "$(dirname "$0")")"
 
+    NGINX_CONF="$NGINX_DIR/nginx.conf"
     STRFRY_CONF="$STRFRY_DIR/strfry.conf"
 
+    gunicorn_workers=2
+
     strfry_bin="$STRFRY_GIT_DIR/strfry"
 
     # to prevent having same command names and killing wrong programs
@@ -100,16 +106,60 @@ _services_environment_set() {
 
     # docker-compose.yml
 
+    DAPHNE_COMMAND="daphne -b 0.0.0.0 -p $DAPHNE_PORT robosats.asgi:application"
+    GUNICORN_COMMAND="gunicorn --bind :$GUNICORN_PORT --max-requests 1000 --max-requests-jitter 200 -w $gunicorn_workers robosats.wsgi:application"
+    # -e stderr otherwise nginx complains that it can not open default root log file
+    NGINX_COMMAND="nginx -c $NGINX_CONF -p $NGINX_DIR -e stderr"
     RUNSERVER_COMMAND="$python_bin manage.py runserver 0.0.0.0:$RUNSERVER_PORT"
 
     CLEAN_ORDERS_COMMAND="$python_bin manage.py clean_orders"
     FOLLOW_INVOICES_COMMAND="$python_bin manage.py follow_invoices"
-    # TELEGRAM_WATCHER_COMMAND="$python_bin manage.py telegram_watcher"
+    TELEGRAM_WATCHER_COMMAND="$python_bin manage.py telegram_watcher"
 
+    CELERY_PROD_COMMAND="$celery_bin -A robosats worker --loglevel=WARNING"
     CELERY_DEV_COMMAND="$celery_bin -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000"
     CELERY_BEAT_COMMAND="$celery_bin -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler"
 
     STRFRY_RELAY_COMMAND="$strfry_bin --config $STRFRY_CONF relay"
+    STRFRY_SYNC_FEDERATION_COMMAND="$TRADITIONAL_DIR/strfry-sync-federation"
+    STRFRY_SYNC_EXTERNAL_COMMAND="$TRADITIONAL_DIR/strfry-sync-external"
+}
+
+nginx_setup() {
+    if [ -e "$NGINX_CONF" ]; then
+        echo "nginx conf $NGINX_CONF already exists"
+        return 0
+    fi
+
+    ONION_LOCATION="$(_get_env_var "ONION_LOCATION")" || return "$?"
+
+    _create_dir "$NGINX_DIR" "0700" || return "$?"
+    _create_dir "$NGINX_DIR/root" || return "$?"
+    _create_dir "$NGINX_DIR/static" || return "$?"
+    _create_dir "$NGINX_DIR/well-known" || return "$?"
+    _create_dir "$NGINX_DIR/temp" || return "$?"
+
+    cp "$TRADITIONAL_DIR/templates/nginx.conf" "$NGINX_CONF"
+
+    sed -i "s/\$ONION_LOCATION/$ONION_LOCATION/g" "$NGINX_CONF"
+    sed -i "s|\$NGINX_DIR|$NGINX_DIR|g" "$NGINX_CONF"
+    sed -i "s/\$GUNICORN_PORT/$GUNICORN_PORT/g" "$NGINX_CONF"
+    sed -i "s/\$DAPHNE_PORT/$DAPHNE_PORT/g" "$NGINX_CONF"
+    sed -i "s/\$RUNSERVER_PORT/$RUNSERVER_PORT/g" "$NGINX_CONF"
+    sed -i "s/\$STRFRY_PORT/$STRFRY_PORT/g" "$NGINX_CONF"
+
+    cp "$TRADITIONAL_DIR/templates/index.html" "$NGINX_DIR/root"
+
+    cp -r "$(
+        find "$VIRTUAL_ENV" \
+            -type d \
+            -wholename "*site-packages/django/contrib/admin/static/admin"
+    )" "$NGINX_DIR/static"
+
+    # copy favicon into $NGINX_DIR/favicon.ico
+    # copy custom index.html in $NGINX_DIR/root/index.html
+
+    echo "nginx directory set up"
 }
 
 strfry_setup() {
@@ -128,6 +178,12 @@ strfry_setup() {
     echo "strfry directory set up"
 }
 
+git_setup() {
+    git rev-parse HEAD > commit_sha
+
+    echo "git commit_sha set up"
+}
+
 postgres_action() {
     if [ "$#" -lt 1 ]; then
         echo "error: insert postgres action" >&2
@@ -136,7 +192,7 @@ postgres_action() {
     action="$1"
     shift 1
     case "$action" in
-        setup|database) ;;
+        setup|database|database-production) ;;
         *)
             echo "error: wrong action" >&2
             return 1
@@ -171,7 +227,7 @@ port = $POSTGRES_PORT
 unix_socket_directories = '$POSTGRES_DIR'
 EOF
         ;;
-        database)
+        database|database-production)
             if [ ! -d "$POSTGRES_DIR" ]; then
                 printf "%s%s\n" \
                     "error: $POSTGRES_DIR is not a directory, " \
@@ -207,7 +263,7 @@ ALTER ROLE $POSTGRES_USER CREATEDB;
 EOF
             )
         ;;
-        database)
+        database|database-production)
             psql_stdin=$(cat << EOF
 CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
 EOF
@@ -221,7 +277,7 @@ EOF
     sleep 5
 
     case "$action" in
-        database)
+        database|database-production)
             if ! DJANGO_SUPERUSER_USERNAME="$(_get_env_var "ESCROW_USERNAME")"; then
                 _postgres_shut_down || return "$?"
                 return 1
@@ -232,16 +288,25 @@ EOF
                 return 1
             fi
 
-            # shellcheck disable=SC2034
+            if [ "$action" = "database" ]; then
-            DJANGO_SUPERUSER_PASSWORD="password"
+                # shellcheck disable=SC2034
-            DJANGO_SUPERUSER_EMAIL="superuser@email.com"
+                DJANGO_SUPERUSER_PASSWORD="password"
-            if ! python3 manage.py createsuperuser \
+                DJANGO_SUPERUSER_EMAIL="superuser@email.com"
-                --noinput \
+                if ! python3 manage.py createsuperuser \
-                --username "$DJANGO_SUPERUSER_USERNAME" \
+                    --noinput \
-                --email "$DJANGO_SUPERUSER_EMAIL"
+                    --username "$DJANGO_SUPERUSER_USERNAME" \
-            then
+                    --email "$DJANGO_SUPERUSER_EMAIL"
-                _postgres_shut_down || return "$?"
+                then
-                return 1
+                    _postgres_shut_down || return "$?"
+                    return 1
+                fi
+            elif [ "$action" = "database-production" ]; then
+                if ! python3 manage.py createsuperuser \
+                    --username "$DJANGO_SUPERUSER_USERNAME"
+                then
+                    _postgres_shut_down || return "$?"
+                    return 1
+                fi
             fi
         ;;
     esac
@@ -255,13 +320,20 @@ cleanup_signal() {
     printf "\n"
     printf "%s\n" "Caught $1 signal, sending it to services..."
 
+    pkill -"$2" -f "$STRFRY_SYNC_EXTERNAL_COMMAND"
+    pkill -"$2" -f "$STRFRY_SYNC_FEDERATION_COMMAND"
     pkill -"$2" -f "$STRFRY_RELAY_COMMAND"
     pkill -"$2" -f "$CELERY_BEAT_COMMAND"
-    pkill -"$2" -f "$CELERY_DEV_COMMAND"
+    if [ -n "$CELERY_COMMAND" ]; then
+        pkill -"$2" -f "$CELERY_COMMAND"
+    fi
     pkill -"$2" -f "$FOLLOW_INVOICES_COMMAND"
-    # pkill -"$2" -f "$TELEGRAM_WATCHER_COMMAND"
+    pkill -"$2" -f "$TELEGRAM_WATCHER_COMMAND"
     pkill -"$2" -f "$CLEAN_ORDERS_COMMAND"
     pkill -"$2" -f "$RUNSERVER_COMMAND"
+    pkill -"$2" -f "$DAPHNE_COMMAND"
+    pkill -"$2" -f "$GUNICORN_COMMAND"
+    pkill -"$2" -f "$NGINX_COMMAND"
     pkill -"$2" -f "redis-server \*:${REDIS_PORT}"
     pkill -"$2" -f "postgres -D $POSTGRES_DIR"
 
@@ -272,13 +344,20 @@ cleanup_int() {
     printf "\n"
     printf "%s\n" "Caught INT signal, shutting down services..."
 
+    pkill -TERM -f "$STRFRY_SYNC_EXTERNAL_COMMAND"
+    pkill -TERM -f "$STRFRY_SYNC_FEDERATION_COMMAND"
     pkill -TERM -f "$STRFRY_RELAY_COMMAND"
     pkill -INT -f "$CELERY_BEAT_COMMAND"
-    pkill -INT -f "$CELERY_DEV_COMMAND"
+    if [ -n "$CELERY_COMMAND" ]; then
+        pkill -INT -f "$CELERY_COMMAND"
+    fi
     pkill -TERM -f "$FOLLOW_INVOICES_COMMAND"
-    # pkill -TERM -f "$TELEGRAM_WATCHER_COMMAND"
+    pkill -TERM -f "$TELEGRAM_WATCHER_COMMAND"
     pkill -TERM -f "$CLEAN_ORDERS_COMMAND"
     pkill -TERM -f "$RUNSERVER_COMMAND"
+    pkill -TERM -f "$DAPHNE_COMMAND"
+    pkill -INT -f "$GUNICORN_COMMAND"
+    pkill -QUIT -f "$NGINX_COMMAND"
     pkill -INT -f "redis-server \*:${REDIS_PORT}"
     pkill -INT -f "postgres -D $POSTGRES_DIR"
 
@@ -293,7 +372,7 @@ main_loop() {
     action="$1"
     shift 1
     case "$action" in
-        test|server) ;;
+        test|server|production) ;;
         *)
             echo "error: $1 is invalid" >&2
             return 1
@@ -306,7 +385,18 @@ main_loop() {
     if ! _command_exist redis-server; then
         return 1
     fi
-    if [ "$action" = "server" ]; then
+    if [ "$action" = "server" ] || [ "$action" = "production" ]; then
+        if [ "$action" = "production" ]; then
+            if ! _command_exist daphne; then
+                return 1
+            fi
+            if ! _command_exist gunicorn; then
+                return 1
+            fi
+            if ! _command_exist nginx; then
+                return 1
+            fi
+        fi
         if ! _command_exist celery; then
             return 1
         fi
@@ -319,7 +409,16 @@ main_loop() {
             >&2
         return 1
     fi
-    if [ "$action" = "server" ]; then
+    if [ "$action" = "server" ] || [ "$action" = "production" ]; then
+        if [ "$action" = "production" ]; then
+            if [ ! -d "$NGINX_DIR" ]; then
+                printf "%s%s\n" \
+                    "error: $NGINX_DIR is not a directory, " \
+                    "should run setup" \
+                    >&2
+                return 1
+            fi
+        fi
         if [ ! -d "$STRFRY_DIR" ]; then
             printf "%s%s\n" \
                 "error: $STRFRY_DIR is not a directory, " \
@@ -333,9 +432,14 @@ main_loop() {
         ! pgrep -a lightningd >/dev/null 2>&1 &&
         ! pgrep -a lnd >/dev/null 2>&1
     }; then
+        if [ "$action" = "production" ]; then
+            btc_lnc_suggestion="start them before running this script"
+        else
+            btc_lnc_suggestion="make sure to run this script after running regtest-nodes"
+        fi
         printf "%s%s\n" \
             "error: bitcoin or lightning not running, " \
-            "make sure to run this script after running regtest-nodes" \
+            "$btc_lnc_suggestion" \
             >&2
         return 1
     fi
@@ -343,6 +447,12 @@ main_loop() {
     _create_dir "$REDIS_DIR" || return "$?"
     _create_dir "$GNUPG_DIR" "0700" || return "$?"
 
+    if [ "$action" = "server" ]; then
+        CELERY_COMMAND="$CELERY_DEV_COMMAND"
+    elif [ "$action" = "production" ]; then
+        CELERY_COMMAND="$CELERY_PROD_COMMAND"
+    fi
+
     trap "cleanup_signal HUP" HUP
     trap "cleanup_signal QUIT" QUIT
     trap "cleanup_signal TERM" TERM
@@ -363,11 +473,51 @@ main_loop() {
             redis-server - >> "$TRADITIONAL_LOGS_DIR/redis" 2>&1 &
         fi
 
-        if [ "$action" = "server" ]; then
+        if [ "$action" = "server" ] || [ "$action" = "production" ]; then
-            if ! pgrep -f "$RUNSERVER_COMMAND" >/dev/null; then
+            if [ "$action" = "server" ]; then
-                echo "starting runserver"
+                if ! pgrep -f "$RUNSERVER_COMMAND" >/dev/null; then
-                $RUNSERVER_COMMAND \
+                    echo "starting runserver"
-                    >> "$TRADITIONAL_LOGS_DIR/runserver" 2>&1 &
+                    $RUNSERVER_COMMAND \
+                        >> "$TRADITIONAL_LOGS_DIR/runserver" 2>&1 &
+                fi
+            elif [ "$action" = "production" ]; then
+                if ! pgrep -f "$DAPHNE_COMMAND" >/dev/null; then
+                    echo "starting daphne"
+                    $DAPHNE_COMMAND \
+                        >> "$TRADITIONAL_LOGS_DIR/daphne" 2>&1 &
+                fi
+
+                if ! pgrep -f "$GUNICORN_COMMAND" >/dev/null; then
+                    echo "starting gunicorn"
+                    $GUNICORN_COMMAND \
+                        >> "$TRADITIONAL_LOGS_DIR/gunicorn" 2>&1 &
+                fi
+
+                if ! pgrep -f "$NGINX_COMMAND" >/dev/null; then
+                    echo "starting nginx"
+                    $NGINX_COMMAND -g 'daemon off; master_process on;' \
+                        >> "$TRADITIONAL_LOGS_DIR/nginx" 2>&1 &
+                fi
+
+                if ! pgrep -f "$STRFRY_SYNC_FEDERATION_COMMAND" >/dev/null; then
+                    echo "starting strfry sync federation"
+                    $STRFRY_SYNC_FEDERATION_COMMAND \
+                        >> "$TRADITIONAL_LOGS_DIR/strfry-sync-federation" 2>&1 &
+                fi
+
+                if ! pgrep -f "$STRFRY_SYNC_EXTERNAL_COMMAND" >/dev/null; then
+                    echo "starting strfry sync external"
+                    $STRFRY_SYNC_EXTERNAL_COMMAND \
+                        >> "$TRADITIONAL_LOGS_DIR/strfry-sync-external" 2>&1 &
+                fi
+
+                if _get_env_var "TELEGRAM_TOKEN" >/dev/null 2>&1; then
+                    if ! pgrep -f "$TELEGRAM_WATCHER_COMMAND" >/dev/null; then
+                        echo "starting telegram-watcher"
+                        $TELEGRAM_WATCHER_COMMAND \
+                            >> "$TRADITIONAL_LOGS_DIR/telegram-watcher" 2>&1 &
+                    fi
+                fi
             fi
 
             if ! pgrep -f "$STRFRY_RELAY_COMMAND" >/dev/null; then
@@ -388,17 +538,9 @@ main_loop() {
                     >> "$TRADITIONAL_LOGS_DIR/follow-invoices" 2>&1 &
             fi
 
-            # if _get_env_var "TELEGRAM_TOKEN" >/dev/null 2>&1; then
+            if ! pgrep -f "$CELERY_COMMAND" >/dev/null; then
-            #     if ! pgrep -f "$TELEGRAM_WATCHER_COMMAND" >/dev/null; then
-            #         echo "starting telegram-watcher"
-            #         $TELEGRAM_WATCHER_COMMAND \
-            #             >> "$TRADITIONAL_LOGS_DIR/telegram-watcher" 2>&1 &
-            #     fi
-            # fi
-
-            if ! pgrep -f "$CELERY_DEV_COMMAND" >/dev/null; then
                 echo "starting celery worker"
-                $CELERY_DEV_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-worker" 2>&1 &
+                $CELERY_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-worker" 2>&1 &
             fi
 
             if ! pgrep -f "$CELERY_BEAT_COMMAND" >/dev/null; then
@@ -434,10 +576,12 @@ traditional-services action
 
 postgres-setup
 postgres-database
+postgres-database-production
 strfry-setup
 nginx-setup
 test
 server
+production
 EOF
             return 0
         ;;
@@ -452,10 +596,19 @@ EOF
         postgres-database)
             postgres_action "database"
         ;;
+        postgres-database-production)
+            postgres_action "database-production"
+        ;;
         strfry-setup)
             strfry_setup
         ;;
-        test|server)
+        nginx-setup)
+            nginx_setup
+        ;;
+        # git-setup)
+        #     git_setup
+        # ;;
+        test|server|production)
             main_loop "$action"
         ;;
         *)