+robosats coordinator +
++available at /api +
+ + diff --git a/scripts/traditional/templates/nginx.conf b/scripts/traditional/templates/nginx.conf new file mode 100644 index 00000000..25cdb5a6 --- /dev/null +++ b/scripts/traditional/templates/nginx.conf @@ -0,0 +1,236 @@ +# adapted from: +# * debian /etc/nginx/nginx.conf +# * robosats-deploy/compose/nginx/mn.conf.d/local.conf + +# user robosats; +worker_processes auto; +pid $NGINX_DIR/nginx.pid; +error_log $NGINX_DIR/error.log; + +events { + worker_connections 1024; + # multi_accept on; +} + +http { + client_body_temp_path $NGINX_DIR/temp/client_body; + proxy_temp_path $NGINX_DIR/temp/proxy; + fastcgi_temp_path $NGINX_DIR/temp/fastcgi; + uwsgi_temp_path $NGINX_DIR/temp/uwsgi; + scgi_temp_path $NGINX_DIR/temp/scgi; + + sendfile on; + tcp_nopush on; + types_hash_max_size 2048; + server_tokens off; + + # nginx: [emerg] could not build map_hash, you should increase map_hash_bucket_size: 64 + map_hash_bucket_size 4096; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/avif avif; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + + font/woff woff; + font/woff2 woff2; + + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.wap.wmlc wmlc; + application/wasm wasm; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; + } + + default_type application/octet-stream; + + access_log $NGINX_DIR/access.log; + + limit_req_zone $binary_remote_addr zone=tenpersec:10m rate=100r/s; + + upstream robosats_gunicorn_rest { + server 127.0.0.1:$GUNICORN_PORT; + } + + upstream robosats_daphne_websocket { + server 127.0.0.1:$DAPHNE_PORT; + } + + geo $allowed_localIP { + default 0; + # 127.0.0.1 1; + 192.168.0.0/16 1; + # 192.168.x.x 1; + } + + map $host $allowed_onion { + default 0; + # "$ONION_LOCATION" 1; + } + + server { + listen $RUNSERVER_PORT; + server_name robosats.com; + large_client_header_buffers 4 64k; + + location / { + root $NGINX_DIR/root; + try_files $uri $uri/ /index.html =404; + # try_files /index.html =404; + } + + location /static { + set $allow_access 0; + if ($allowed_localIP = 1) { + set $allow_access 1; + } + if ($allow_access = 0){ + return 403; + } + + alias $NGINX_DIR/static; + } + + # tor to web providers (identification files) + location /.well-known { + alias $NGINX_DIR/well-known; + } + + location /api/ { + # requests are passed to gunicorn + proxy_pass http://robosats_gunicorn_rest; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + add_header Onion-Location $ONION_LOCATION$request_uri; + limit_req zone=tenpersec burst=10; + } + + location /coordinator { + set $allow_access 0; + if ($allowed_localIP = 1) { + set $allow_access 1; + } + if ($allowed_onion = 1) { + set $allow_access 1; + } + if ($allow_access = 0){ + return 403; + } + + proxy_pass http://robosats_gunicorn_rest; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + add_header Onion-Location $ONION_LOCATION$request_uri; + } + + location /ws/ { + # websockets are passed to daphne + proxy_pass http://robosats_daphne_websocket; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + limit_req zone=tenpersec burst=10; + } + + location /relay { + proxy_pass http://127.0.0.1:$STRFRY_PORT; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + } + + location = /favicon.ico { + alias $NGINX_DIR/favicon.ico; + access_log off; + log_not_found off; + } + } +} diff --git a/scripts/traditional/traditional-services b/scripts/traditional/traditional-services index c2e8b723..f27567f5 100755 --- a/scripts/traditional/traditional-services +++ b/scripts/traditional/traditional-services @@ -57,6 +57,7 @@ _services_environment_set() { POSTGRES_DIR="$TRADITIONAL_SERVICES_DIR/postgres" REDIS_DIR="$TRADITIONAL_SERVICES_DIR/redis" + NGINX_DIR="$TRADITIONAL_SERVICES_DIR/nginx" STRFRY_DIR="$TRADITIONAL_SERVICES_DIR/strfry" GNUPG_DIR="$(_get_env_var_path "GNUPG_DIR")" || return "$?" @@ -76,6 +77,8 @@ _services_environment_set() { STRFRY_GIT_DIR="$(_get_env_var_path "STRFRY_GIT_DIR")" || return "$?" + DAPHNE_PORT="$(_get_env_var "DAPHNE_PORT")" || return "$?" + GUNICORN_PORT="$(_get_env_var "GUNICORN_PORT")" || return "$?" RUNSERVER_PORT="$(_get_env_var "RUNSERVER_PORT")" || return "$?" STRFRY_PORT="$(_get_env_var "STRFRY_PORT")" || return "$?" @@ -90,8 +93,11 @@ _services_environment_set() { TRADITIONAL_DIR="$(realpath "$(dirname "$0")")" + NGINX_CONF="$NGINX_DIR/nginx.conf" STRFRY_CONF="$STRFRY_DIR/strfry.conf" + gunicorn_workers=2 + strfry_bin="$STRFRY_GIT_DIR/strfry" # to prevent having same command names and killing wrong programs @@ -100,16 +106,60 @@ _services_environment_set() { # docker-compose.yml + DAPHNE_COMMAND="daphne -b 0.0.0.0 -p $DAPHNE_PORT robosats.asgi:application" + GUNICORN_COMMAND="gunicorn --bind :$GUNICORN_PORT --max-requests 1000 --max-requests-jitter 200 -w $gunicorn_workers robosats.wsgi:application" + # -e stderr otherwise nginx complains that it can not open default root log file + NGINX_COMMAND="nginx -c $NGINX_CONF -p $NGINX_DIR -e stderr" RUNSERVER_COMMAND="$python_bin manage.py runserver 0.0.0.0:$RUNSERVER_PORT" CLEAN_ORDERS_COMMAND="$python_bin manage.py clean_orders" FOLLOW_INVOICES_COMMAND="$python_bin manage.py follow_invoices" - # TELEGRAM_WATCHER_COMMAND="$python_bin manage.py telegram_watcher" + TELEGRAM_WATCHER_COMMAND="$python_bin manage.py telegram_watcher" + CELERY_PROD_COMMAND="$celery_bin -A robosats worker --loglevel=WARNING" CELERY_DEV_COMMAND="$celery_bin -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000" CELERY_BEAT_COMMAND="$celery_bin -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" STRFRY_RELAY_COMMAND="$strfry_bin --config $STRFRY_CONF relay" + STRFRY_SYNC_FEDERATION_COMMAND="$TRADITIONAL_DIR/strfry-sync-federation" + STRFRY_SYNC_EXTERNAL_COMMAND="$TRADITIONAL_DIR/strfry-sync-external" +} + +nginx_setup() { + if [ -e "$NGINX_CONF" ]; then + echo "nginx conf $NGINX_CONF already exists" + return 0 + fi + + ONION_LOCATION="$(_get_env_var "ONION_LOCATION")" || return "$?" + + _create_dir "$NGINX_DIR" "0700" || return "$?" + _create_dir "$NGINX_DIR/root" || return "$?" + _create_dir "$NGINX_DIR/static" || return "$?" + _create_dir "$NGINX_DIR/well-known" || return "$?" + _create_dir "$NGINX_DIR/temp" || return "$?" + + cp "$TRADITIONAL_DIR/templates/nginx.conf" "$NGINX_CONF" + + sed -i "s/\$ONION_LOCATION/$ONION_LOCATION/g" "$NGINX_CONF" + sed -i "s|\$NGINX_DIR|$NGINX_DIR|g" "$NGINX_CONF" + sed -i "s/\$GUNICORN_PORT/$GUNICORN_PORT/g" "$NGINX_CONF" + sed -i "s/\$DAPHNE_PORT/$DAPHNE_PORT/g" "$NGINX_CONF" + sed -i "s/\$RUNSERVER_PORT/$RUNSERVER_PORT/g" "$NGINX_CONF" + sed -i "s/\$STRFRY_PORT/$STRFRY_PORT/g" "$NGINX_CONF" + + cp "$TRADITIONAL_DIR/templates/index.html" "$NGINX_DIR/root" + + cp -r "$( + find "$VIRTUAL_ENV" \ + -type d \ + -wholename "*site-packages/django/contrib/admin/static/admin" + )" "$NGINX_DIR/static" + + # copy favicon into $NGINX_DIR/favicon.ico + # copy custom index.html in $NGINX_DIR/root/index.html + + echo "nginx directory set up" } strfry_setup() { @@ -128,6 +178,12 @@ strfry_setup() { echo "strfry directory set up" } +git_setup() { + git rev-parse HEAD > commit_sha + + echo "git commit_sha set up" +} + postgres_action() { if [ "$#" -lt 1 ]; then echo "error: insert postgres action" >&2 @@ -136,7 +192,7 @@ postgres_action() { action="$1" shift 1 case "$action" in - setup|database) ;; + setup|database|database-production) ;; *) echo "error: wrong action" >&2 return 1 @@ -171,7 +227,7 @@ port = $POSTGRES_PORT unix_socket_directories = '$POSTGRES_DIR' EOF ;; - database) + database|database-production) if [ ! -d "$POSTGRES_DIR" ]; then printf "%s%s\n" \ "error: $POSTGRES_DIR is not a directory, " \ @@ -207,7 +263,7 @@ ALTER ROLE $POSTGRES_USER CREATEDB; EOF ) ;; - database) + database|database-production) psql_stdin=$(cat << EOF CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER; EOF @@ -221,7 +277,7 @@ EOF sleep 5 case "$action" in - database) + database|database-production) if ! DJANGO_SUPERUSER_USERNAME="$(_get_env_var "ESCROW_USERNAME")"; then _postgres_shut_down || return "$?" return 1 @@ -232,16 +288,25 @@ EOF return 1 fi - # shellcheck disable=SC2034 - DJANGO_SUPERUSER_PASSWORD="password" - DJANGO_SUPERUSER_EMAIL="superuser@email.com" - if ! python3 manage.py createsuperuser \ - --noinput \ - --username "$DJANGO_SUPERUSER_USERNAME" \ - --email "$DJANGO_SUPERUSER_EMAIL" - then - _postgres_shut_down || return "$?" - return 1 + if [ "$action" = "database" ]; then + # shellcheck disable=SC2034 + DJANGO_SUPERUSER_PASSWORD="password" + DJANGO_SUPERUSER_EMAIL="superuser@email.com" + if ! python3 manage.py createsuperuser \ + --noinput \ + --username "$DJANGO_SUPERUSER_USERNAME" \ + --email "$DJANGO_SUPERUSER_EMAIL" + then + _postgres_shut_down || return "$?" + return 1 + fi + elif [ "$action" = "database-production" ]; then + if ! python3 manage.py createsuperuser \ + --username "$DJANGO_SUPERUSER_USERNAME" + then + _postgres_shut_down || return "$?" + return 1 + fi fi ;; esac @@ -255,13 +320,20 @@ cleanup_signal() { printf "\n" printf "%s\n" "Caught $1 signal, sending it to services..." + pkill -"$2" -f "$STRFRY_SYNC_EXTERNAL_COMMAND" + pkill -"$2" -f "$STRFRY_SYNC_FEDERATION_COMMAND" pkill -"$2" -f "$STRFRY_RELAY_COMMAND" pkill -"$2" -f "$CELERY_BEAT_COMMAND" - pkill -"$2" -f "$CELERY_DEV_COMMAND" + if [ -n "$CELERY_COMMAND" ]; then + pkill -"$2" -f "$CELERY_COMMAND" + fi pkill -"$2" -f "$FOLLOW_INVOICES_COMMAND" - # pkill -"$2" -f "$TELEGRAM_WATCHER_COMMAND" + pkill -"$2" -f "$TELEGRAM_WATCHER_COMMAND" pkill -"$2" -f "$CLEAN_ORDERS_COMMAND" pkill -"$2" -f "$RUNSERVER_COMMAND" + pkill -"$2" -f "$DAPHNE_COMMAND" + pkill -"$2" -f "$GUNICORN_COMMAND" + pkill -"$2" -f "$NGINX_COMMAND" pkill -"$2" -f "redis-server \*:${REDIS_PORT}" pkill -"$2" -f "postgres -D $POSTGRES_DIR" @@ -272,13 +344,20 @@ cleanup_int() { printf "\n" printf "%s\n" "Caught INT signal, shutting down services..." + pkill -TERM -f "$STRFRY_SYNC_EXTERNAL_COMMAND" + pkill -TERM -f "$STRFRY_SYNC_FEDERATION_COMMAND" pkill -TERM -f "$STRFRY_RELAY_COMMAND" pkill -INT -f "$CELERY_BEAT_COMMAND" - pkill -INT -f "$CELERY_DEV_COMMAND" + if [ -n "$CELERY_COMMAND" ]; then + pkill -INT -f "$CELERY_COMMAND" + fi pkill -TERM -f "$FOLLOW_INVOICES_COMMAND" - # pkill -TERM -f "$TELEGRAM_WATCHER_COMMAND" + pkill -TERM -f "$TELEGRAM_WATCHER_COMMAND" pkill -TERM -f "$CLEAN_ORDERS_COMMAND" pkill -TERM -f "$RUNSERVER_COMMAND" + pkill -TERM -f "$DAPHNE_COMMAND" + pkill -INT -f "$GUNICORN_COMMAND" + pkill -QUIT -f "$NGINX_COMMAND" pkill -INT -f "redis-server \*:${REDIS_PORT}" pkill -INT -f "postgres -D $POSTGRES_DIR" @@ -293,7 +372,7 @@ main_loop() { action="$1" shift 1 case "$action" in - test|server) ;; + test|server|production) ;; *) echo "error: $1 is invalid" >&2 return 1 @@ -306,7 +385,18 @@ main_loop() { if ! _command_exist redis-server; then return 1 fi - if [ "$action" = "server" ]; then + if [ "$action" = "server" ] || [ "$action" = "production" ]; then + if [ "$action" = "production" ]; then + if ! _command_exist daphne; then + return 1 + fi + if ! _command_exist gunicorn; then + return 1 + fi + if ! _command_exist nginx; then + return 1 + fi + fi if ! _command_exist celery; then return 1 fi @@ -319,7 +409,16 @@ main_loop() { >&2 return 1 fi - if [ "$action" = "server" ]; then + if [ "$action" = "server" ] || [ "$action" = "production" ]; then + if [ "$action" = "production" ]; then + if [ ! -d "$NGINX_DIR" ]; then + printf "%s%s\n" \ + "error: $NGINX_DIR is not a directory, " \ + "should run setup" \ + >&2 + return 1 + fi + fi if [ ! -d "$STRFRY_DIR" ]; then printf "%s%s\n" \ "error: $STRFRY_DIR is not a directory, " \ @@ -333,9 +432,14 @@ main_loop() { ! pgrep -a lightningd >/dev/null 2>&1 && ! pgrep -a lnd >/dev/null 2>&1 }; then + if [ "$action" = "production" ]; then + btc_lnc_suggestion="start them before running this script" + else + btc_lnc_suggestion="make sure to run this script after running regtest-nodes" + fi printf "%s%s\n" \ "error: bitcoin or lightning not running, " \ - "make sure to run this script after running regtest-nodes" \ + "$btc_lnc_suggestion" \ >&2 return 1 fi @@ -343,6 +447,12 @@ main_loop() { _create_dir "$REDIS_DIR" || return "$?" _create_dir "$GNUPG_DIR" "0700" || return "$?" + if [ "$action" = "server" ]; then + CELERY_COMMAND="$CELERY_DEV_COMMAND" + elif [ "$action" = "production" ]; then + CELERY_COMMAND="$CELERY_PROD_COMMAND" + fi + trap "cleanup_signal HUP" HUP trap "cleanup_signal QUIT" QUIT trap "cleanup_signal TERM" TERM @@ -363,11 +473,51 @@ main_loop() { redis-server - >> "$TRADITIONAL_LOGS_DIR/redis" 2>&1 & fi - if [ "$action" = "server" ]; then - if ! pgrep -f "$RUNSERVER_COMMAND" >/dev/null; then - echo "starting runserver" - $RUNSERVER_COMMAND \ - >> "$TRADITIONAL_LOGS_DIR/runserver" 2>&1 & + if [ "$action" = "server" ] || [ "$action" = "production" ]; then + if [ "$action" = "server" ]; then + if ! pgrep -f "$RUNSERVER_COMMAND" >/dev/null; then + echo "starting runserver" + $RUNSERVER_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/runserver" 2>&1 & + fi + elif [ "$action" = "production" ]; then + if ! pgrep -f "$DAPHNE_COMMAND" >/dev/null; then + echo "starting daphne" + $DAPHNE_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/daphne" 2>&1 & + fi + + if ! pgrep -f "$GUNICORN_COMMAND" >/dev/null; then + echo "starting gunicorn" + $GUNICORN_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/gunicorn" 2>&1 & + fi + + if ! pgrep -f "$NGINX_COMMAND" >/dev/null; then + echo "starting nginx" + $NGINX_COMMAND -g 'daemon off; master_process on;' \ + >> "$TRADITIONAL_LOGS_DIR/nginx" 2>&1 & + fi + + if ! pgrep -f "$STRFRY_SYNC_FEDERATION_COMMAND" >/dev/null; then + echo "starting strfry sync federation" + $STRFRY_SYNC_FEDERATION_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/strfry-sync-federation" 2>&1 & + fi + + if ! pgrep -f "$STRFRY_SYNC_EXTERNAL_COMMAND" >/dev/null; then + echo "starting strfry sync external" + $STRFRY_SYNC_EXTERNAL_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/strfry-sync-external" 2>&1 & + fi + + if _get_env_var "TELEGRAM_TOKEN" >/dev/null 2>&1; then + if ! pgrep -f "$TELEGRAM_WATCHER_COMMAND" >/dev/null; then + echo "starting telegram-watcher" + $TELEGRAM_WATCHER_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/telegram-watcher" 2>&1 & + fi + fi fi if ! pgrep -f "$STRFRY_RELAY_COMMAND" >/dev/null; then @@ -388,17 +538,9 @@ main_loop() { >> "$TRADITIONAL_LOGS_DIR/follow-invoices" 2>&1 & fi - # if _get_env_var "TELEGRAM_TOKEN" >/dev/null 2>&1; then - # if ! pgrep -f "$TELEGRAM_WATCHER_COMMAND" >/dev/null; then - # echo "starting telegram-watcher" - # $TELEGRAM_WATCHER_COMMAND \ - # >> "$TRADITIONAL_LOGS_DIR/telegram-watcher" 2>&1 & - # fi - # fi - - if ! pgrep -f "$CELERY_DEV_COMMAND" >/dev/null; then + if ! pgrep -f "$CELERY_COMMAND" >/dev/null; then echo "starting celery worker" - $CELERY_DEV_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-worker" 2>&1 & + $CELERY_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-worker" 2>&1 & fi if ! pgrep -f "$CELERY_BEAT_COMMAND" >/dev/null; then @@ -434,10 +576,12 @@ traditional-services action postgres-setup postgres-database +postgres-database-production strfry-setup nginx-setup test server +production EOF return 0 ;; @@ -452,10 +596,19 @@ EOF postgres-database) postgres_action "database" ;; + postgres-database-production) + postgres_action "database-production" + ;; strfry-setup) strfry_setup ;; - test|server) + nginx-setup) + nginx_setup + ;; + # git-setup) + # git_setup + # ;; + test|server|production) main_loop "$action" ;; *)