From bff872130da7ff4492dd531b623d407ed1fce7e4 Mon Sep 17 00:00:00 2001 From: jerryfletcher21 Date: Sat, 17 May 2025 16:52:08 +0200 Subject: [PATCH] traditional environment --- .gitignore | 3 + api/nostr.py | 3 +- api/utils.py | 4 +- api/views.py | 4 + requirements_dev.txt | 3 +- scripts/traditional/README.md | 277 ++++ scripts/traditional/regtest-nodes | 1438 ++++++++++++++++++ scripts/traditional/robosats.bash-completion | 281 ++++ scripts/traditional/templates/strfry.conf | 138 ++ scripts/traditional/traditional-services | 468 ++++++ setup.md | 6 + tests/test_frontend_fetch.py | 5 + tests/utils/node.py | 31 +- tests/utils/pgp.py | 3 +- 14 files changed, 2649 insertions(+), 15 deletions(-) create mode 100644 scripts/traditional/README.md create mode 100755 scripts/traditional/regtest-nodes create mode 100755 scripts/traditional/robosats.bash-completion create mode 100644 scripts/traditional/templates/strfry.conf create mode 100755 scripts/traditional/traditional-services diff --git a/.gitignore b/.gitignore index 01e3d2d3..f57dee36 100755 --- a/.gitignore +++ b/.gitignore @@ -661,3 +661,6 @@ nodeapp/*.html # Protocol Buffers api/lightning/*.proto + +# Traditional environment +/traditional/ diff --git a/api/nostr.py b/api/nostr.py index 54c9ceb4..a471be2b 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -27,7 +27,8 @@ class Nostr: # Add relays and connect await client.add_relay("ws://localhost:7777") - await client.add_relay("ws://localhost:7778") + strfry_port = config("STRFRY_PORT", cast=str, default="7778") + await client.add_relay(f"ws://localhost:{strfry_port}") await client.connect() robot_name = await self.get_robot_name(order) diff --git a/api/utils.py b/api/utils.py index 83dfb25d..e74107e1 100644 --- a/api/utils.py +++ b/api/utils.py @@ -385,7 +385,7 @@ def compute_avg_premium(queryset): def validate_pgp_keys(pub_key, enc_priv_key): """Validates PGP valid keys. Formats them in a way understandable by the frontend""" - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) # Standardize format with linux linebreaks '\n'. Windows users submitting their own keys have '\r\n' breaking communication. enc_priv_key = enc_priv_key.replace("\r\n", "\n").replace("\\", "\n") @@ -439,7 +439,7 @@ def verify_signed_message(pub_key, signed_message): Verifies a signed cleartext PGP message. Returns whether the signature is valid (was made by the given pub_key) and the content of the message. """ - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) # import the public key import_result = gpg.import_keys(pub_key) diff --git a/api/views.py b/api/views.py index 31ffe73f..069303d1 100644 --- a/api/views.py +++ b/api/views.py @@ -16,6 +16,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from api.logics import Logics +from api.tasks import cache_market from api.models import ( Currency, LNPayment, @@ -150,6 +151,9 @@ class MakerView(CreateAPIView): status.HTTP_400_BAD_REQUEST, ) + if len(Currency.objects.all()) == 0: + cache_market() + # Creates a new order order = Order( type=type, diff --git a/requirements_dev.txt b/requirements_dev.txt index 930a8585..48bc97c2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,5 @@ coverage==7.8.0 ruff==0.11.10 git+https://github.com/Reckless-Satoshi/drf-openapi-tester.git@soften-django-requirements -pre-commit==4.2.0 \ No newline at end of file +pre-commit==4.2.0 +python-dotenv[cli]==1.1.0 diff --git a/scripts/traditional/README.md b/scripts/traditional/README.md new file mode 100644 index 00000000..f9707e0e --- /dev/null +++ b/scripts/traditional/README.md @@ -0,0 +1,277 @@ +# Robosats traditional environment + +Robosats backend development and testing without docker and containers. + +Binaries needed: +* postgresql (`postgres`, `initdb`, `psql`) +* redis (`redis-server`) +* bitcoin (`bitcoind`, `bitcoin-cli`) +* cln (`lightningd`, `lightning-cli`, `holdinvoice`) +* lnd (`lnd`, `lncli`) + +## Preparation + +Postgresql and redis can be found in all linux distros and bsds. + +Some distros do not put postgresql binaries in `PATH`, if this is the +case simply link them somewhere in your `PATH`. + +Example on debian: +``` +ln -sf /usr/lib/postgresql/16/bin/postgres ~/.local/bin/ +ln -sf /usr/lib/postgresql/16/bin/initdb ~/.local/bin/ +ln -sf /usr/lib/postgresql/16/bin/psql ~/.local/bin/ +``` + +Bitcoin nodes if not already installed need to be manually downloaded. +* bitcoin core binaries can be found here: https://bitcoincore.org/en/download +* cln binaries can be found here: https://github.com/ElementsProject/lightning/releases +* holdinvoice binary can be found here: https://github.com/daywalker90/holdinvoice/releases +* lnd binaries can be found here: https://github.com/lightningnetwork/lnd/releases + +Example preparation: +``` +$ python3 -m venv venv +$ . venv/bin/activate +$ pip install -r requirements_dev.txt +$ pip install -r requirements.txt + +$ mkdir traditional +$ mkdir traditional/programs +$ cd traditional/programs + +# if you do not have them already installed +$ mkdir bitcoin cln lnd +# download bitcoin, cln (and holdinvoice) and lnd binaries + +# follow https://github.com/hoytech/strfry#compile +$ git clone https://github.com/hoytech/strfry +$ cd strfry +$ git submodule update --init +$ git checkout 1.0.4 +$ make setup-golpe +$ make -j4 + +# if you want to use roboauto +# still in traditional/programs +$ git clone https://github.com/jerryfletcher21/roboauto +$ cd roboauto +$ pip install -r requirements.txt +$ pip install . +``` + +## env file + +``` +$ cp .env-sample .env +``` +Edit `.env`, both robosats and traditional scripts will read from there. + +Variables to change: +``` +LNVENDOR = "CLN" +# LNVENDOR = "LND" + +BITCOIND_RPCURL = "http://127.0.0.1:18443" +BITCOIND_RPCUSER = "test" +BITCOIND_RPCPASSWORD = "test" +CLN_DIR = "traditional/nodes/cln-coord/regtest/" +CLN_GRPC_HOST = "localhost:9999" +CLN_GRPC_HOLD_HOST = "localhost:9998" +LND_DIR = "traditional/nodes/lnd-coord/" +MACAROON_PATH = "data/chain/bitcoin/regtest/admin.macaroon" +LND_GRPC_HOST = "localhost:10009" +# POSTGRES_DB should not be postgres +POSTGRES_DB = "robosats" +# POSTGRES_USER should not be the same as $USER environment variable +POSTGRES_USER = "robosats" +POSTGRES_PASSWORD = "robosats" +USE_TOR = False +LOG_TO_CONSOLE = True +LOGGER_LEVEL = "INFO" +``` + +Variables to add: +``` +DEVELOPMENT = True +TESTING = True +SKIP_FRONTEND_TESTS = True +TIMING_EXTRA_IN_TESTS = True + +# LNVENDOR_USER is what robosats calls robot node in tests +LNVENDOR_USER = "LND" +# LNVENDOR_USER = "CLN" + +DAPHNE_PORT = 9000 +GUNICORN_PORT = 8080 +RUNSERVER_PORT = 8000 +STRFRY_PORT = 7778 + +BITCOIND_BIN = "traditional/programs/bitcoin/bin/bitcoind" +BITCOIN_CLI_BIN = "traditional/programs/bitcoin/bin/bitcoin-cli" +LIGHTNINGD_BIN = "traditional/programs/cln/bin/lightningd" +LIGHTNING_CLI_BIN = "traditional/programs/cln/bin/lightning-cli" +HOLDINVOICE_PLUGIN_BIN = "traditional/programs/cln/holdinvoice" +LND_BIN = "traditional/programs/lnd/lnd" +LNCLI_BIN = "traditional/programs/lnd/lncli" +STRFRY_GIT_DIR = "traditional/programs/strfry" +ROBOAUTO_GIT_DIR = "traditional/programs/roboauto" +TRADITIONAL_NODES_DIR = "traditional/nodes" +TRADITIONAL_SERVICES_DIR = "traditional/services" +TRADITIONAL_LOGS_DIR = "traditional/logs" +GNUPG_DIR = "traditional/services/gnupg" + +BITCOIN_TEST_ZMQ_BLOCK_PORT = 28432 +BITCOIN_TEST_ZMQ_TX_PORT = 28433 + +# CLN_GRPC_HOST and CLN_GRPC_HOLD_HOST are for coord +CLN_TEST_COORD_LISTEN_PORT = 19846 +CLN_TEST_USER_GRPC_PORT = 9989 +CLN_TEST_USER_LISTEN_PORT = 19836 + +# LND_GRPC_HOST is for coord +LND_TEST_COORD_LISTEN_PORT = 19746 +LND_TEST_COORD_REST_PORT = 8181 +LND_TEST_COORD_MACAROON_PATH = "traditional/nodes/lnd-coord/data/chain/bitcoin/regtest/admin.macaroon" + +LND_TEST_USER_GRPC_PORT = 10010 +LND_TEST_USER_LISTEN_PORT = 19736 +LND_TEST_USER_REST_PORT = 8182 +LND_TEST_USER_MACAROON_PATH = "traditional/nodes/lnd-user/data/chain/bitcoin/regtest/admin.macaroon" +``` + +Paths can be relative or absolute. Binaries should be paths, they are +not resolved with `PATH`. + +Roboauto can be disabled by not setting `ROBOAUTO_GIT_DIR` or setting it +to `false`. + +If some ports are already in use, change their value. + +To check which port are already in use, `netstat -tulnp` with root +privileges can be used. + +For example if there is alread an instance of postgresql running on the +default port, change `POSTGRES_PORT = "5433"`. + +## Usage + +For development and testing two scripts are provided: +* `traditional-services` for non bitcoin related services +* `regtest-nodes` for bitcoin and lightning nodes + +`traditional-services` sets up the database and manages the services. + +Everything is done locally, so no root privileges/service managers are +needed. + +`regtest-nodes` is a script that should be sourced, it sets up a regtest +environment, with bitcoin core, cln, lnd and roboauto, connecting them +and creating channels. It then exposes the functions `btc_reg`, +`cln_coord`, `cln_user`, `lnd_coord`, `lnd_user`, `ra_reg` to interact +with the nodes and roboauto. + +If the script is sourced in a `bash` shell, it will also source +completions for all the functions. + +`regtest-nodes` can also be run without arguments to simply expose the +functions to start and remove the nodes and to create the channels +between them, without setting up a specific environment. + +Setup: +``` +# change .env file + +$ . venv/bin/activate + +# generate cln and lnd grpc +$ scripts/generate_grpc.sh + +$ . scripts/traditional/regtest-nodes test + +$ scripts/traditional/traditional-services postgres-setup + +$ scripts/traditional/traditional-services postgres-database + +$ scripts/traditional/traditional-services strfry-setup + +# remove the nodes +$ robosats_regtest_stop_and_remove_all + +# postgres-database will create the database specified by +# POSTGRES_DB in .env, it can be run multiple times with +# different values of POSTGRES_DB for different databases +``` + +Testing: +``` +# edit .env setting LNVENDOR to either "CLN" or "LND" +# LNVENDOR_USER while running tests should be set to "LND" + +# in the main window +$ . venv/bin/activate +$ . scripts/traditional/regtest-nodes test + +# in a secondary window +$ . venv/bin/activate +# can be stopped with Control-C +$ scripts/traditional/traditional-services test + +# back in the main window +$ python3 manage.py test + +# after having run the tests run +$ robosats_regtest_stop_and_remove_all +# to remove the nodes, they will be recreated when +# running the tests again + +# python3 manage.py test can be run multiple times with the same database +# to have a clean database either use a different value +# of POSTGRES_DB or use a different directory (and run again the setup) +# by moving traditional/services/postgres somewhere and the moving it back when +# you want to use the old database again +``` + +Development: +``` +# edit .env setting LNVENDOR to either "CLN" or "LND" +# and LNVENDOR_USER to either "CLN" or "LND" + +# in the main window +$ . venv/bin/activate +$ . scripts/traditional/regtest-nodes server + +# in a secondary window +$ . venv/bin/activate +# can be stopped with Control-C +$ scripts/traditional/traditional-services server + +# to see the output of the django runserver command +# in a third window +$ tail -f traditional/logs/runserver + +# if roboauto is active, use it to test the backend +# back in the main window +$ ra_reg --help +... +$ ra_reg create-order "$(ra_reg generate-robot --loc)" type=buy currency=btc min_amount=0.001 max_amount=0.002 payment_method="On-Chain BTC" premium=-1.5 +... +$ ra_reg take-order $(ra_reg generate-robot --loc) order-id +... +$ ra_reg escrow-pay RobotName +... +``` + +Update: +``` +$ . venv/bin/activate + +$ pip install -r requirements_dev.txt +$ pip install -r requirements.txt + +# start just postgres and redis in an other window +$ scripts/traditional/traditional-services test + +# in the main window +$ python3 manage.py migrate +``` diff --git a/scripts/traditional/regtest-nodes b/scripts/traditional/regtest-nodes new file mode 100755 index 00000000..478c017a --- /dev/null +++ b/scripts/traditional/regtest-nodes @@ -0,0 +1,1438 @@ +#!/bin/sh + +_command_exist() { + if command -v "$1" >/dev/null 2>&1; then + return 0 + else + echo "error: $1 command not found" >&2 + return 1 + fi +} + +_function_exist() { + if command -V "$1" 2>/dev/null | grep -qwi function >/dev/null 2>&1; then + return 0 + else + echo "error: function $1 not set" >&2 + return 1 + fi +} + +_create_dir() { + if [ ! -e "$1" ]; then + mkdir -p "$1" || return "$?" + elif [ ! -d "$1" ]; then + echo "error: $1 is not a directory" >&2 + return 1 + fi +} + +# if $2 is provided use it as default +# otherwise return error when not found +_get_env_var() { + if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then + if [ "$#" -ge 2 ]; then + env_var="$2" + else + echo "error: getting $1 from .env" >&2 + return 1 + fi + fi + printf "%s\n" "$env_var" + + return 0 +} + +# transform relative path into absolute and remove trailing slashes +_get_env_var_path() { + env_var="$(_get_env_var "$@")" || return "$?" + real_path="$(realpath -m "$env_var")" || return "$?" + printf "%s\n" "$real_path" + + return 0 +} + +# extract the port from an env variable +_get_env_var_port() { + env_var="$(_get_env_var "$@")" || return "$?" + printf "%s\n" "$env_var" | + rev | + cut -d ":" -f 1 | + rev + + return 0 +} + +_nodes_environment_set() { + if [ ! -f "scripts/traditional/regtest-nodes" ]; then + echo "error: source this script from the project root" >&2 + return 1 + fi + if [ ! -f ".env" ]; then + echo "error: .env is not present" >&2 + return 1 + fi + if ! _command_exist dotenv; then + return 1 + fi + + TESTING="$(_get_env_var TESTING false)" || return "$?" + case "$(printf "%s\n" "$TESTING" | tr '[:upper:]' '[:lower:]')" in + true|yes|on|1) + TESTING=true + ;; + *) + TESTING=false + ;; + esac + + RUNSERVER_PORT="$(_get_env_var "RUNSERVER_PORT")" || return "$?" + + cln_set=false + lnd_set=false + + LNVENDOR_COORD="$(_get_env_var LNVENDOR)" || return "$?" + LNVENDOR_COORD="$(printf "%s\n" "$LNVENDOR_COORD" | tr '[:upper:]' '[:lower:]')" + case "$LNVENDOR_COORD" in + cln) cln_set=true ;; + lnd) lnd_set=true ;; + *) + echo "error: LNVENDOR can be cln or lnd" >&2 + return 1 + ;; + esac + LNVENDOR_USER="$(_get_env_var LNVENDOR_USER)" || return "$?" + LNVENDOR_USER="$(printf "%s\n" "$LNVENDOR_USER" | tr '[:upper:]' '[:lower:]')" + case "$LNVENDOR_USER" in + cln) cln_set=true ;; + lnd) lnd_set=true ;; + *) + echo "error: LNVENDOR_USER can be cln or lnd" >&2 + return 1 + ;; + esac + if [ "$TESTING" = true ] && [ "$LNVENDOR_USER" != "lnd" ]; then + echo "error: LNVENDOR_USER should be lnd when running tests" >&2 + return 1 + fi + + BITCOIND_BIN="$(_get_env_var_path "BITCOIND_BIN")" || return "$?" + BITCOIN_CLI_BIN="$(_get_env_var_path "BITCOIN_CLI_BIN")" || return "$?" + if [ "$cln_set" = true ]; then + LIGHTNINGD_BIN="$(_get_env_var_path "LIGHTNINGD_BIN")" || return "$?" + LIGHTNING_CLI_BIN="$(_get_env_var_path "LIGHTNING_CLI_BIN")" || return "$?" + HOLDINVOICE_PLUGIN_BIN="$(_get_env_var_path "HOLDINVOICE_PLUGIN_BIN")" || return "$?" + fi + if [ "$lnd_set" = true ]; then + LND_BIN="$(_get_env_var_path "LND_BIN")" || return "$?" + LNCLI_BIN="$(_get_env_var_path "LNCLI_BIN")" || return "$?" + fi + REGTEST_NODES_DIR="$(_get_env_var_path "TRADITIONAL_NODES_DIR")" || return "$?" + REGTEST_LOGS_DIR="$(_get_env_var_path "TRADITIONAL_LOGS_DIR")" || return "$?" + + ROBOAUTO_GIT_DIR="$(_get_env_var_path "ROBOAUTO_GIT_DIR" false)" || return "$?" + + BITCOIN_REGTEST_RPC_PORT="$(_get_env_var_port "BITCOIND_RPCURL")" || return "$?" + BITCOIN_REGTEST_RPC_USER="$(_get_env_var "BITCOIND_RPCUSER")" || return "$?" + BITCOIN_REGTEST_RPC_PASS="$(_get_env_var "BITCOIND_RPCPASSWORD")" || return "$?" + BITCOIN_REGTEST_ZMQ_BLOCK_PORT="$(_get_env_var "BITCOIN_TEST_ZMQ_BLOCK_PORT")" || return "$?" + BITCOIN_REGTEST_ZMQ_TX_PORT="$(_get_env_var "BITCOIN_TEST_ZMQ_TX_PORT")" || return "$?" + + if [ "$cln_set" = true ]; then + CLN_COORD_GRPC_PORT="$(_get_env_var_port "CLN_GRPC_HOST")" || return "$?" + CLN_COORD_HOLD_PORT="$(_get_env_var_port "CLN_GRPC_HOLD_HOST")" || return "$?" + CLN_COORD_LISTEN_PORT="$(_get_env_var "CLN_TEST_COORD_LISTEN_PORT")" || return "$?" + + CLN_USER_GRPC_PORT="$(_get_env_var "CLN_TEST_USER_GRPC_PORT")" || return "$?" + CLN_USER_LISTEN_PORT="$(_get_env_var "CLN_TEST_USER_LISTEN_PORT")" || return "$?" + fi + if [ "$lnd_set" = true ]; then + LND_COORD_RPC_PORT="$(_get_env_var_port "LND_GRPC_HOST")" || return "$?" + LND_COORD_LISTEN_PORT="$(_get_env_var "LND_TEST_COORD_LISTEN_PORT")" || return "$?" + LND_COORD_REST_PORT="$(_get_env_var "LND_TEST_COORD_REST_PORT")" || return "$?" + + LND_USER_RPC_PORT="$(_get_env_var "LND_TEST_USER_GRPC_PORT")" || return "$?" + LND_USER_LISTEN_PORT="$(_get_env_var "LND_TEST_USER_LISTEN_PORT")" || return "$?" + LND_USER_REST_PORT="$(_get_env_var "LND_TEST_USER_REST_PORT")" || return "$?" + fi + + if [ -z "$BITCOIND_BIN" ]; then + BITCOIND_BIN="bitcoind" + fi + if ! _command_exist "$BITCOIND_BIN"; then + return 1 + fi + if [ -z "$BITCOIN_CLI_BIN" ]; then + BITCOIN_CLI_BIN="bitcoin-cli" + fi + if ! _command_exist "$BITCOIN_CLI_BIN"; then + return 1 + fi + + if [ "$cln_set" = true ]; then + if [ -z "$LIGHTNINGD_BIN" ]; then + LIGHTNINGD_BIN="lightningd" + fi + if ! _command_exist "$LIGHTNINGD_BIN"; then + return 1 + fi + if [ -z "$LIGHTNING_CLI_BIN" ]; then + LIGHTNING_CLI_BIN="lightning-cli" + fi + if ! _command_exist "$LIGHTNING_CLI_BIN"; then + return 1 + fi + if [ -z "$HOLDINVOICE_PLUGIN_BIN" ]; then + echo "error: $HOLDINVOICE_PLUGIN_BIN not set" >&2 + return 1 + fi + if [ ! -f "$HOLDINVOICE_PLUGIN_BIN" ]; then + echo "error: $HOLDINVOICE_PLUGIN_BIN plugin not found" >&2 + return 1 + fi + fi + + if [ "$lnd_set" = true ]; then + if [ -z "$LND_BIN" ]; then + LND_BIN="lnd" + fi + if ! _command_exist "$LND_BIN"; then + return 1 + fi + if [ -z "$LNCLI_BIN" ]; then + LNCLI_BIN="lncli" + fi + if ! _command_exist "$LNCLI_BIN"; then + return 1 + fi + fi + + if [ -z "$REGTEST_NODES_DIR" ]; then + echo "error: REGTEST_NODES_DIR not set" >&2 + return 1 + fi + _create_dir "$REGTEST_NODES_DIR" || return "$?" + + if [ -z "$REGTEST_LOGS_DIR" ]; then + echo "error: REGTEST_LOGS_DIR not set" >&2 + return 1 + fi + _create_dir "$REGTEST_LOGS_DIR" || return "$?" + + BITCOIN_DIR="$REGTEST_NODES_DIR/bitcoin" + ROBOAUTO_DIR="$REGTEST_NODES_DIR/roboauto" + + if [ "$cln_set" = true ]; then + CLN_COORD_DIR="$REGTEST_NODES_DIR/cln-coord" + CLN_USER_DIR="$REGTEST_NODES_DIR/cln-user" + + cln_coor_dir_env="$(_get_env_var_path "CLN_DIR")" || return "$?" + if [ "$cln_coor_dir_env" != "$CLN_COORD_DIR/regtest" ]; then + echo "error: CLN_DIR not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + fi + + if [ "$lnd_set" = true ]; then + LND_COORD_DIR="$REGTEST_NODES_DIR/lnd-coord" + LND_USER_DIR="$REGTEST_NODES_DIR/lnd-user" + + lnd_coord_dir_env="$(_get_env_var_path "LND_DIR")" || return "$?" + if [ "$lnd_coord_dir_env" != "$LND_COORD_DIR" ]; then + echo "error: LND_DIR not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + + correct_macaroon_path="data/chain/bitcoin/regtest/admin.macaroon" + macaroon_path="$( + _get_env_var "MACAROON_PATH" + )" || return "$?" + if [ "$macaroon_path" != "$correct_macaroon_path" ]; then + echo "error: MACAROON_PATH should be $correct_macaroon_path" >&2 + return 1 + fi + + lnd_test_coord_macaroon_path="$( + _get_env_var_path "LND_TEST_COORD_MACAROON_PATH" + )" || return "$?" + if [ \ + "$lnd_test_coord_macaroon_path" != \ + "$LND_COORD_DIR/$correct_macaroon_path" \ + ]; then + echo "error: LND_TEST_COORD_MACAROON_PATH not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + + lnd_test_user_macaroon_path="$( + _get_env_var_path "LND_TEST_USER_MACAROON_PATH" + )" || return "$?" + if [ \ + "$lnd_test_user_macaroon_path" != \ + "$LND_USER_DIR/$correct_macaroon_path" \ + ]; then + echo "error: LND_TEST_USER_MACAROON_PATH not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + fi +} + +_pgrep_bitcoin_regtest() { + pgrep -f "$BITCOIND_BIN -datadir=$BITCOIN_DIR -regtest -daemon" >/dev/null +} + +_bitcoin_regtest_check_started() { + if ! _pgrep_bitcoin_regtest; then + echo "error: bitcoin regtest not started" >&2 + return 1 + fi + if ! _function_exist btc_reg; then + return 1 + fi + return 0 +} + +bitcoin_regtest_mine() { + _bitcoin_regtest_check_started || return "$?" + + if [ "$#" -lt 2 ]; then + return 1 + fi + if [ "$2" = "new" ]; then + if ! new_address="$(btc_reg getnewaddress)"; then + echo "error: could not create new address" >&2 + return 1 + fi + else + new_address="$2" + fi + if ! new_block_hash="$( + btc_reg -named generatetoaddress nblocks="$1" address="$new_address" | + jq -r '.[-1]' + )"; then + echo "error: could not generate to $new_address" >&2 + return 1 + fi + while [ "$(btc_reg getbestblockhash)" != "$new_block_hash" ]; do + echo "waiting for $1 blocks to be mined..." + sleep 1 + done + unset new_block_hash + echo "mined $1 blocks to $new_address" + unset new_address +} + +bitcoin_regtest_start() { + _create_dir "$BITCOIN_DIR" || return "$?" + + echo "writing bitcoin regtest config" + cat << EOF > "$BITCOIN_DIR/bitcoin.conf" +txindex=1 +rest=1 +server=1 +logips=1 +listenonion=0 +zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT +[regtest] +rpcport=$BITCOIN_REGTEST_RPC_PORT +rpcuser=$BITCOIN_REGTEST_RPC_USER +rpcpassword=$BITCOIN_REGTEST_RPC_PASS +EOF + + if ! _pgrep_bitcoin_regtest; then + echo "starting bitcoin regtest" + if ! "$BITCOIND_BIN" -datadir="$BITCOIN_DIR" -regtest -daemon >/dev/null; then + echo "error: starting bitcoind regtest daemon" >&2 + return 1 + fi + else + echo "bitcoin regtest already started" + fi + + btc_reg() { + "$BITCOIN_CLI_BIN" \ + -datadir="$BITCOIN_DIR" \ + -regtest \ + -rpcuser="$BITCOIN_REGTEST_RPC_USER" \ + -rpcpassword="$BITCOIN_REGTEST_RPC_PASS" \ + "$@" + } + + # wait for bitcoin to start + while ! btc_reg ping >/dev/null 2>&1; do + echo "waiting for bitcoind regtes to start..." + sleep 1 + done + + # check if default wallet exists + if + ! btc_reg listwalletdir | + jq -r '.wallets[] | .name' | + grep "^default$" >/dev/null 2>&1 + then + if ! btc_reg -named createwallet \ + wallet_name=default \ + disable_private_keys=false \ + blank=false \ + avoid_reuse=false \ + descriptors=true \ + load_on_startup=true \ + external_signer=false >/dev/null + then + echo "error: could not create default wallet" >&2 + return 1 + fi + echo "default bitcoind wallet successfully created" + fi + + # check if default wallet is loaded + if + ! btc_reg listwallets | + jq -r '.[]' | + grep "^default$" >/dev/null 2>&1 + then + if ! btc_reg loadwallet default; then + echo "error: could not load default wallet" >&2 + return 1 + fi + echo "default bitcoind wallet successfully loaded" + fi + + bitcoin_regtest_mine 1 "new" || return "$?" + while [ "$(btc_reg getblockchaininfo | jq -r '.initialblockdownload')" != false ]; do + echo "waiting for bitcoind regtes to download blocks..." + sleep 1 + done + + echo "bitcoin regtest started, data directory is $BITCOIN_DIR" + echo "btc_reg function set" +} + +_bitcoin_regtest_stop() { + if ! _pgrep_bitcoin_regtest; then + echo "bitcoin regtest already stopped" + else + if + ! "$BITCOIN_CLI_BIN" \ + -datadir="$BITCOIN_DIR" \ + -regtest \ + stop >/dev/null + then + echo "error: bitcoin regtest not stopped" >&2 + fi + + while _pgrep_bitcoin_regtest; do + echo "waiting for bitcoin regtest to stop..." + sleep 1 + done + echo "bitcoin regtest stopped" + fi +} + +bitcoin_regtest_stop_and_remove() { + _bitcoin_regtest_stop || return "$?" + + if [ ! -e "$BITCOIN_DIR" ]; then + echo "bitcoin directory not present" + else + if ! rm -rf "$BITCOIN_DIR"; then + echo "error: removing bitcoin directory $BITCOIN_DIR" + return 1 + fi + echo "removed bitcoin directory $BITCOIN_DIR" + fi +} + +_pgrep_cln_coord() { + pgrep -f \ + "$LIGHTNINGD_BIN --lightning-dir=$CLN_COORD_DIR --regtest --daemon" \ + >/dev/null +} + +cln_coord_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$CLN_COORD_DIR" || return "$?" + + echo "writing cln coordinator config" + cat << EOF > "$CLN_COORD_DIR/config" +bitcoin-cli=$BITCOIN_CLI_BIN +bitcoin-datadir=$BITCOIN_DIR +bitcoin-rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoin-rpcpassword=$BITCOIN_REGTEST_RPC_PASS +bitcoin-rpcport=$BITCOIN_REGTEST_RPC_PORT +alias=cln-coordinator +rgb=100000 +fee-base=0 +fee-per-satoshi=0 +min-capacity-sat=990000 +ignore-fee-limits=true +funding-confirms=1 +max-concurrent-htlcs=64 +max-dust-htlc-exposure-msat=1000000000 +cltv-delta=144 +cltv-final=144 +bitcoin-retry-timeout=120 +database-upgrade=true +log-file=lightning.log +addr=localhost:$CLN_COORD_LISTEN_PORT +grpc-port=$CLN_COORD_GRPC_PORT +important-plugin=$HOLDINVOICE_PLUGIN_BIN +grpc-hold-port=$CLN_COORD_HOLD_PORT +disable-plugin=clnrest +disable-plugin=wss-proxy +EOF + + if ! _pgrep_cln_coord; then + echo "starting cln coordinator" + if + ! "$LIGHTNINGD_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + --daemon \ + >/dev/null + then + echo "error: starting cln coordinator daemon" >&2 + return 1 + fi + else + echo "cln coordinator already started" + fi + + cln_coord() { + "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + "$@" + } + + while [ \ + "$(cln_coord getinfo | jq -r '.blockheight')" -ne \ + "$(btc_reg getblockcount)" \ + ]; do + echo "waiting for cln coordinator to sync with the chain..." + sleep 1 + done + + echo "cln coordinator started, data directory is $CLN_COORD_DIR" + echo "cln_coord function set" +} + +_cln_coord_stop() { + if ! _pgrep_cln_coord; then + echo "cln coordinator already stopped" + else + if + ! "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + stop >/dev/null + then + echo "error: cln coordinator not stopped" >&2 + fi + + while _pgrep_cln_coord; do + echo "waiting for cln coordinator to stop..." + sleep 1 + done + echo "cln coordinator stopped" + fi +} + +cln_coord_stop_and_remove() { + _cln_coord_stop || return "$?" + + if [ ! -e "$CLN_COORD_DIR" ]; then + echo "cln coordinator directory not present" + else + if ! rm -rf "$CLN_COORD_DIR"; then + echo "error: removing cln coordinator directory $CLN_COORD_DIR" + return 1 + fi + echo "removed cln coordinator directory $CLN_COORD_DIR" + fi +} + +_pgrep_cln_user() { + pgrep -f \ + "$LIGHTNINGD_BIN --lightning-dir=$CLN_USER_DIR --regtest --daemon" \ + >/dev/null +} + +cln_user_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$CLN_USER_DIR" || return "$?" + + echo "writing cln user config" + cat << EOF > "$CLN_USER_DIR/config" +bitcoin-cli=$BITCOIN_CLI_BIN +bitcoin-datadir=$BITCOIN_DIR +bitcoin-rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoin-rpcpassword=$BITCOIN_REGTEST_RPC_PASS +bitcoin-rpcport=$BITCOIN_REGTEST_RPC_PORT +alias=cln-user +rgb=200000 +fee-base=0 +fee-per-satoshi=0 +min-capacity-sat=990000 +ignore-fee-limits=true +funding-confirms=1 +max-concurrent-htlcs=64 +max-dust-htlc-exposure-msat=1000000000 +cltv-delta=144 +cltv-final=144 +bitcoin-retry-timeout=120 +database-upgrade=true +log-file=lightning.log +addr=localhost:$CLN_USER_LISTEN_PORT +grpc-port=$CLN_USER_GRPC_PORT +disable-plugin=clnrest +disable-plugin=wss-proxy +EOF + + if ! _pgrep_cln_user; then + echo "starting cln user" + if + ! "$LIGHTNINGD_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + --daemon \ + >/dev/null + then + echo "error: starting cln user daemon" >&2 + return 1 + fi + else + echo "cln user already started" + fi + + cln_user() { + "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + "$@" + } + + while [ \ + "$(cln_user getinfo | jq -r '.blockheight')" -ne \ + "$(btc_reg getblockcount)" \ + ]; do + echo "waiting for cln user to sync with the chain..." + sleep 1 + done + + echo "cln user started, data directory is $CLN_USER_DIR" + echo "cln_user function set" +} + +_cln_user_stop() { + if ! _pgrep_cln_user; then + echo "cln user already stopped" + else + if + ! "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + stop >/dev/null + then + echo "error: cln user not stopped" >&2 + fi + + while _pgrep_cln_user; do + echo "waiting for cln user to stop..." + sleep 1 + done + echo "cln user stopped" + fi +} + +cln_user_stop_and_remove() { + _cln_user_stop || return "$?" + + if [ ! -e "$CLN_USER_DIR" ]; then + echo "cln user directory not present" + else + if ! rm -rf "$CLN_USER_DIR"; then + echo "error: removing cln user directory $CLN_USER_DIR" + return 1 + fi + echo "removed cln user directory $CLN_USER_DIR" + fi +} + +_pgrep_lnd_coord() { + pgrep -f "$LND_BIN --lnddir=$LND_COORD_DIR --bitcoin.regtest" >/dev/null +} + +lnd_coord_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$LND_COORD_DIR" || return "$?" + + echo "writing lnd coordinator config" + cat << EOF > "$LND_COORD_DIR/lnd.conf" +[Application Options] +listen=localhost:$LND_COORD_LISTEN_PORT +rpclisten=localhost:$LND_COORD_RPC_PORT +restlisten=localhost:$LND_COORD_REST_PORT +no-rest-tls=true +tlsdisableautofill=true +noseedbackup=true +maxpendingchannels=16 +minchansize=20000 +alias=lnd-coordinator +color=#300000 + +[Bitcoin] +bitcoin.node=bitcoind +bitcoin.defaultchanconfs=1 +bitcoin.basefee=0 +bitcoin.feerate=0 + +[Bitcoind] +bitcoind.dir=$BITCOIN_DIR +bitcoind.config=$BITCOIN_DIR/bitcoin.conf +bitcoind.rpchost=localhost:$BITCOIN_REGTEST_RPC_PORT +bitcoind.rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoind.rpcpass=$BITCOIN_REGTEST_RPC_PASS +bitcoind.zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +bitcoind.zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT + +[protocol] +protocol.wumbo-channels=true +EOF + + if ! _pgrep_lnd_coord; then + echo "starting lnd coordinator" + "$LND_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --bitcoin.regtest \ + >"$REGTEST_LOGS_DIR/lnd-coord" 2>&1 & + else + echo "lnd coordinator already started" + fi + + lnd_coord() { + "$LNCLI_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_COORD_RPC_PORT" \ + "$@" + } + + while [ "$(lnd_coord getinfo 2>/dev/null | jq -r '.synced_to_chain')" != true ]; do + echo "waiting for lnd coordinator to sync with the chain..." + sleep 1 + done + + # while [ "$(lnd_coord getinfo 2>/dev/null | jq -r '.synced_to_graph')" != true ]; do + # echo "waiting for lnd coordinator to sync with the graph..." + # sleep 1 + # done + + echo "lnd coordinator started, data directory is $LND_COORD_DIR" + echo "lnd_coord function set" +} + +_lnd_coord_stop() { + if ! _pgrep_lnd_coord; then + echo "lnd coordinator already stopped" + else + if + ! "$LNCLI_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_COORD_RPC_PORT" \ + stop >/dev/null + then + echo "error: lnd coordinator not stopped" >&2 + fi + + while _pgrep_lnd_coord; do + echo "waiting for lnd coordinator to stop..." + sleep 1 + done + echo "lnd coordinator stopped" + fi +} + +lnd_coord_stop_and_remove() { + _lnd_coord_stop || return "$?" + + if [ ! -e "$LND_COORD_DIR" ]; then + echo "lnd coordinator directory not present" + else + if ! rm -rf "$LND_COORD_DIR"; then + echo "error: removing lnd coordinator directory $LND_COORD_DIR" + return 1 + fi + echo "removed lnd coordinator directory $LND_COORD_DIR" + fi +} + +_pgrep_lnd_user() { + pgrep -f "$LND_BIN --lnddir=$LND_USER_DIR --bitcoin.regtest" >/dev/null +} + +lnd_user_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$LND_USER_DIR" || return "$?" + + echo "writing lnd user config" + cat << EOF > "$LND_USER_DIR/lnd.conf" +[Application Options] +listen=localhost:$LND_USER_LISTEN_PORT +rpclisten=localhost:$LND_USER_RPC_PORT +restlisten=localhost:$LND_USER_REST_PORT +no-rest-tls=true +tlsdisableautofill=true +noseedbackup=true +maxpendingchannels=16 +minchansize=20000 +alias=lnd-user +color=#400000 + +[Bitcoin] +bitcoin.node=bitcoind +bitcoin.defaultchanconfs=1 +bitcoin.basefee=0 +bitcoin.feerate=0 + +[Bitcoind] +bitcoind.dir=$BITCOIN_DIR +bitcoind.config=$BITCOIN_DIR/bitcoin.conf +bitcoind.rpchost=localhost:$BITCOIN_REGTEST_RPC_PORT +bitcoind.rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoind.rpcpass=$BITCOIN_REGTEST_RPC_PASS +bitcoind.zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +bitcoind.zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT + +[protocol] +protocol.wumbo-channels=true +EOF + + if ! _pgrep_lnd_user; then + echo "starting lnd user" + "$LND_BIN" \ + --lnddir="$LND_USER_DIR" \ + --bitcoin.regtest \ + >"$REGTEST_LOGS_DIR/lnd-user" 2>&1 & + else + echo "lnd user already started" + fi + + lnd_user() { + "$LNCLI_BIN" \ + --lnddir="$LND_USER_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_USER_RPC_PORT" \ + "$@" + } + + while [ "$(lnd_user getinfo 2>/dev/null | jq -r '.synced_to_chain')" != true ]; do + echo "waiting for lnd user to sync with the chain..." + sleep 1 + done + + # while [ "$(lnd_user getinfo 2>/dev/null | jq -r '.synced_to_graph')" != true ]; do + # echo "waiting for lnd user to sync with the graph..." + # sleep 1 + # done + + echo "lnd user started, data directory is $LND_USER_DIR" + echo "lnd_user function set" +} + +_lnd_user_stop() { + if ! _pgrep_lnd_user; then + echo "lnd user already stopped" + else + if + ! "$LNCLI_BIN" \ + --lnddir="$LND_USER_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_USER_RPC_PORT" \ + stop >/dev/null + then + echo "error: lnd user not stopped" >&2 + fi + + while _pgrep_lnd_user; do + echo "waiting for lnd user to stop..." + sleep 1 + done + echo "lnd user stopped" + fi +} + +lnd_user_stop_and_remove() { + _lnd_user_stop || return "$?" + + if [ ! -e "$LND_USER_DIR" ]; then + echo "lnd user directory not present" + else + if ! rm -rf "$LND_USER_DIR"; then + echo "error: removing lnd user directory $LND_USER_DIR" + return 1 + fi + echo "removed lnd user directory $LND_USER_DIR" + fi +} + +_mine_blocks_coordinator() { + coord_node="$1" + case "$coord_node" in + cln) + if ! coord_address="$( + cln_coord -k newaddr addresstype=p2tr | jq -r '.p2tr' + )"; then + echo "error: generating cln coordinator address" >&2 + return 1 + fi + ;; + lnd) + if ! coord_address="$( + lnd_coord newaddress p2tr | jq -r '.address' + )"; then + echo "error: generating lnd coordinator address" >&2 + return 1 + fi + ;; + esac + + bitcoin_regtest_mine 101 "$coord_address" || return "$?" + unset user_address +} + +_robosats_regtest_channel_create_cln_user() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + _bitcoin_regtest_check_started || return "$?" + + case "$coord_node" in + cln) + if ! _pgrep_cln_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist cln_coord; then + return 1 + fi + coord_id="$(cln_coord getinfo | jq -r '.id')" + coord_port="$CLN_COORD_LISTEN_PORT" + ;; + lnd) + if ! _pgrep_lnd_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist lnd_coord; then + return 1 + fi + coord_id="$(lnd_coord getinfo | jq -r '.identity_pubkey')" + coord_port="$LND_COORD_LISTEN_PORT" + ;; + *) + echo "error: coordinator node can only be cln and lnd" >&2 + return 1 + ;; + esac + + if ! _pgrep_cln_user; then + echo "error: cln user not started" >&2 + return 1 + fi + if ! _function_exist cln_user; then + return 1 + fi + + if ! cln_user connect "$coord_id"@localhost:"$coord_port" >/dev/null; then + echo "error: connection lightning nodes" >&2 + return 1 + fi + echo "lightning nodes connected" + + # check if channel not already present + if [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels | length' + )" -ge 1 ]; then + echo "lightning nodes already have a channel, not opening a new one" + else + echo "mining blocks to coordinator $coord_node" + _mine_blocks_coordinator "$coord_node" || return "$?" + + previous_output_number="$(cln_user listfunds | jq -r '.outputs | length')" + + if ! user_address="$(cln_user -k newaddr addresstype=p2tr | jq -r '.p2tr')"; then + echo "error: generating user address" >&2 + return 1 + fi + echo "mining blocks to user cln" + bitcoin_regtest_mine 101 "$user_address" || return "$?" + unset user_address + + while [ \ + "$(cln_user listfunds | jq -r '.outputs | length')" -le \ + "$previous_output_number" \ + ]; do + echo "waiting for cln user to see the new blocks..." + sleep 5 + done + while [ "$(cln_user listfunds | jq -r '.outputs[0].status')" != "confirmed" ]; do + echo "waiting for cln user funds to mature..." + sleep 1 + done + + unset previous_output_number + + if ! cln_user -k fundchannel \ + id="$coord_id" \ + amount=1btc \ + feerate=10000perkb \ + announce=true \ + >/dev/null + then + echo "error: funding lightning channel" >&2 + return 1 + fi + echo "lightning channel created" + + bitcoin_regtest_mine 10 "new" || return "$?" + while [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels | length' + )" -lt 1 ]; do + echo "waiting for channel to confirm..." + sleep 1 + done + echo "lightning channel opened" + fi + + while [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels[].state' + )" != "CHANNELD_NORMAL" ]; do + echo "waiting for channel to be active..." + sleep 5 + done + echo "lightning channel is active" + + unset coord_port + unset coord_id + unset coord_node +} + +_robosats_regtest_channel_create_lnd_user() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + _bitcoin_regtest_check_started || return "$?" + + case "$coord_node" in + cln) + if ! _pgrep_cln_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist cln_coord; then + return 1 + fi + coord_id="$(cln_coord getinfo | jq -r '.id')" + coord_port="$CLN_COORD_LISTEN_PORT" + ;; + lnd) + if ! _pgrep_lnd_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist lnd_coord; then + return 1 + fi + coord_id="$(lnd_coord getinfo | jq -r '.identity_pubkey')" + coord_port="$LND_COORD_LISTEN_PORT" + ;; + *) + echo "error: coordinator node can only be cln and lnd" >&2 + return 1 + ;; + esac + + if ! _pgrep_lnd_user; then + echo "error: lnd user not started" >&2 + return 1 + fi + if ! _function_exist lnd_user; then + return 1 + fi + + already_connected=false + for pub_key in $(lnd_user listpeers | jq -r '.peers[].pub_key'); do + if [ "$pub_key" = "$coord_id" ]; then + already_connected=true + break + fi + done + unset pub_key + if [ "$already_connected" = false ]; then + if ! lnd_user connect "$coord_id"@localhost:"$coord_port" >/dev/null; then + echo "error: connection lightning nodes" >&2 + return 1 + fi + fi + unset already_connected + echo "lightning nodes connected" + + # check if channel not already present + if [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels | length' + )" -ge 1 ]; then + echo "lightning nodes already have a channel, not opening a new one" + else + echo "mining blocks to coordinator $coord_node" + _mine_blocks_coordinator "$coord_node" || return "$?" + + previous_output_number="$(lnd_user listunspent | jq -r '.utxos | length')" + + if ! user_address="$(lnd_user newaddress p2tr | jq -r '.address')"; then + echo "error: generating user address" >&2 + return 1 + fi + echo "mining blocks to user lnd" + bitcoin_regtest_mine 101 "$user_address" || return "$?" + unset user_address + + while [ \ + "$(lnd_user listunspent | jq -r '.utxos | length')" -le \ + "$previous_output_number" \ + ]; do + echo "waiting for lnd user to see the new blocks..." + sleep 5 + done + while [ "$(lnd_user listunspent | jq -r '.utxos[0].confirmations')" -lt 100 ]; do + echo "waiting for lnd user funds to mature..." + sleep 1 + done + + unset previous_output_number + + if ! lnd_user openchannel \ + --node_key "$coord_id" \ + --local_amt "100000000" \ + --sat_per_vbyte "10" \ + --min_confs "1" \ + --channel_type "anchors" \ + >/dev/null + then + echo "error: funding lightning channel" >&2 + return 1 + fi + echo "lightning channel created" + + bitcoin_regtest_mine 10 "new" || return "$?" + while [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels | length' + )" -lt 1 ]; do + echo "waiting for channel to confirm..." + sleep 1 + done + echo "lightning channel opened" + fi + + while [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels[].active' + )" != true ]; do + echo "waiting for channel to be active..." + sleep 5 + done + echo "lightning channel is active" + + unset coord_port + unset coord_id + unset coord_node +} + +robosats_regtest_channel_create() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + if [ "$#" -lt 1 ]; then + echo "error: insert user node" >&2 + return 1 + fi + user_node="$1" + shift 1 + + case "$coord_node" in + cln|lnd) ;; + *) + echo "error: $coord_node should be cln or lnd" >&2 + return 1 + ;; + esac + + case "$user_node" in + cln) + _robosats_regtest_channel_create_cln_user "$coord_node" + ;; + lnd) + _robosats_regtest_channel_create_lnd_user "$coord_node" + ;; + *) + echo "error: $user_node should be cln or lnd" >&2 + return 1 + ;; + esac + + unset user_node + unset coord_node +} + +roboauto_regtest_setup() { + if [ "$ROBOAUTO_GIT_DIR" = false ]; then + echo "error: roboauto is disable, set ROBOAUTO_GIT_DIR in .env to activate it" >&2 + return 1 + fi + + _command_exist roboauto || return "$?" + + _create_dir "$ROBOAUTO_DIR" || return "$?" + + ra_reg() { + roboauto \ + --config-dir "$REGTEST_NODES_DIR/roboauto" \ + --data-dir "$REGTEST_NODES_DIR/roboauto" \ + "$@" + } + + echo "writing roboauto regtest config" + cat << EOF > "$ROBOAUTO_DIR/config.ini" +[federation] +exp = None +sau = None +tos = None +tbl = None +bve = None +loc = "http://127.0.0.1:$RUNSERVER_PORT" +EOF + + case "$LNVENDOR_USER" in + cln) + roboauto_cln_script="$ROBOAUTO_GIT_DIR/data/lightning-node-core-lightning" + if [ ! -f "$roboauto_cln_script" ]; then + echo "error: roboauto cln script not found in $ROBOAUTO_GIT_DIR" >&2 + return 1 + fi + + search_data=$(cat << EOF +lightning-cli "\$@" +EOF + ) + new_data=$(cat << EOF +"$LIGHTNING_CLI_BIN" \ +--lightning-dir="$CLN_USER_DIR" \ +--regtest \ +"\$@" +EOF + ) + sed "s|$search_data|$new_data|" \ + "$roboauto_cln_script" > "$ROBOAUTO_DIR/lightning-node" || \ + return "$?" + + echo "roboauto cln node script set up" + ;; + lnd) + roboauto_lnd_script="$ROBOAUTO_GIT_DIR/data/lightning-node-lnd" + if [ ! -f "$roboauto_lnd_script" ]; then + echo "error: roboauto lnd script not found in $ROBOAUTO_GIT_DIR" >&2 + return 1 + fi + + search_data=$(cat << EOF +lncli "\$@" +EOF + ) + new_data=$(cat << EOF +"$LNCLI_BIN" \ +--lnddir="$LND_USER_DIR" \ +--network regtest \ +--rpcserver localhost:"$LND_USER_RPC_PORT" \ +"\$@" +EOF + ) + sed "s|$search_data|$new_data|" \ + "$roboauto_lnd_script" > "$ROBOAUTO_DIR/lightning-node" || \ + return "$?" + + echo "roboauto lnd node script set up" + ;; + esac + if ! chmod u+x "$ROBOAUTO_DIR/lightning-node"; then + echo "error: changing file mode on roboauto lightning-node" >&2 + return 1 + fi +} + +roboauto_regtest_remove() { + if ! rm -rf "$ROBOAUTO_DIR"; then + echo "error: removing roboauto directory $ROBOAUTO_DIR" >&2 + return 1 + fi +} + +robosats_regtest_stop_all() { + _lnd_user_stop + _lnd_coord_stop + _cln_user_stop + _cln_coord_stop + _bitcoin_regtest_stop +} + +robosats_regtest_stop_and_remove_all() { + roboauto_regtest_remove + lnd_user_stop_and_remove + lnd_coord_stop_and_remove + cln_user_stop_and_remove + cln_coord_stop_and_remove + bitcoin_regtest_stop_and_remove +} + +_node_test_setup() { + bitcoin_regtest_start || return "$?" + + bitcoin_regtest_mine 1 "new" || return "$?" + + printf "\n" + + case "$LNVENDOR_COORD" in + cln) + cln_coord_start || return "$?" + ;; + lnd) + lnd_coord_start || return "$?" + ;; + esac + + printf "\n" + + case "$LNVENDOR_USER" in + cln) + cln_user_start || return "$?" + ;; + lnd) + lnd_user_start || return "$?" + ;; + esac +} + +_node_server_setup() { + _node_test_setup || return "$?" + + printf "\n" + + robosats_regtest_channel_create \ + "$LNVENDOR_COORD" "$LNVENDOR_USER" || return "$?" + + if [ "$ROBOAUTO_GIT_DIR" != false ]; then + printf "\n" + + roboauto_regtest_setup + fi +} + +_robosats_regtest_info_print() { + cat << EOF +regtest nodes directory is $REGTEST_NODES_DIR +available command: + +bitcoin_regtest_start +bitcoin_regtest_stop_and_remove +bitcoin_regtest_mine number-of-blocks
|new +cln_coord_start +cln_coord_stop_and_remove +cln_user_start +cln_user_stop_and_remove +lnd_coord_start +lnd_coord_stop_and_remove +lnd_user_start +lnd_user_stop_and_remove +roboauto_regtest_setup +roboauto_regtest_remove +robosats_regtest_stop_all +robosats_regtest_stop_and_remove_all +robosats_regtest_channel_create cln|lnd cln|lnd +EOF +} + +_nodes_main() { + _nodes_environment_set || return "$?" + + # if shell is bash and script is sourced source also bash completions + if [ -n "$BASH_VERSION" ]; then + case "$0" in + /*|./*|../*) ;; + *) + if [ -f "scripts/traditional/robosats.bash-completion" ]; then + # shellcheck disable=SC1091 + . scripts/traditional/robosats.bash-completion + fi + if + [ "$ROBOAUTO_GIT_DIR" != false ] && + [ -f "$ROBOAUTO_GIT_DIR/completions/roboauto.bash-completion" ] + then + # shellcheck disable=SC1091 + # shellcheck disable=SC3044 + . "$ROBOAUTO_GIT_DIR/completions/roboauto.bash-completion" && + complete -F __roboauto_completion ra_reg + fi + ;; + esac + fi + + if [ "$#" -lt 1 ]; then + _robosats_regtest_info_print || return "$?" + else + case "$1" in + -h|--help) + cat << EOF +regtest-nodes [server|test] +EOF + return 0 + ;; + test) + _robosats_regtest_info_print || return "$?" + printf "\n" + _node_test_setup || return "$?" + ;; + server) + _robosats_regtest_info_print || return "$?" + printf "\n" + _node_server_setup || return "$?" + ;; + *) + echo "error: action $1 not recognized" >&2 + return 1 + ;; + esac + fi +} + +_nodes_main "$@" diff --git a/scripts/traditional/robosats.bash-completion b/scripts/traditional/robosats.bash-completion new file mode 100755 index 00000000..2b3dee15 --- /dev/null +++ b/scripts/traditional/robosats.bash-completion @@ -0,0 +1,281 @@ +#!/usr/bin/env bash + +# adapted from +# https://github.com/bitcoin/bitcoin/blob/master/contrib/completions/bash/bitcoin-cli.bash +_bitcoin_cli() { + local cur prev words=() cword + local bitcoin_cli + + # save and use original argument to invoke bitcoin-cli for -help, help and RPC + # as bitcoin-cli might not be in $PATH + bitcoin_cli="$1" + + if ! command -v "$bitcoin_cli" >/dev/null 2>&1; then + return 0 + fi + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + if ((cword > 5)); then + case ${words[cword-5]} in + sendtoaddress) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 4)); then + case ${words[cword-4]} in + importaddress|listtransactions|setban) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + signrawtransactionwithkey|signrawtransactionwithwallet) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "ALL NONE SINGLE ALL|ANYONECANPAY NONE|ANYONECANPAY SINGLE|ANYONECANPAY" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 3)); then + case ${words[cword-3]} in + addmultisigaddress) + return 0 + ;; + getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 2)); then + case ${words[cword-2]} in + addnode) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "add remove onetry" -- "$cur" ) ) + return 0 + ;; + setban) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) ) + return 0 + ;; + fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + case "$prev" in + backupwallet|dumpwallet|importwallet) + _filedir + return 0 + ;; + getaddednodeinfo|getrawmempool|lockunspent) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + getbalance|getnewaddress|listtransactions|sendmany) + return 0 + ;; + esac + + # determine already specified args necessary for RPC + local rpcargs=() + local i + for i in ${COMP_LINE}; do + case "$i" in + -conf=*|-datadir=*|-rpc*|-chain=*|-testnet|-signet|-regtest) + rpcargs=( "${rpcargs[@]}" "$i" ) + ;; + esac + done + + case "$cur" in + -conf=*) + cur="${cur#*=}" + _filedir + return 0 + ;; + -datadir=*) + cur="${cur#*=}" + _filedir -d + return 0 + ;; + -rpcwallet=*) + cur="${cur#*=}" + wallets="$($bitcoin_cli "${rpcargs[@]}" listwallets | jq -r '.[]')" + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$wallets" -- "$cur" ) ) + return 0 + ;; + -*=*) # prevent nonsense completions + return 0 + ;; + *) + local helpopts commands completions + + # only parse -help if senseful + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) + fi + + # only parse help if senseful + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + commands=$($bitcoin_cli "${rpcargs[@]}" help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }') + fi + + completions="$helpopts $commands generatetoaddress" + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$completions" -- "$cur" ) ) + + # Prevent space if an argument is desired + local word + for word in "${COMPREPLY[@]}"; do + case "$word" in + *=) + compopt -o nospace + break + ;; + esac + done + return 0 + ;; + esac +} && +complete -F _bitcoin_cli btc_reg + +# adapted from +# https://github.com/ElementsProject/lightning/blob/master/contrib/lightning-cli.bash-completion +_lightning_cli() { + local command_name="$1" + # local current_word="$2" + local previous_word="$3" + + local lightning_cli + + # lightning_cli might not be in $PATH + lightning_cli="$command_name" + + if ! command -v "$lightning_cli" >/dev/null 2>&1; then + return 0 + fi + + if [ "${COMP_CWORD}" -eq 1 ]; then + complete_opt=true + else + case "$previous_word" in + --help|-h) complete_opt=false ;; + help|-*) complete_opt=true ;; + *) complete_opt=false ;; + esac + fi + + if [ "$complete_opt" = true ]; then + # shellcheck disable=SC2034 + local cur prev words=() cword + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + case "$cur" in + -*=*) # prevent nonsense completions + return 0 + ;; + *) + local helpopts globalcmds + + # get the global options, starting with -- + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + globalcmds="$( + $lightning_cli --help 2>&1 | + tr '|' '\n' | + sed -n -e 's/ .*//' -e 's/\(-[-a-z0-9A-Z]*\).*/\1/p' + )" + fi + + # get the regular commands + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + helpopts="$( + $lightning_cli help 2>/dev/null | + sed -n 's/^\([a-z][a-z_-]*\).*/\1/p' | + sed '$ d' + )" + fi + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$helpopts $globalcmds" -X "*," -- "$cur" ) ) + ;; + esac + else + _minimal + fi +} && +complete -F _lightning_cli cln_coord && +complete -F _lightning_cli cln_user + +# adapted from +# https://github.com/lightningnetwork/lnd/blob/master/contrib/lncli.bash-completion +_lncli() { + local cur prev words=() cword + local lncli + + # lncli might not be in $PATH + lncli="$1" + + if ! command -v "$lncli" >/dev/null 2>&1; then + return 0 + fi + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + case "$prev" in + # example of further completion + newaddress) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "p2wkh np2wkh" -- "$cur" ) ) + return 0 + ;; + esac + + case "$cur" in + -*=*) # prevent nonsense completions + return 0 + ;; + esac + + if [ "$cword" -eq 1 ] || { + [ "$cword" -eq 2 ] && [ "$prev" = "help" ] + }; then + local helpopts globalcmds completions + + # get the global options, starting with -- + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + globalcmds=$($lncli help 2>&1 | awk '$1 ~ /^-/ { sub(/,/, ""); print $1}') + fi + + # get the regular commands + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + helpopts=$($lncli help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }' ) + fi + + completions="$helpopts $globalcmds help" + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$completions" -X "*," -- "$cur" ) ) + fi +} && +complete -F _lncli lnd_coord && +complete -F _lncli lnd_user diff --git a/scripts/traditional/templates/strfry.conf b/scripts/traditional/templates/strfry.conf new file mode 100644 index 00000000..2b119f08 --- /dev/null +++ b/scripts/traditional/templates/strfry.conf @@ -0,0 +1,138 @@ +## +## Default strfry config +## + +# Directory that contains the strfry LMDB database (restart required) +db = "$STRFRY_DIR/" + +dbParams { + # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) + maxreaders = 256 + + # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) + mapsize = 10995116277760 + + # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required) + noReadAhead = false +} + +events { + # Maximum size of normalised JSON, in bytes + maxEventSize = 65536 + + # Events newer than this will be rejected + rejectEventsNewerThanSeconds = 900 + + # Events older than this will be rejected + rejectEventsOlderThanSeconds = 94608000 + + # Ephemeral events older than this will be rejected + rejectEphemeralEventsOlderThanSeconds = 60 + + # Ephemeral events will be deleted from the DB when older than this + ephemeralEventsLifetimeSeconds = 300 + + # Maximum number of tags allowed + maxNumTags = 2000 + + # Maximum size for tag values, in bytes + maxTagValSize = 1024 +} + +relay { + # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) + bind = "127.0.0.1" + + # Port to open for the nostr websocket protocol (restart required) + port = $STRFRY_PORT + + # Set OS-limit on maximum number of open files/sockets (if 0, don't attempt to set) (restart required) + nofiles = 1000000 + + # HTTP header that contains the client's real IP, before reverse proxying (ie x-real-ip) (MUST be all lower-case) + realIpHeader = "" + + info { + # NIP-11: Name of this server. Short/descriptive (< 30 characters) + name = "Robosats" + + # NIP-11: Detailed information about relay, free-form + description = "Federation cache system." + + # NIP-11: Administrative nostr pubkey, for contact purposes + pubkey = "" + + # NIP-11: Alternative administrative contact (email, website, etc) + contact = "" + } + + # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required) + maxWebsocketPayloadSize = 131072 + + # Websocket-level PING message frequency (should be less than any reverse proxy idle timeouts) (restart required) + autoPingSeconds = 55 + + # If TCP keep-alive should be enabled (detect dropped connections to upstream reverse proxy) + enableTcpKeepalive = false + + # How much uninterrupted CPU time a REQ query should get during its DB scan + queryTimesliceBudgetMicroseconds = 10000 + + # Maximum records that can be returned per filter + maxFilterLimit = 500 + + # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time + maxSubsPerConnection = 3 + + writePolicy { + # If non-empty, path to an executable script that implements the writePolicy plugin logic + plugin = "" + } + + compression { + # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required) + enabled = true + + # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required) + slidingWindow = false + } + + logging { + # Dump all incoming messages + dumpInAll = false + + # Dump all incoming EVENT messages + dumpInEvents = false + + # Dump all incoming REQ/CLOSE messages + dumpInReqs = false + + # Log performance metrics for initial REQ database scans + dbScanPerf = false + + # Log reason for invalid event rejection? Can be disabled to silence excessive logging + invalidEvents = true + } + + numThreads { + # Ingester threads: route incoming requests, validate events/sigs (restart required) + ingester = 3 + + # reqWorker threads: Handle initial DB scan for events (restart required) + reqWorker = 3 + + # reqMonitor threads: Handle filtering of new events (restart required) + reqMonitor = 3 + + # negentropy threads: Handle negentropy protocol messages (restart required) + negentropy = 2 + } + + negentropy { + # Support negentropy protocol messages + enabled = true + + # Maximum records that sync will process before returning an error + maxSyncEvents = 1000000 + } +} diff --git a/scripts/traditional/traditional-services b/scripts/traditional/traditional-services new file mode 100755 index 00000000..c2e8b723 --- /dev/null +++ b/scripts/traditional/traditional-services @@ -0,0 +1,468 @@ +#!/bin/sh + +_command_exist() { + if command -v "$1" >/dev/null 2>&1; then + return 0 + else + echo "error: $1 command not found" >&2 + return 1 + fi +} + +_create_dir() { + if [ ! -e "$1" ]; then + mkdir -p "$1" || return "$?" + if [ "$#" -ge 2 ]; then + if ! chmod "$2" "$1"; then + echo "error: setting chmod $2 of $1" >&2 + return 1 + fi + fi + elif [ ! -d "$1" ]; then + echo "error: $1 is not a directory" >&2 + return 1 + fi +} + +# if $2 is provided use it as default +# otherwise return error when not found +_get_env_var() { + if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then + if [ "$#" -ge 2 ]; then + env_var="$2" + else + echo "error: getting $1 from .env" >&2 + return 1 + fi + fi + printf "%s\n" "$env_var" + + return 0 +} + +# transform relative path into absolute and remove trailing slashes +_get_env_var_path() { + env_var="$(_get_env_var "$@")" || return "$?" + real_path="$(realpath -m "$env_var")" || return "$?" + printf "%s\n" "$real_path" + + return 0 +} + +_services_environment_set() { + TRADITIONAL_SERVICES_DIR="$(_get_env_var_path "TRADITIONAL_SERVICES_DIR")" || return "$?" + TRADITIONAL_LOGS_DIR="$(_get_env_var_path "TRADITIONAL_LOGS_DIR")" || return "$?" + _create_dir "$TRADITIONAL_SERVICES_DIR" || return "$?" + _create_dir "$TRADITIONAL_LOGS_DIR" || return "$?" + + POSTGRES_DIR="$TRADITIONAL_SERVICES_DIR/postgres" + REDIS_DIR="$TRADITIONAL_SERVICES_DIR/redis" + STRFRY_DIR="$TRADITIONAL_SERVICES_DIR/strfry" + GNUPG_DIR="$(_get_env_var_path "GNUPG_DIR")" || return "$?" + + POSTGRES_DB="$(_get_env_var "POSTGRES_DB")" || return "$?" + POSTGRES_USER="$(_get_env_var "POSTGRES_USER")" || return "$?" + POSTGRES_PASS="$(_get_env_var "POSTGRES_PASSWORD")" || return "$?" + POSTGRES_PORT="$(_get_env_var "POSTGRES_PORT")" || return "$?" + + REDIS_URL="$(_get_env_var "REDIS_URL")" || return "$?" + REDIS_PORT="$( + printf "%s\n" "$REDIS_URL" | + rev | + cut -d ":" -f 1 | + rev | + cut -d "/" -f 1 + )" + + STRFRY_GIT_DIR="$(_get_env_var_path "STRFRY_GIT_DIR")" || return "$?" + + RUNSERVER_PORT="$(_get_env_var "RUNSERVER_PORT")" || return "$?" + STRFRY_PORT="$(_get_env_var "STRFRY_PORT")" || return "$?" + + if [ "$POSTGRES_DB" = "postgres" ]; then + echo "error: POSTGRES_DB should not be postgres" >&2 + return 1 + fi + if [ "$POSTGRES_USER" = "$USER" ]; then + echo "error: POSTGRES_USER should not be the same as USER" >&2 + return 1 + fi + + TRADITIONAL_DIR="$(realpath "$(dirname "$0")")" + + STRFRY_CONF="$STRFRY_DIR/strfry.conf" + + strfry_bin="$STRFRY_GIT_DIR/strfry" + + # to prevent having same command names and killing wrong programs + python_bin="$(which python3)" + celery_bin="$(which celery)" + + # docker-compose.yml + + RUNSERVER_COMMAND="$python_bin manage.py runserver 0.0.0.0:$RUNSERVER_PORT" + + CLEAN_ORDERS_COMMAND="$python_bin manage.py clean_orders" + FOLLOW_INVOICES_COMMAND="$python_bin manage.py follow_invoices" + # TELEGRAM_WATCHER_COMMAND="$python_bin manage.py telegram_watcher" + + CELERY_DEV_COMMAND="$celery_bin -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000" + CELERY_BEAT_COMMAND="$celery_bin -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + + STRFRY_RELAY_COMMAND="$strfry_bin --config $STRFRY_CONF relay" +} + +strfry_setup() { + if [ -e "$STRFRY_CONF" ]; then + echo "strfry conf $STRFRY_CONF already exists" + return 0 + fi + + _create_dir "$STRFRY_DIR" "0700" || return "$?" + + cp "$TRADITIONAL_DIR/templates/strfry.conf" "$STRFRY_CONF" + + sed -i "s|\$STRFRY_DIR|$STRFRY_DIR|g" "$STRFRY_CONF" + sed -i "s/\$STRFRY_PORT/$STRFRY_PORT/g" "$STRFRY_CONF" + + echo "strfry directory set up" +} + +postgres_action() { + if [ "$#" -lt 1 ]; then + echo "error: insert postgres action" >&2 + return 1 + fi + action="$1" + shift 1 + case "$action" in + setup|database) ;; + *) + echo "error: wrong action" >&2 + return 1 + ;; + esac + + if ! _command_exist postgres; then + return 1 + fi + if ! _command_exist initdb; then + return 1 + fi + if ! _command_exist psql; then + return 1 + fi + + case "$action" in + setup) + if [ -e "$POSTGRES_DIR" ]; then + echo "postgres directory $POSTGRES_DIR already exists" + return 0 + fi + _create_dir "$POSTGRES_DIR" "0700" || return "$?" + + if ! initdb -D "$POSTGRES_DIR"; then + echo "error: running initdb" >&2 + return 1 + fi + + cat << EOF > "$POSTGRES_DIR/postgresql.conf" +port = $POSTGRES_PORT +unix_socket_directories = '$POSTGRES_DIR' +EOF + ;; + database) + if [ ! -d "$POSTGRES_DIR" ]; then + printf "%s%s\n" \ + "error: $POSTGRES_DIR is not a directory, " \ + "should run postgres-setup" \ + >&2 + return 1 + fi + ;; + esac + + postgres_setup_log_file="$TRADITIONAL_LOGS_DIR/postgres-setup-log" + echo "starting postgres, setup log file is $postgres_setup_log_file" + postgres -D "$POSTGRES_DIR" >>"$postgres_setup_log_file" 2>&1 & + postgres_pid="$!" + + _postgres_shut_down() { + echo "shutting down postgres" + if ! kill "$postgres_pid"; then + echo "error: shutting down postgres" >&2 + return 1 + fi + } + + echo "wait..." + sleep 5 + + case "$action" in + setup) + echo "setting up postgres user $POSTGRES_USER" + psql_stdin=$(cat << EOF +CREATE ROLE $POSTGRES_USER WITH LOGIN PASSWORD '$POSTGRES_PASS'; +ALTER ROLE $POSTGRES_USER CREATEDB; +EOF + ) + ;; + database) + psql_stdin=$(cat << EOF +CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER; +EOF + ) + ;; + esac + printf "%s\n" "$psql_stdin" | + psql -h localhost -p "$POSTGRES_PORT" -U "$USER" -d postgres + + echo "wait..." + sleep 5 + + case "$action" in + database) + if ! DJANGO_SUPERUSER_USERNAME="$(_get_env_var "ESCROW_USERNAME")"; then + _postgres_shut_down || return "$?" + return 1 + fi + + if ! python3 manage.py migrate; then + _postgres_shut_down || return "$?" + return 1 + fi + + # shellcheck disable=SC2034 + DJANGO_SUPERUSER_PASSWORD="password" + DJANGO_SUPERUSER_EMAIL="superuser@email.com" + if ! python3 manage.py createsuperuser \ + --noinput \ + --username "$DJANGO_SUPERUSER_USERNAME" \ + --email "$DJANGO_SUPERUSER_EMAIL" + then + _postgres_shut_down || return "$?" + return 1 + fi + ;; + esac + + _postgres_shut_down || return "$?" + + return 0 +} + +cleanup_signal() { + printf "\n" + printf "%s\n" "Caught $1 signal, sending it to services..." + + pkill -"$2" -f "$STRFRY_RELAY_COMMAND" + pkill -"$2" -f "$CELERY_BEAT_COMMAND" + pkill -"$2" -f "$CELERY_DEV_COMMAND" + pkill -"$2" -f "$FOLLOW_INVOICES_COMMAND" + # pkill -"$2" -f "$TELEGRAM_WATCHER_COMMAND" + pkill -"$2" -f "$CLEAN_ORDERS_COMMAND" + pkill -"$2" -f "$RUNSERVER_COMMAND" + pkill -"$2" -f "redis-server \*:${REDIS_PORT}" + pkill -"$2" -f "postgres -D $POSTGRES_DIR" + + exit 0 +} + +cleanup_int() { + printf "\n" + printf "%s\n" "Caught INT signal, shutting down services..." + + pkill -TERM -f "$STRFRY_RELAY_COMMAND" + pkill -INT -f "$CELERY_BEAT_COMMAND" + pkill -INT -f "$CELERY_DEV_COMMAND" + pkill -TERM -f "$FOLLOW_INVOICES_COMMAND" + # pkill -TERM -f "$TELEGRAM_WATCHER_COMMAND" + pkill -TERM -f "$CLEAN_ORDERS_COMMAND" + pkill -TERM -f "$RUNSERVER_COMMAND" + pkill -INT -f "redis-server \*:${REDIS_PORT}" + pkill -INT -f "postgres -D $POSTGRES_DIR" + + exit 0 +} + +main_loop() { + if [ "$#" -lt 1 ]; then + echo "error: insert main loop action" >&2 + return 1 + fi + action="$1" + shift 1 + case "$action" in + test|server) ;; + *) + echo "error: $1 is invalid" >&2 + return 1 + ;; + esac + + if ! _command_exist postgres; then + return 1 + fi + if ! _command_exist redis-server; then + return 1 + fi + if [ "$action" = "server" ]; then + if ! _command_exist celery; then + return 1 + fi + fi + + if [ ! -d "$POSTGRES_DIR" ]; then + printf "%s%s\n" \ + "error: $POSTGRES_DIR is not a directory, " \ + "should run postgres-setup and postgres-database" \ + >&2 + return 1 + fi + if [ "$action" = "server" ]; then + if [ ! -d "$STRFRY_DIR" ]; then + printf "%s%s\n" \ + "error: $STRFRY_DIR is not a directory, " \ + "should run setup" \ + >&2 + return 1 + fi + fi + + if ! pgrep -a bitcoind >/dev/null 2>&1 || { + ! pgrep -a lightningd >/dev/null 2>&1 && + ! pgrep -a lnd >/dev/null 2>&1 + }; then + printf "%s%s\n" \ + "error: bitcoin or lightning not running, " \ + "make sure to run this script after running regtest-nodes" \ + >&2 + return 1 + fi + + _create_dir "$REDIS_DIR" || return "$?" + _create_dir "$GNUPG_DIR" "0700" || return "$?" + + trap "cleanup_signal HUP" HUP + trap "cleanup_signal QUIT" QUIT + trap "cleanup_signal TERM" TERM + trap "cleanup_int" INT + + while true; do + if ! pgrep -f "postgres -D $POSTGRES_DIR" >/dev/null; then + echo "starting postgres" + postgres -D "$POSTGRES_DIR" >> "$TRADITIONAL_LOGS_DIR/postgres" 2>&1 & + fi + + if ! pgrep -f "redis-server \*:${REDIS_PORT}" >/dev/null; then + echo "starting redis" + printf "%s\n%s\n%s\n" \ + "dir $REDIS_DIR" \ + "port $REDIS_PORT" \ + "maxclients 1024" | + redis-server - >> "$TRADITIONAL_LOGS_DIR/redis" 2>&1 & + fi + + if [ "$action" = "server" ]; then + if ! pgrep -f "$RUNSERVER_COMMAND" >/dev/null; then + echo "starting runserver" + $RUNSERVER_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/runserver" 2>&1 & + fi + + if ! pgrep -f "$STRFRY_RELAY_COMMAND" >/dev/null; then + echo "starting strfry relay" + $STRFRY_RELAY_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/strfry-relay" 2>&1 & + fi + + if ! pgrep -f "$CLEAN_ORDERS_COMMAND" >/dev/null; then + echo "starting clean-orders" + $CLEAN_ORDERS_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/clean-orders" 2>&1 & + fi + + if ! pgrep -f "$FOLLOW_INVOICES_COMMAND" >/dev/null; then + echo "starting follow-invoices" + $FOLLOW_INVOICES_COMMAND \ + >> "$TRADITIONAL_LOGS_DIR/follow-invoices" 2>&1 & + fi + + # if _get_env_var "TELEGRAM_TOKEN" >/dev/null 2>&1; then + # if ! pgrep -f "$TELEGRAM_WATCHER_COMMAND" >/dev/null; then + # echo "starting telegram-watcher" + # $TELEGRAM_WATCHER_COMMAND \ + # >> "$TRADITIONAL_LOGS_DIR/telegram-watcher" 2>&1 & + # fi + # fi + + if ! pgrep -f "$CELERY_DEV_COMMAND" >/dev/null; then + echo "starting celery worker" + $CELERY_DEV_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-worker" 2>&1 & + fi + + if ! pgrep -f "$CELERY_BEAT_COMMAND" >/dev/null; then + echo "starting celery beat" + $CELERY_BEAT_COMMAND >> "$TRADITIONAL_LOGS_DIR/celery-beat" 2>&1 & + fi + fi + + sleep 2 + done +} + +_services_main() { + if [ "$#" -lt 1 ]; then + echo "error: insert action" >&2 + return 1 + fi + action="$1" + shift 1 + + if [ ! -f ".env" ]; then + echo "error: .env is not present" >&2 + return 1 + fi + if ! _command_exist dotenv; then + return 1 + fi + + case "$action" in + -h|--help) + cat << EOF +traditional-services action + +postgres-setup +postgres-database +strfry-setup +nginx-setup +test +server +EOF + return 0 + ;; + esac + + _services_environment_set || return "$?" + + case "$action" in + postgres-setup) + postgres_action "setup" + ;; + postgres-database) + postgres_action "database" + ;; + strfry-setup) + strfry_setup + ;; + test|server) + main_loop "$action" + ;; + *) + echo "error: action $action not recognized" >&2 + return 1 + ;; + esac +} + +_services_main "$@" diff --git a/setup.md b/setup.md index 09b6bd8a..2c79dec0 100644 --- a/setup.md +++ b/setup.md @@ -103,3 +103,9 @@ You will need these commands also often or eventually: `docker exec -it lnd-dev lncli -network=testnet payinvoice --allow_self_payment` **RoboSats development site should be accessible on 127.0.0.1:8000** + +# Backend Development + +## Traditional environment without docker + +See [scripts/traditional/README.md](scripts/traditional/README.md) diff --git a/tests/test_frontend_fetch.py b/tests/test_frontend_fetch.py index 08b68b2d..a52fded0 100644 --- a/tests/test_frontend_fetch.py +++ b/tests/test_frontend_fetch.py @@ -1,7 +1,12 @@ +import unittest +from decouple import config from django.test import Client, TestCase from django.urls import reverse +SKIP_FRONTEND_TESTS = config('SKIP_FRONTEND_TESTS', default=False, cast=bool) + +@unittest.skipIf(SKIP_FRONTEND_TESTS, "Skipping frontend tests") class FrontendFetchTest(TestCase): def setUp(self): self.client = Client() diff --git a/tests/utils/node.py b/tests/utils/node.py index 002de9bc..a30c7b9f 100644 --- a/tests/utils/node.py +++ b/tests/utils/node.py @@ -16,17 +16,24 @@ def get_node(name="robot"): We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user) """ if name == "robot": - macaroon = codecs.encode( - open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), - "hex", + admin_macaroon_file = config( + "LND_TEST_USER_MACAROON_PATH", cast=str, + default="/lndrobot/data/chain/bitcoin/regtest/admin.macaroon" ) - port = 8080 + macaroon = codecs.encode( + open(admin_macaroon_file, "rb").read(), "hex", + ) + port = config("LND_TEST_USER_REST_PORT", cast=int, default=8080) elif name == "coordinator": - macaroon = codecs.encode( - open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), "hex" + admin_macaroon_file = config( + "LND_TEST_COORD_MACAROON_PATH", cast=str, + default="/lnd/data/chain/bitcoin/regtest/admin.macaroon" ) - port = 8081 + macaroon = codecs.encode( + open(admin_macaroon_file, "rb").read(), "hex" + ) + port = config("LND_TEST_COORD_REST_PORT", cast=int, default=8081) return {"port": port, "headers": {"Grpc-Metadata-macaroon": macaroon}} @@ -158,10 +165,10 @@ def set_up_regtest_network(): # Coordinator is either LND or CLN. Robot user is always LND. if LNVENDOR == "LND": coordinator_node_id = get_lnd_node_id("coordinator") - coordinator_port = 9735 + coordinator_port = config("LND_TEST_COORD_LISTEN_PORT", cast=int, default=9735) elif LNVENDOR == "CLN": coordinator_node_id = get_cln_node_id() - coordinator_port = 9737 + coordinator_port = config("CLN_TEST_COORD_LISTEN_PORT", cast=int, default=9737) print("Coordinator Node ID: ", coordinator_node_id) @@ -258,8 +265,12 @@ def generate_blocks(address, num_blocks): "method": "generatetoaddress", "params": [num_blocks, address], } + # set in docker-tests.yml + rpc_url = config("BITCOIND_RPCURL", cast=str, default="http://localhost:18443") + rpc_user = config("BITCOIND_RPCUSER", cast=str, default="test") + rpc_pass = config("BITCOIND_RPCPASSWORD", cast=str, default="test") response = requests.post( - "http://localhost:18443", json=data, auth=HTTPBasicAuth("test", "test") + rpc_url, json=data, auth=HTTPBasicAuth(rpc_user, rpc_pass) ) return response.json() diff --git a/tests/utils/pgp.py b/tests/utils/pgp.py index 7150c64f..61501603 100644 --- a/tests/utils/pgp.py +++ b/tests/utils/pgp.py @@ -1,8 +1,9 @@ +from decouple import config import gnupg def sign_message(message, private_key_path, passphrase_path): - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) with open(private_key_path, "r") as f: private_key = f.read()