From 3af8287da7f45eadb76d617343309394b892ed30 Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi <90936742+Reckless-Satoshi@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:13:37 +0000 Subject: [PATCH 1/6] Add mempool.space as fee estimator and remote read devfund pubkey (#1055) --- api/logics.py | 12 ++---- api/tasks.py | 10 +---- api/utils.py | 90 +++++++++++++++++++++++++++++++++++++++++++++ devfund_pubkey.json | 4 ++ 4 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 devfund_pubkey.json diff --git a/api/logics.py b/api/logics.py index 30629347..51ec07af 100644 --- a/api/logics.py +++ b/api/logics.py @@ -9,7 +9,7 @@ from django.utils import timezone from api.lightning.node import LNNode from api.models import Currency, LNPayment, MarketTick, OnchainPayment, Order from api.tasks import send_devfund_donation, send_notification -from api.utils import validate_onchain_address +from api.utils import get_minning_fee, validate_onchain_address from chat.models import Message FEE = float(config("FEE")) @@ -640,10 +640,7 @@ class Logics: ): # Not enough onchain balance to commit for this swap. return False - suggested_mining_fee_rate = LNNode.estimate_fee( - amount_sats=preliminary_amount, - target_conf=config("SUGGESTED_TARGET_CONF", cast=int, default=2), - )["mining_fee_rate"] + suggested_mining_fee_rate = get_minning_fee("suggested", preliminary_amount) # Hardcap mining fee suggested at 1000 sats/vbyte if suggested_mining_fee_rate > 1000: @@ -795,10 +792,7 @@ class Logics: num_satoshis = cls.payout_amount(order, user)[1]["invoice_amount"] if mining_fee_rate: # not a valid mining fee - min_mining_fee_rate = LNNode.estimate_fee( - amount_sats=num_satoshis, - target_conf=config("MINIMUM_TARGET_CONF", cast=int, default=24), - )["mining_fee_rate"] + min_mining_fee_rate = get_minning_fee("minimum", num_satoshis) min_mining_fee_rate = max(2, min_mining_fee_rate) diff --git a/api/tasks.py b/api/tasks.py index fa3cdb9f..aa8afe18 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -98,15 +98,9 @@ def send_devfund_donation(order_id, proceeds, reason): from api.lightning.node import LNNode from api.models import LNPayment, Order + from api.utils import get_devfund_pubkey - if config("NETWORK", cast=str) == "testnet": - target_pubkey = ( - "03ecb271b3e2e36f2b91c92c65bab665e5165f8cdfdada1b5f46cfdd3248c87fd6" - ) - else: - target_pubkey = ( - "02187352cc4b1856b9604e0a79e1bc9b301be7e0c14acbbb8c29f7051d507127d7" - ) + target_pubkey = get_devfund_pubkey(config("NETWORK", cast=str)) order = Order.objects.get(id=order_id) coordinator_alias = config("COORDINATOR_ALIAS", cast=str, default="NoAlias") diff --git a/api/utils.py b/api/utils.py index 4348892b..45b62f1a 100644 --- a/api/utils.py +++ b/api/utils.py @@ -70,6 +70,96 @@ def validate_onchain_address(address): return True, None +mining_fee = {} + + +@ring.dict(mining_fee, expire=60) # keeps in cache for 60 seconds +def get_minning_fee(priority: str, preliminary_amount: int) -> int: + """ + priority: (str) 'suggested' | 'minimum' + Fetches suggested and minimum mining fee rates from mempool.space + uses LND/CLN fee estimator as fallback. + + mempool.space response object: + { + fastestFee: 1, + halfHourFee: 1, + hourFee: 1, + economyFee: 1, + minimumFee: 1 + } + Where 'suggested' is 'fastestFee' and 'minimum' is 'economyFee' + """ + + from api.lightning.node import LNNode + + session = get_session() + mempool_url = ( + "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion" + if USE_TOR + else "https://mempool.space" + ) + api_path = "/api/v1/fees/recommended" + + try: + response = session.get(mempool_url + api_path) + response.raise_for_status() # Raises stored HTTPError, if one occurred + data = response.json() + if priority == "suggested": + value = data.get("fastestFee") + elif priority == "minimum": + value = data.get("economyFee") + else: + raise Exception( + "an error occurred", + "unexpected value for mining fee priority", + priority, + ) + + except Exception: + # Fetch mining fee from LND/CLN instance + if priority == "suggested": + target_conf = config("SUGGESTED_TARGET_CONF", cast=int, default=2) + if priority == "minimum": + target_conf = config("MINIMUM_TARGET_CONF", cast=int, default=24) + + value = LNNode.estimate_fee( + amount_sats=preliminary_amount, + target_conf=target_conf, + )["mining_fee_rate"] + + return value + + +devfund_pubkey = {} + + +@ring.dict(devfund_pubkey, expire=3600) # keeps in cache for 3600 seconds +def get_devfund_pubkey(network: str) -> str: + """ + network: (str) "mainnet" | "testnet"; + Fetches devfund pubkey from `main` branch in the repository + fallback to hardcoded pubkey + """ + + session = get_session() + url = "https://raw.githubusercontent.com/RoboSats/robosats/main/devfund_pubkey.json" + # hardcoded fallback + mainnet = "02187352cc4b1856b9604e0a79e1bc9b301be7e0c14acbbb8c29f7051d507127d7" + testnet = "03ecb271b3e2e36f2b91c92c65bab665e5165f8cdfdada1b5f46cfdd3248c87fd6" + + try: + response = session.get(url) + response.raise_for_status() # Raises stored HTTPError, if one occurred + value = response.json().get(network) + if len(value) == 66: + return value + except Exception as e: + print(e) + + return mainnet if network == "mainnet" else testnet + + market_cache = {} diff --git a/devfund_pubkey.json b/devfund_pubkey.json new file mode 100644 index 00000000..92d8c667 --- /dev/null +++ b/devfund_pubkey.json @@ -0,0 +1,4 @@ +{ + "mainnet": "02187352cc4b1856b9604e0a79e1bc9b301be7e0c14acbbb8c29f7051d507127d7", + "testnet": "03ecb271b3e2e36f2b91c92c65bab665e5165f8cdfdada1b5f46cfdd3248c87fd6" +} From 84ea69344e9c811f3207e461ee360124faeaee01 Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi Date: Tue, 9 Jan 2024 00:43:38 +0000 Subject: [PATCH 2/6] Fix simplify read pubkey from file --- api/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/utils.py b/api/utils.py index 45b62f1a..0cf70103 100644 --- a/api/utils.py +++ b/api/utils.py @@ -143,21 +143,21 @@ def get_devfund_pubkey(network: str) -> str: """ session = get_session() - url = "https://raw.githubusercontent.com/RoboSats/robosats/main/devfund_pubkey.json" - # hardcoded fallback - mainnet = "02187352cc4b1856b9604e0a79e1bc9b301be7e0c14acbbb8c29f7051d507127d7" - testnet = "03ecb271b3e2e36f2b91c92c65bab665e5165f8cdfdada1b5f46cfdd3248c87fd6" + url = "https://raw.githubusercontent.com/RoboSats/robosats/main/devfund_pubey.json" try: response = session.get(url) response.raise_for_status() # Raises stored HTTPError, if one occurred value = response.json().get(network) - if len(value) == 66: - return value + if len(value) != 66: + raise Exception() except Exception as e: print(e) + with open("devfund_pubkey.json", "r") as f: + data = json.load(f) + value = data.get(network) - return mainnet if network == "mainnet" else testnet + return value market_cache = {} From ae27b18d30e33fe79980c0bd2600d328c6468817 Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi <90936742+Reckless-Satoshi@users.noreply.github.com> Date: Tue, 9 Jan 2024 01:22:50 +0000 Subject: [PATCH 3/6] Dev: replace isort, black, flake8 with ruff (#1057) --- .pre-commit-config.yaml | 44 ++++++++------------- api/management/commands/telegram_watcher.py | 1 - api/models/currency.py | 1 - control/admin.py | 2 - pyproject.toml | 8 +++- 5 files changed, 23 insertions(+), 33 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5abbaf2e..5887c50c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,3 @@ -# TODO -# add a hook which automatically generates the OpenApi schema on API changes -# and places them in an appropriate location exclude: '(api|chat|control)/migrations/.*' repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' @@ -50,27 +47,20 @@ repos: files: ^mobile/ types_or: [javascript, jsx, ts, tsx, css, markdown, json] # uses https://github.com/pre-commit/identify entry: bash -c 'cd mobile && npm run format' - - id: isort - name: isort - stages: - - commit - - merge-commit - language: system - types: [python] - entry: isort - - id: black - name: black - stages: - - commit - - merge-commit - language: system - types: [python] - entry: black - - id: flake8 - name: flake8 - stages: - - commit - - merge-commit - language: system - types: [python] - entry: flake8 \ No newline at end of file + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.11 + hooks: + - id: ruff + stages: + - commit + - merge-commit + language: system + args: [ --fix ] + types: [python] + - id: ruff-format + stages: + - commit + - merge-commit + language: system + types: [python] + diff --git a/api/management/commands/telegram_watcher.py b/api/management/commands/telegram_watcher.py index efc57a04..40e3cb4c 100644 --- a/api/management/commands/telegram_watcher.py +++ b/api/management/commands/telegram_watcher.py @@ -11,7 +11,6 @@ from api.utils import get_session class Command(BaseCommand): - help = "Polls telegram /getUpdates method" rest = 3 # seconds between consecutive polls diff --git a/api/models/currency.py b/api/models/currency.py index 7ac957d3..b9713bc2 100644 --- a/api/models/currency.py +++ b/api/models/currency.py @@ -6,7 +6,6 @@ from django.utils import timezone class Currency(models.Model): - with open("frontend/static/assets/currencies.json") as f: currency_dict = json.load(f) currency_choices = [(int(val), label) for val, label in list(currency_dict.items())] diff --git a/control/admin.py b/control/admin.py index 5bfd2299..932a8d18 100755 --- a/control/admin.py +++ b/control/admin.py @@ -8,7 +8,6 @@ from control.models import AccountingDay, BalanceLog @admin.register(AccountingDay) class AccountingDayAdmin(ImportExportModelAdmin): - list_display = ( "day", "contracted", @@ -34,7 +33,6 @@ class AccountingDayAdmin(ImportExportModelAdmin): @admin.register(BalanceLog) class BalanceLogAdmin(ImportExportModelAdmin): - list_display = ( "time", "total", diff --git a/pyproject.toml b/pyproject.toml index 9ad50bca..e1ddb6e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,9 @@ -[tool.isort] -profile = "black" +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + "*migrations/*", + "api/nick_generator/nick_generator.py", + ] [tool.coverage.run] omit = [ From b97cdcf285fbe2b3f010bfe76cf7694209f548ad Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi Date: Tue, 9 Jan 2024 01:24:54 +0000 Subject: [PATCH 4/6] Fix ensure unique coverage report artifacts --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0c619b5d..649f1587 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -79,5 +79,5 @@ jobs: - name: 'Upload coverage report' uses: actions/upload-artifact@v4 with: - name: coverage-report-${{ matrix.python-tag }}-${{ matrix.ln-vendor }} + name: coverage-report-${{ matrix.python-tag }}-${{ matrix.ln-vendor }}-${{ github.run_id }} path: htmlcov/ \ No newline at end of file From 18a22dcc36d4f075cbcfe514b6739a9d379abe09 Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi Date: Tue, 9 Jan 2024 01:29:11 +0000 Subject: [PATCH 5/6] Update integration test dependencies --- .github/workflows/integration-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 649f1587..a77f816c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -20,9 +20,9 @@ jobs: strategy: max-parallel: 2 matrix: - python-tag: ['3.11.6-slim-bookworm', '3.12-slim-bookworm'] - lnd-version: ['v0.17.0-beta', 'v0.17.2-beta.rc1'] - cln-version: ['v23.08.1'] + python-tag: ['3.11.6-slim-bookworm', '3.12.1-slim-bookworm'] + lnd-version: ['v0.17.3-beta'] + cln-version: ['v23.11.2'] ln-vendor: ['LND'] #, 'CLN'] steps: @@ -37,7 +37,7 @@ jobs: - uses: satackey/action-docker-layer-caching@v0.0.11 continue-on-error: true with: - key: coordinator-docker-cache-${{ hashFiles('Dockerfile', 'requirements.txt', 'requirements_dev.txt') }} + key: coordinator-docker-cache-${{ hashFiles('Dockerfile', 'requirements.txt', 'requirements_dev.txt') }}-${{ matrix.ln-vendor }} restore-keys: | coordinator-docker-cache- From 5e58f0c8ce54beca4ac96e45e38580a27b0dc651 Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi Date: Tue, 9 Jan 2024 14:36:27 +0000 Subject: [PATCH 6/6] Change default max_order_size --- .env-sample | 2 +- api/logics.py | 2 +- api/views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env-sample b/.env-sample index ac1ec6a9..7a1f48ab 100644 --- a/.env-sample +++ b/.env-sample @@ -108,7 +108,7 @@ MAX_PUBLIC_ORDERS = 100 # Minimum order size (must be bigger than DB constrain in /robosats/settings.py MIN_TRADE, currently 20_000 Sats) MIN_ORDER_SIZE = 20000 # Minimum order size (must be smaller than DB constrain in /robosats/settings.py MAX_TRADE, currently 5_000_000 Sats) -MAX_ORDER_SIZE = 5000000 +MAX_ORDER_SIZE = 500000 # For CLTV_expiry calculation # Assume 8 min/block assumed diff --git a/api/logics.py b/api/logics.py index 51ec07af..54918014 100644 --- a/api/logics.py +++ b/api/logics.py @@ -19,7 +19,7 @@ ESCROW_USERNAME = config("ESCROW_USERNAME") PENALTY_TIMEOUT = int(config("PENALTY_TIMEOUT")) MIN_ORDER_SIZE = config("MIN_ORDER_SIZE", cast=int, default=20_000) -MAX_ORDER_SIZE = config("MAX_ORDER_SIZE", cast=int, default=5_000_000) +MAX_ORDER_SIZE = config("MAX_ORDER_SIZE", cast=int, default=500_000) EXP_MAKER_BOND_INVOICE = int(config("EXP_MAKER_BOND_INVOICE")) EXP_TAKER_BOND_INVOICE = int(config("EXP_TAKER_BOND_INVOICE")) diff --git a/api/views.py b/api/views.py index 453c69e4..47da524e 100644 --- a/api/views.py +++ b/api/views.py @@ -913,7 +913,7 @@ class LimitView(ListAPIView): def get(self, request): # Trade limits as BTC min_trade = config("MIN_ORDER_SIZE", cast=int, default=20_000) / 100_000_000 - max_trade = config("MAX_ORDER_SIZE", cast=int, default=5_000_000) / 100_000_000 + max_trade = config("MAX_ORDER_SIZE", cast=int, default=500_000) / 100_000_000 payload = {} queryset = Currency.objects.all().order_by("currency")