Add core-lightning as backend lightning node vendor (#611)

* Add CLN node backend image and service (#418)

* Add cln service

* Add hodlvoice Dockerfile and entrypoint

* Add lnnode vendor switch (#431)

* Add LNNode vendor switch

* Add CLN version to frontend and other fixes

* init

* first draft

* add unsettled_local_balance and unsettled_remote_balance

* gen_hold_invoice now takes 3 more variables to build a label for cln

* remove unneeded payment_hash from gen_hold_invoice

* remove comment

* add get_cln_version

* first draft of clns follow_send_payment

* fix name of get_lnd_version

* enable flake8

* flake8 fixes

* renaming cln file, class and get_version

* remove lnd specific commented code

* get_version: add try/except, refactor to top to mimic lnd.py

* rename htlc_cltv to htlc_expiry

* add clns lookup_invoice_status

* refactored double_check_htlc_is_settled to the end to match lnds file

* fix generate_rpc

* Add sample environmental variables, small fixes

* Fix CLN gRPC port

* Fix gen_hold_invoice, plus some other tiny fixes (#435)

* Fix channel_balance to use int object inside Amount (#438)

* Add CLN/LND volume to celery-beat service

* Add CLN/LND volume to celery-beat service

* Bump CLN to v23.05

* changes for 0.5 and some small fixes

* change invoice expiry from absolute to relative duration

* add try/except to catch timeout error

* fix failure_reason to be ln_payment failure reasons, albeit inaccurate sometimes

* refactor follow_send_payment and add pending check to expired case

* fix status comments

* add send_keysend method

* fix wrong state ints in cancel and settle

* switch to use hodlinvoicelookup in double_check

* move pay command after lnpayment status update

* remove loop in follow_send_payment and add error result for edge case

* fix typeerror for payment_hash

* rework follow_send_payment logic and payment_hash, watch harder if pending

* use fully qualified names for status instead of raw int

* missed 2 status from prev commit

* Always copy the cln-grpc-hodl plugin on start up

* Fix ALLOW_SELF_KEYSEND linting error

* Fix missing definition of failure_reason

---------

Co-authored-by: daywalker90 <admin@noserver4u.de>

.env-sample

@@ -1,16 +1,21 @@
 # Coordinator Alias (Same as longAlias)
 COORDINATOR_ALIAS="Local Dev"
+# Lightning node vendor: CLN | LND
+LNVENDOR='CLN'
 
 # LND directory to read TLS cert and macaroon
 LND_DIR='/lnd/'
 MACAROON_PATH='data/chain/bitcoin/testnet/admin.macaroon'
-
 # LND directory can not be specified, instead cert and macaroon can be provided as base64 strings
 # base64 ~/.lnd/tls.cert | tr -d '\n'
 LND_CERT_BASE64='LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNLVENDQWRDZ0F3SUJBZ0lRQ0VoeGpPZXY1bGQyVFNPTXhKalFvekFLQmdncWhrak9QUVFEQWpBNE1SOHcKSFFZRFZRUUtFeFpzYm1RZ1lYVjBiMmRsYm1WeVlYUmxaQ0JqWlhKME1SVXdFd1lEVlFRREV3d3dNakJtTVRnMQpZelkwTnpVd0hoY05Nakl3TWpBNE1UWXhOalV3V2hjTk1qTXdOREExTVRZeE5qVXdXakE0TVI4d0hRWURWUVFLCkV4WnNibVFnWVhWMGIyZGxibVZ5WVhSbFpDQmpaWEowTVJVd0V3WURWUVFERXd3d01qQm1NVGcxWXpZME56VXcKV1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVNJVWdkcVMrWFZKL3EzY0JZeWd6ZDc2endaanlmdQpLK3BzcWNYVkFyeGZjU2NXQ25jbXliNGRaMy9Lc3lLWlRaamlySDE3aEY0OGtIMlp5clRZSW9hZG80RzdNSUc0Ck1BNEdBMVVkRHdFQi93UUVBd0lDcERBVEJnTlZIU1VFRERBS0JnZ3JCZ0VGQlFjREFUQVBCZ05WSFJNQkFmOEUKQlRBREFRSC9NQjBHQTFVZERnUVdCQlEwWUJjZXdsd1BqYTJPRXFyTGxzZnJscEswUFRCaEJnTlZIUkVFV2pCWQpnZ3d3TWpCbU1UZzFZelkwTnpXQ0NXeHZZMkZzYUc5emRJSUVkVzVwZUlJS2RXNXBlSEJoWTJ0bGRJSUhZblZtClkyOXVib2NFZndBQUFZY1FBQUFBQUFBQUFBQUFBQUFBQUFBQUFZY0V3S2dRQW9jRUFBQUFBREFLQmdncWhrak8KUFFRREFnTkhBREJFQWlBd0dMY05qNXVZSkVwanhYR05OUnNFSzAwWmlSUUh2Qm50NHp6M0htWHBiZ0lnSWtvUQo3cHFvNGdWNGhiczdrSmt1bnk2bkxlNVg0ZzgxYjJQOW52ZnZ2bkk9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K'
 # base64 ~/.lnd/data/chain/bitcoin/testnet/admin.macaroon | tr -d '\n'
 LND_MACAROON_BASE64='AgEDbG5kAvgBAwoQsyI+PK+fyb7F2UyTeZ4seRIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgMt90uD6v4truTadWCjlppoeJ4hZrL1SBb09Y+4WOiI0='
 
+# CLN directory
+CLN_DIR='/cln/testnet/'
+CLN_GRPC_HOST='localhost:9999'
+
 # Bitcoin Core Daemon RPC, used to validate addresses
 BITCOIND_RPCURL = 'http://127.0.0.1:18332'
 BITCOIND_RPCUSER = 'robodev'

api/lightning/cln.py

@@ -0,0 +1,826 @@
+import hashlib
+import os
+import secrets
+import struct
+import time
+from datetime import datetime, timedelta
+
+import grpc
+import ring
+from decouple import config
+from django.utils import timezone
+
+from . import node_pb2 as noderpc
+from . import node_pb2_grpc as nodestub
+from . import primitives_pb2 as primitives__pb2
+
+#######
+# Works with CLN
+#######
+
+# Load the client's certificate and key
+with open(os.path.join(config("CLN_DIR"), "client.pem"), "rb") as f:
+    client_cert = f.read()
+with open(os.path.join(config("CLN_DIR"), "client-key.pem"), "rb") as f:
+    client_key = f.read()
+
+# Load the server's certificate
+with open(os.path.join(config("CLN_DIR"), "server.pem"), "rb") as f:
+    server_cert = f.read()
+
+
+CLN_GRPC_HOST = config("CLN_GRPC_HOST")
+DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
+MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000)
+
+
+class CLNNode:
+
+    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
+
+    # Create the SSL credentials object
+    creds = grpc.ssl_channel_credentials(
+        root_certificates=server_cert,
+        private_key=client_key,
+        certificate_chain=client_cert,
+    )
+    # Create the gRPC channel using the SSL credentials
+    channel = grpc.secure_channel(CLN_GRPC_HOST, creds)
+
+    # Create the gRPC stub
+    stub = nodestub.NodeStub(channel)
+
+    noderpc = noderpc
+
+    payment_failure_context = {
+        -1: "Catchall nonspecific error.",
+        201: "Already paid with this hash using different amount or destination.",
+        203: "Permanent failure at destination.",
+        205: "Unable to find a route.",
+        206: "Route too expensive.",
+        207: "Invoice expired.",
+        210: "Payment timed out without a payment in progress.",
+    }
+
+    @classmethod
+    def get_version(cls):
+        try:
+            request = noderpc.GetinfoRequest()
+            print(request)
+            response = cls.stub.Getinfo(request)
+            print(response)
+            return response.version
+        except Exception as e:
+            print(e)
+            return None
+
+    @classmethod
+    def decode_payreq(cls, invoice):
+        """Decodes a lightning payment request (invoice)"""
+        request = noderpc.DecodeBolt11Request(bolt11=invoice)
+
+        response = cls.stub.DecodeBolt11(request)
+        return response
+
+    @classmethod
+    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
+        """Returns estimated fee for onchain payouts"""
+        # feerate estimaes work a bit differently in cln see https://lightning.readthedocs.io/lightning-feerates.7.html
+        request = noderpc.FeeratesRequest(style="PERKB")
+
+        response = cls.stub.Feerates(request)
+
+        # "opening" -> ~12 block target
+        return {
+            "mining_fee_sats": response.onchain_fee_estimates.opening_channel_satoshis,
+            "mining_fee_rate": response.perkb.opening / 1000,
+        }
+
+    wallet_balance_cache = {}
+
+    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def wallet_balance(cls):
+        """Returns onchain balance"""
+        request = noderpc.ListfundsRequest()
+
+        response = cls.stub.ListFunds(request)
+
+        unconfirmed_balance = 0
+        confirmed_balance = 0
+        total_balance = 0
+        for utxo in response.outputs:
+            if not utxo.reserved:
+                if (
+                    utxo.status
+                    == noderpc.ListfundsOutputs.ListfundsOutputsStatus.UNCONFIRMED
+                ):
+                    unconfirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
+                elif (
+                    utxo.status
+                    == noderpc.ListfundsOutputs.ListfundsOutputsStatus.CONFIRMED
+                ):
+                    confirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
+
+        return {
+            "total_balance": total_balance,
+            "confirmed_balance": confirmed_balance,
+            "unconfirmed_balance": unconfirmed_balance,
+        }
+
+    channel_balance_cache = {}
+
+    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def channel_balance(cls):
+        """Returns channels balance"""
+        request = noderpc.ListpeerchannelsRequest()
+
+        response = cls.stub.ListPeerChannels(request)
+
+        local_balance_sat = 0
+        remote_balance_sat = 0
+        unsettled_local_balance = 0
+        unsettled_remote_balance = 0
+        for channel in response.channels:
+            if (
+                channel.state
+                == noderpc.ListpeerchannelsChannels.ListpeerchannelsChannelsState.CHANNELD_NORMAL
+            ):
+                local_balance_sat += channel.to_us_msat.msat // 1_000
+                remote_balance_sat += (
+                    channel.total_msat.msat - channel.to_us_msat.msat
+                ) // 1_000
+            for htlc in channel.htlcs:
+                if (
+                    htlc.direction
+                    == noderpc.ListpeerchannelsChannelsHtlcs.ListpeerchannelsChannelsHtlcsDirection.IN
+                ):
+                    unsettled_local_balance += htlc.amount_msat.msat // 1_000
+                elif (
+                    htlc.direction
+                    == noderpc.ListpeerchannelsChannelsHtlcs.ListpeerchannelsChannelsHtlcsDirection.OUT
+                ):
+                    unsettled_remote_balance += htlc.amount_msat.msat // 1_000
+
+        return {
+            "local_balance": local_balance_sat,
+            "remote_balance": remote_balance_sat,
+            "unsettled_local_balance": unsettled_local_balance,
+            "unsettled_remote_balance": unsettled_remote_balance,
+        }
+
+    @classmethod
+    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
+        """Send onchain transaction for buyer payouts"""
+
+        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
+            return False
+
+        request = noderpc.WithdrawRequest(
+            destination=onchainpayment.address,
+            satoshi=primitives__pb2.AmountOrAll(
+                amount=primitives__pb2.Amount(msat=onchainpayment.sent_satoshis * 1_000)
+            ),
+            feerate=primitives__pb2.Feerate(
+                perkb=int(onchainpayment.mining_fee_rate) * 1_000
+            ),
+            minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool)),
+        )
+
+        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
+        delay = (
+            secrets.randbelow(2**256) / (2**256) * 10
+        )  # Random uniform 0 to 5 secs with good entropy
+        time.sleep(3 + delay)
+
+        if onchainpayment.status == queue_code:
+            # Changing the state to "MEMPO" should be atomic with SendCoins.
+            onchainpayment.status = on_mempool_code
+            onchainpayment.save(update_fields=["status"])
+            response = cls.stub.Withdraw(request)
+
+            if response.txid:
+                onchainpayment.txid = response.txid.hex()
+                onchainpayment.broadcasted = True
+            onchainpayment.save(update_fields=["txid", "broadcasted"])
+            return True
+
+        elif onchainpayment.status == on_mempool_code:
+            # Bug, double payment attempted
+            return True
+
+    @classmethod
+    def cancel_return_hold_invoice(cls, payment_hash):
+        """Cancels or returns a hold invoice"""
+        request = noderpc.HodlInvoiceCancelRequest(
+            payment_hash=bytes.fromhex(payment_hash)
+        )
+        response = cls.stub.HodlInvoiceCancel(request)
+
+        return response.state == noderpc.HodlInvoiceCancelResponse.Hodlstate.CANCELED
+
+    @classmethod
+    def settle_hold_invoice(cls, preimage):
+        """settles a hold invoice"""
+        request = noderpc.HodlInvoiceSettleRequest(
+            payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest()
+        )
+        response = cls.stub.HodlInvoiceSettle(request)
+
+        return response.state == noderpc.HodlInvoiceSettleResponse.Hodlstate.SETTLED
+
+    @classmethod
+    def gen_hold_invoice(
+        cls,
+        num_satoshis,
+        description,
+        invoice_expiry,
+        cltv_expiry_blocks,
+        order_id,
+        lnpayment_concept,
+        time,
+    ):
+        """Generates hold invoice"""
+
+        # constant 100h invoice expiry because cln has to cancel htlcs if invoice expires
+        # or it can't associate them anymore
+        invoice_expiry = cltv_expiry_blocks * 10 * 60
+
+        hold_payment = {}
+        # The preimage is a random hash of 256 bits entropy
+        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
+
+        request = noderpc.InvoiceRequest(
+            description=description,
+            amount_msat=primitives__pb2.AmountOrAny(
+                amount=primitives__pb2.Amount(msat=num_satoshis * 1_000)
+            ),
+            label=f"Order:{order_id}-{lnpayment_concept}-{time}",
+            expiry=invoice_expiry,
+            cltv=cltv_expiry_blocks,
+            preimage=preimage,  # preimage is actually optional in cln, as cln would generate one by default
+        )
+        response = cls.stub.HodlInvoice(request)
+
+        hold_payment["invoice"] = response.bolt11
+        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
+        hold_payment["preimage"] = preimage.hex()
+        hold_payment["payment_hash"] = response.payment_hash.hex()
+        hold_payment["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        hold_payment["expires_at"] = timezone.make_aware(
+            datetime.fromtimestamp(response.expires_at)
+        )
+        hold_payment["cltv_expiry"] = cltv_expiry_blocks
+
+        return hold_payment
+
+    @classmethod
+    def validate_hold_invoice_locked(cls, lnpayment):
+        """Checks if hold invoice is locked"""
+        from api.models import LNPayment
+
+        request = noderpc.HodlInvoiceLookupRequest(
+            payment_hash=bytes.fromhex(lnpayment.payment_hash)
+        )
+        response = cls.stub.HodlInvoiceLookup(request)
+
+        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
+        # time has passed (but these are 15% padded at the moment). Should catch it
+        # and report back that the invoice has expired (better robustness)
+        if response.state == noderpc.HodlInvoiceLookupResponse.Hodlstate.OPEN:
+            pass
+        if response.state == noderpc.HodlInvoiceLookupResponse.Hodlstate.SETTLED:
+            pass
+        if response.state == noderpc.HodlInvoiceLookupResponse.Hodlstate.CANCELED:
+            pass
+        if response.state == noderpc.HodlInvoiceLookupResponse.Hodlstate.ACCEPTED:
+            lnpayment.expiry_height = response.htlc_expiry
+            lnpayment.status = LNPayment.Status.LOCKED
+            lnpayment.save(update_fields=["expiry_height", "status"])
+            return True
+
+    @classmethod
+    def lookup_invoice_status(cls, lnpayment):
+        """
+        Returns the status (as LNpayment.Status) of the given payment_hash
+        If unchanged, returns the previous status
+        """
+        from api.models import LNPayment
+
+        status = lnpayment.status
+        expiry_height = 0
+
+        cln_response_state_to_lnpayment_status = {
+            0: LNPayment.Status.INVGEN,  # OPEN
+            1: LNPayment.Status.SETLED,  # SETTLED
+            2: LNPayment.Status.CANCEL,  # CANCELLED
+            3: LNPayment.Status.LOCKED,  # ACCEPTED
+        }
+
+        try:
+            # this is similar to LNNnode.validate_hold_invoice_locked
+            request = noderpc.HodlInvoiceLookupRequest(
+                payment_hash=bytes.fromhex(lnpayment.payment_hash)
+            )
+            response = cls.stub.HodlInvoiceLookup(request)
+
+            status = cln_response_state_to_lnpayment_status[response.state]
+
+            # try saving expiry height
+            if hasattr(response, "htlc_expiry"):
+                try:
+                    expiry_height = response.htlc_expiry
+                except Exception:
+                    pass
+
+        except Exception as e:
+            # If it fails at finding the invoice: it has been expired for more than an hour (and could be paid or just expired).
+            # In RoboSats DB we make a distinction between cancelled and returned
+            #  (cln-grpc-hodl has separate state for hodl-invoices, which it forgets after an invoice expired more than an hour ago)
+            if "empty result for listdatastore_state" in str(e):
+                print(str(e))
+                request2 = noderpc.ListinvoicesRequest(
+                    payment_hash=bytes.fromhex(lnpayment.payment_hash)
+                )
+                try:
+                    response2 = cls.stub.ListInvoices(request2).invoices
+                except Exception as e:
+                    print(str(e))
+
+                if (
+                    response2[0].status
+                    == noderpc.ListinvoicesInvoices.ListinvoicesInvoicesStatus.PAID
+                ):
+                    status = LNPayment.Status.SETLED
+                elif (
+                    response2[0].status
+                    == noderpc.ListinvoicesInvoices.ListinvoicesInvoicesStatus.EXPIRED
+                ):
+                    status = LNPayment.Status.CANCEL
+                else:
+                    print(str(e))
+
+            # Other write to logs
+            else:
+                print(str(e))
+
+        return status, expiry_height
+
+    @classmethod
+    def resetmc(cls):
+        # don't think an equivalent exists for cln, maybe deleting gossip_store file?
+        return False
+
+    @classmethod
+    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
+        """Checks if the submited LN invoice comforms to expectations"""
+
+        payout = {
+            "valid": False,
+            "context": None,
+            "description": None,
+            "payment_hash": None,
+            "created_at": None,
+            "expires_at": None,
+        }
+
+        try:
+            payreq_decoded = cls.decode_payreq(invoice)
+        except Exception:
+            payout["context"] = {
+                "bad_invoice": "Does not look like a valid lightning invoice"
+            }
+            return payout
+
+        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
+        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
+        route_hints = payreq_decoded.route_hints.hints
+
+        # Max amount RoboSats will pay for routing
+        if routing_budget_ppm == 0:
+            max_routing_fee_sats = max(
+                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        else:
+            max_routing_fee_sats = int(
+                float(num_satoshis) * float(routing_budget_ppm) / 1000000
+            )
+
+        if route_hints:
+            routes_cost = []
+            # For every hinted route...
+            for hinted_route in route_hints:
+                route_cost = 0
+                # ...add up the cost of every hinted hop...
+                for hop_hint in hinted_route.hops:
+                    route_cost += hop_hint.feebase.msat / 1_000
+                    route_cost += hop_hint.feeprop * num_satoshis / 1_000_000
+
+                # ...and store the cost of the route to the array
+                routes_cost.append(route_cost)
+
+            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
+            if min(routes_cost) >= max_routing_fee_sats:
+                payout["context"] = {
+                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
+                }
+                return payout
+
+        if payreq_decoded.amount_msat.msat == 0:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has no explicit amount"
+            }
+            return payout
+
+        if not payreq_decoded.amount_msat.msat // 1_000 == num_satoshis:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided is not for "
+                + "{:,}".format(num_satoshis)
+                + " Sats"
+            }
+            return payout
+
+        payout["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        payout["expires_at"] = payout["created_at"] + timedelta(
+            seconds=payreq_decoded.expiry
+        )
+
+        if payout["expires_at"] < timezone.now():
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has already expired"
+            }
+            return payout
+
+        payout["valid"] = True
+        payout["description"] = payreq_decoded.description
+        payout["payment_hash"] = payreq_decoded.payment_hash.hex()
+
+        return payout
+
+    @classmethod
+    def pay_invoice(cls, lnpayment):
+        """Sends sats. Used for rewards payouts"""
+        from api.models import LNPayment
+
+        fee_limit_sat = int(
+            max(
+                lnpayment.num_satoshis
+                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        )  # 200 ppm or 10 sats
+        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=primitives__pb2.Amount(msat=fee_limit_sat * 1_000),
+            retry_for=timeout_seconds,
+        )
+
+        try:
+            response = cls.stub.Pay(request)
+
+            if response.status == noderpc.PayResponse.PayStatus.COMPLETE:
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = (
+                    float(response.amount_sent_msat.msat - response.amount_msat.msat)
+                    / 1000
+                )
+                lnpayment.preimage = response.payment_preimage.hex()
+                lnpayment.save(update_fields=["fee", "status", "preimage"])
+                return True, None
+            elif response.status == noderpc.PayResponse.PayStatus.PENDING:
+                failure_reason = "Payment isn't failed (yet)"
+                lnpayment.failure_reason = LNPayment.FailureReason.NOTYETF
+                lnpayment.status = LNPayment.Status.FLIGHT
+                lnpayment.save(update_fields=["failure_reason", "status"])
+                return False, failure_reason
+            else:  # response.status == noderpc.PayResponse.PayStatus.FAILED
+                failure_reason = "All possible routes were tried and failed permanently. Or were no routes to the destination at all."
+                lnpayment.failure_reason = LNPayment.FailureReason.NOROUTE
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.save(update_fields=["failure_reason", "status"])
+                return False, failure_reason
+        except grpc._channel._InactiveRpcError as e:
+            status_code = int(e.details().split("code: Some(")[1].split(")")[0])
+            failure_reason = cls.payment_failure_context[status_code]
+            lnpayment.failure_reason = LNPayment.FailureReason.NOROUTE
+            lnpayment.status = LNPayment.Status.FAILRO
+            lnpayment.save(update_fields=["failure_reason", "status"])
+            return False, failure_reason
+
+    @classmethod
+    def follow_send_payment(cls, lnpayment, fee_limit_sat, timeout_seconds):
+        """Sends sats to buyer, continuous update"""
+
+        from api.models import LNPayment, Order
+
+        hash = lnpayment.payment_hash
+
+        # retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
+        # allow_self_payment=True, No such thing in pay command and self_payments do not work with pay!
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=primitives__pb2.Amount(msat=fee_limit_sat * 1_000),
+            retry_for=timeout_seconds,
+        )
+
+        order = lnpayment.order_paid_LN
+        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
+            print(f"Order: {order.id} Payout is larger than collateral !?")
+            return
+
+        def watchpayment():
+            request_listpays = noderpc.ListpaysRequest(payment_hash=bytes.fromhex(hash))
+            while True:
+                try:
+                    response_listpays = cls.stub.ListPays(request_listpays)
+                except Exception as e:
+                    print(str(e))
+                    time.sleep(2)
+                    continue
+
+                if (
+                    len(response_listpays.pays) == 0
+                    or response_listpays.pays[0].status
+                    != noderpc.ListpaysPays.ListpaysPaysStatus.PENDING
+                ):
+                    return response_listpays
+                else:
+                    time.sleep(2)
+
+        def handle_response():
+            try:
+                lnpayment.status = LNPayment.Status.FLIGHT
+                lnpayment.in_flight = True
+                lnpayment.save(update_fields=["in_flight", "status"])
+
+                order.status = Order.Status.PAY
+                order.save(update_fields=["status"])
+
+                response = cls.stub.Pay(request)
+
+                if response.status == noderpc.PayResponse.PayStatus.PENDING:
+                    print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
+
+                    watchpayment()
+
+                    handle_response()
+
+                if response.status == noderpc.PayResponse.PayStatus.FAILED:
+                    lnpayment.status = LNPayment.Status.FAILRO
+                    lnpayment.last_routing_time = timezone.now()
+                    lnpayment.routing_attempts += 1
+                    lnpayment.failure_reason = LNPayment.FailureReason.NOROUTE
+                    lnpayment.in_flight = False
+                    if lnpayment.routing_attempts > 2:
+                        lnpayment.status = LNPayment.Status.EXPIRE
+                        lnpayment.routing_attempts = 0
+                    lnpayment.save(
+                        update_fields=[
+                            "status",
+                            "last_routing_time",
+                            "routing_attempts",
+                            "failure_reason",
+                            "in_flight",
+                        ]
+                    )
+
+                    order.status = Order.Status.FAI
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.FAI)
+                    )
+                    order.save(update_fields=["status", "expires_at"])
+                    print(
+                        f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[-1]}"
+                    )
+                    return {
+                        "succeded": False,
+                        "context": f"payment failure reason: {cls.payment_failure_context[-1]}",
+                    }
+
+                if response.status == noderpc.PayResponse.PayStatus.COMPLETE:
+                    print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
+                    lnpayment.status = LNPayment.Status.SUCCED
+                    lnpayment.fee = (
+                        float(
+                            response.amount_sent_msat.msat - response.amount_msat.msat
+                        )
+                        / 1000
+                    )
+                    lnpayment.preimage = response.payment_preimage.hex()
+                    lnpayment.save(update_fields=["status", "fee", "preimage"])
+                    order.status = Order.Status.SUC
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.SUC)
+                    )
+                    order.save(update_fields=["status", "expires_at"])
+
+                    results = {"succeded": True}
+                    return results
+
+            except grpc._channel._InactiveRpcError as e:
+                if "code: Some" in str(e):
+                    status_code = int(e.details().split("code: Some(")[1].split(")")[0])
+                    if (
+                        status_code == 201
+                    ):  # Already paid with this hash using different amount or destination
+                        # i don't think this can happen really, since we don't use the amount_msat in request
+                        # and if you just try 'pay' 2x where the first time it succeeds you get the same
+                        # non-error result the 2nd time.
+                        print(
+                            f"Order: {order.id} ALREADY PAID using different amount or destination THIS SHOULD NEVER HAPPEN! Hash: {hash}."
+                        )
+
+                    # Permanent failure at destination. or Unable to find a route. or Route too expensive.
+                    elif (
+                        status_code == 203
+                        or status_code == 205
+                        or status_code == 206
+                        or status_code == 210
+                    ):
+                        lnpayment.status = LNPayment.Status.FAILRO
+                        lnpayment.last_routing_time = timezone.now()
+                        lnpayment.routing_attempts += 1
+                        lnpayment.failure_reason = LNPayment.FailureReason.NOROUTE
+                        lnpayment.in_flight = False
+                        if lnpayment.routing_attempts > 2:
+                            lnpayment.status = LNPayment.Status.EXPIRE
+                            lnpayment.routing_attempts = 0
+                        lnpayment.save(
+                            update_fields=[
+                                "status",
+                                "last_routing_time",
+                                "routing_attempts",
+                                "in_flight",
+                                "failure_reason",
+                            ]
+                        )
+
+                        order.status = Order.Status.FAI
+                        order.expires_at = timezone.now() + timedelta(
+                            seconds=order.t_to_expire(Order.Status.FAI)
+                        )
+                        order.save(update_fields=["status", "expires_at"])
+                        print(
+                            f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[status_code]}"
+                        )
+                        return {
+                            "succeded": False,
+                            "context": f"payment failure reason: {cls.payment_failure_context[status_code]}",
+                        }
+                    elif status_code == 207:  # invoice expired
+                        print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
+
+                        last_payresponse = watchpayment()
+
+                        # check if succeeded while pending and expired
+                        if (
+                            len(last_payresponse.pays) > 0
+                            and last_payresponse.pays[0].status
+                            == noderpc.ListpaysPays.ListpaysPaysStatus.COMPLETE
+                        ):
+                            handle_response()
+                        else:
+                            lnpayment.status = LNPayment.Status.EXPIRE
+                            lnpayment.last_routing_time = timezone.now()
+                            lnpayment.in_flight = False
+                            lnpayment.save(
+                                update_fields=[
+                                    "status",
+                                    "last_routing_time",
+                                    "in_flight",
+                                ]
+                            )
+                            order.status = Order.Status.FAI
+                            order.expires_at = timezone.now() + timedelta(
+                                seconds=order.t_to_expire(Order.Status.FAI)
+                            )
+                            order.save(update_fields=["status", "expires_at"])
+                            results = {
+                                "succeded": False,
+                                "context": "The payout invoice has expired",
+                            }
+                            return results
+                    else:  # -1 (general error)
+                        print(str(e))
+                else:
+                    print(str(e))
+
+        handle_response()
+
+    @classmethod
+    def send_keysend(
+        cls, target_pubkey, message, num_satoshis, routing_budget_sats, timeout, sign
+    ):
+        # keysends for dev donations
+        from api.models import LNPayment
+
+        # Cannot perform selfpayments
+        # config("ALLOW_SELF_KEYSEND", cast=bool, default=False)
+
+        keysend_payment = {}
+        keysend_payment["created_at"] = timezone.now()
+        keysend_payment["expires_at"] = timezone.now()
+        try:
+            custom_records = []
+
+            msg = str(message)
+
+            if len(msg) > 0:
+                custom_records.append(
+                    primitives__pb2.TlvEntry(
+                        type=34349334, value=bytes.fromhex(msg.encode("utf-8").hex())
+                    )
+                )
+                if sign:
+                    self_pubkey = cls.stub.GetInfo(noderpc.GetinfoRequest()).id
+                    timestamp = struct.pack(">i", int(time.time()))
+                    signature = cls.stub.SignMessage(
+                        noderpc.SignmessageRequest(
+                            message=(
+                                bytes.fromhex(self_pubkey)
+                                + bytes.fromhex(target_pubkey)
+                                + timestamp
+                                + bytes.fromhex(msg.encode("utf-8").hex())
+                            ),
+                        )
+                    ).zbase
+                    custom_records.append(
+                        primitives__pb2.TlvEntry(type=34349337, value=signature)
+                    )
+                    custom_records.append(
+                        primitives__pb2.TlvEntry(
+                            type=34349339, value=bytes.fromhex(self_pubkey)
+                        )
+                    )
+                    custom_records.append(
+                        primitives__pb2.TlvEntry(type=34349343, value=timestamp)
+                    )
+
+            # no maxfee for Keysend
+            maxfeepercent = (routing_budget_sats / num_satoshis) * 100
+            request = noderpc.KeysendRequest(
+                destination=bytes.fromhex(target_pubkey),
+                extratlvs=primitives__pb2.TlvStream(entries=custom_records),
+                maxfeepercent=maxfeepercent,
+                retry_for=timeout,
+                amount_msat=primitives__pb2.Amount(msat=num_satoshis * 1000),
+            )
+            response = cls.stub.KeySend(request)
+
+            keysend_payment["preimage"] = response.payment_preimage.hex()
+            keysend_payment["payment_hash"] = response.payment_hash.hex()
+
+            waitreq = noderpc.WaitsendpayRequest(
+                payment_hash=response.payment_hash, timeout=timeout
+            )
+            try:
+                waitresp = cls.stub.WaitSendPay(waitreq)
+                keysend_payment["fee"] = (
+                    float(waitresp.amount_sent_msat.msat - waitresp.amount_msat.msat)
+                    / 1000
+                )
+                keysend_payment["status"] = LNPayment.Status.SUCCED
+            except grpc._channel._InactiveRpcError as e:
+                if "code: Some" in str(e):
+                    status_code = int(e.details().split("code: Some(")[1].split(")")[0])
+                if status_code == 200:  # Timed out before the payment could complete.
+                    keysend_payment["status"] = LNPayment.Status.FLIGHT
+                elif status_code == 208:
+                    print(
+                        f"A payment for {response.payment_hash.hex()} was never made and there is nothing to wait for"
+                    )
+                else:
+                    keysend_payment["status"] = LNPayment.Status.FAILRO
+                    keysend_payment["failure_reason"] = response.failure_reason
+            except Exception as e:
+                print("Error while sending keysend payment! Error: " + str(e))
+
+        except Exception as e:
+            print("Error while sending keysend payment! Error: " + str(e))
+
+        return True, keysend_payment
+
+    @classmethod
+    def double_check_htlc_is_settled(cls, payment_hash):
+        """Just as it sounds. Better safe than sorry!"""
+        request = noderpc.HodlInvoiceLookupRequest(
+            payment_hash=bytes.fromhex(payment_hash)
+        )
+        try:
+            response = cls.stub.HodlInvoiceLookup(request)
+        except Exception as e:
+            if "Timed out" in str(e):
+                return False
+            else:
+                raise e
+
+        return response.state == noderpc.HodlInvoiceLookupResponse.Hodlstate.SETTLED

api/lightning/lnd.py

@@ -0,0 +1,716 @@
+import hashlib
+import os
+import secrets
+import struct
+import time
+from base64 import b64decode
+from datetime import datetime, timedelta
+
+import grpc
+import ring
+from decouple import config
+from django.utils import timezone
+
+from . import invoices_pb2 as invoicesrpc
+from . import invoices_pb2_grpc as invoicesstub
+from . import lightning_pb2 as lnrpc
+from . import lightning_pb2_grpc as lightningstub
+from . import router_pb2 as routerrpc
+from . import router_pb2_grpc as routerstub
+from . import signer_pb2 as signerrpc
+from . import signer_pb2_grpc as signerstub
+from . import verrpc_pb2 as verrpc
+from . import verrpc_pb2_grpc as verstub
+
+#######
+# Works with LND (c-lightning in the future for multi-vendor resilience)
+#######
+
+# Read tls.cert from file or .env variable string encoded as base64
+try:
+    with open(os.path.join(config("LND_DIR"), "tls.cert"), "rb") as f:
+        CERT = f.read()
+except Exception:
+    CERT = b64decode(config("LND_CERT_BASE64"))
+
+# Read macaroon from file or .env variable string encoded as base64
+try:
+    with open(os.path.join(config("LND_DIR"), config("MACAROON_path")), "rb") as f:
+        MACAROON = f.read()
+except Exception:
+    MACAROON = b64decode(config("LND_MACAROON_BASE64"))
+
+LND_GRPC_HOST = config("LND_GRPC_HOST")
+DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
+MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000)
+
+
+class LNDNode:
+
+    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
+
+    def metadata_callback(context, callback):
+        callback([("macaroon", MACAROON.hex())], None)
+
+    ssl_creds = grpc.ssl_channel_credentials(CERT)
+    auth_creds = grpc.metadata_call_credentials(metadata_callback)
+    combined_creds = grpc.composite_channel_credentials(ssl_creds, auth_creds)
+    channel = grpc.secure_channel(LND_GRPC_HOST, combined_creds)
+
+    lightningstub = lightningstub.LightningStub(channel)
+    invoicesstub = invoicesstub.InvoicesStub(channel)
+    routerstub = routerstub.RouterStub(channel)
+    signerstub = signerstub.SignerStub(channel)
+    verstub = verstub.VersionerStub(channel)
+
+    payment_failure_context = {
+        0: "Payment isn't failed (yet)",
+        1: "There are more routes to try, but the payment timeout was exceeded.",
+        2: "All possible routes were tried and failed permanently. Or were no routes to the destination at all.",
+        3: "A non-recoverable error has occured.",
+        4: "Payment details incorrect (unknown hash, invalid amt or invalid final cltv delta)",
+        5: "Insufficient local balance.",
+    }
+
+    @classmethod
+    def get_version(cls):
+        try:
+            request = verrpc.VersionRequest()
+            response = cls.verstub.GetVersion(request)
+            return "v" + response.version
+        except Exception as e:
+            print(e)
+            return None
+
+    @classmethod
+    def decode_payreq(cls, invoice):
+        """Decodes a lightning payment request (invoice)"""
+        request = lnrpc.PayReqString(pay_req=invoice)
+        response = cls.lightningstub.DecodePayReq(request)
+        return response
+
+    @classmethod
+    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
+        """Returns estimated fee for onchain payouts"""
+
+        # We assume segwit. Use hardcoded address as shortcut so there is no need of user inputs yet.
+        request = lnrpc.EstimateFeeRequest(
+            AddrToAmount={"bc1qgxwaqe4m9mypd7ltww53yv3lyxhcfnhzzvy5j3": amount_sats},
+            target_conf=target_conf,
+            min_confs=min_confs,
+            spend_unconfirmed=False,
+        )
+
+        response = cls.lightningstub.EstimateFee(request)
+
+        return {
+            "mining_fee_sats": response.fee_sat,
+            "mining_fee_rate": response.sat_per_vbyte,
+        }
+
+    wallet_balance_cache = {}
+
+    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def wallet_balance(cls):
+        """Returns onchain balance"""
+        request = lnrpc.WalletBalanceRequest()
+        response = cls.lightningstub.WalletBalance(request)
+
+        return {
+            "total_balance": response.total_balance,
+            "confirmed_balance": response.confirmed_balance,
+            "unconfirmed_balance": response.unconfirmed_balance,
+        }
+
+    channel_balance_cache = {}
+
+    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def channel_balance(cls):
+        """Returns channels balance"""
+        request = lnrpc.ChannelBalanceRequest()
+        response = cls.lightningstub.ChannelBalance(request)
+
+        return {
+            "local_balance": response.local_balance.sat,
+            "remote_balance": response.remote_balance.sat,
+            "unsettled_local_balance": response.unsettled_local_balance.sat,
+            "unsettled_remote_balance": response.unsettled_remote_balance.sat,
+        }
+
+    @classmethod
+    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
+        """Send onchain transaction for buyer payouts"""
+
+        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
+            return False
+
+        request = lnrpc.SendCoinsRequest(
+            addr=onchainpayment.address,
+            amount=int(onchainpayment.sent_satoshis),
+            sat_per_vbyte=int(onchainpayment.mining_fee_rate),
+            label=str("Payout order #" + str(onchainpayment.order_paid_TX.id)),
+            spend_unconfirmed=config("SPEND_UNCONFIRMED", default=False, cast=bool),
+        )
+
+        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
+        delay = (
+            secrets.randbelow(2**256) / (2**256) * 10
+        )  # Random uniform 0 to 5 secs with good entropy
+        time.sleep(3 + delay)
+
+        if onchainpayment.status == queue_code:
+            # Changing the state to "MEMPO" should be atomic with SendCoins.
+            onchainpayment.status = on_mempool_code
+            onchainpayment.save(update_fields=["status"])
+            response = cls.lightningstub.SendCoins(request)
+
+            if response.txid:
+                onchainpayment.txid = response.txid
+                onchainpayment.broadcasted = True
+            onchainpayment.save(update_fields=["txid", "broadcasted"])
+            return True
+
+        elif onchainpayment.status == on_mempool_code:
+            # Bug, double payment attempted
+            return True
+
+    @classmethod
+    def cancel_return_hold_invoice(cls, payment_hash):
+        """Cancels or returns a hold invoice"""
+        request = invoicesrpc.CancelInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.invoicesstub.CancelInvoice(request)
+        # Fix this: tricky because canceling sucessfully an invoice has no response. TODO
+        return str(response) == ""  # True if no response, false otherwise.
+
+    @classmethod
+    def settle_hold_invoice(cls, preimage):
+        """settles a hold invoice"""
+        request = invoicesrpc.SettleInvoiceMsg(preimage=bytes.fromhex(preimage))
+        response = cls.invoicesstub.SettleInvoice(request)
+        # Fix this: tricky because settling sucessfully an invoice has None response. TODO
+        return str(response) == ""  # True if no response, false otherwise.
+
+    @classmethod
+    def gen_hold_invoice(cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id, lnpayment_concept, time):
+        """Generates hold invoice"""
+
+        hold_payment = {}
+        # The preimage is a random hash of 256 bits entropy
+        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
+
+        # Its hash is used to generate the hold invoice
+        r_hash = hashlib.sha256(preimage).digest()
+
+        request = invoicesrpc.AddHoldInvoiceRequest(
+            memo=description,
+            value=num_satoshis,
+            hash=r_hash,
+            expiry=int(
+                invoice_expiry * 1.5
+            ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
+            cltv_expiry=cltv_expiry_blocks,
+        )
+        response = cls.invoicesstub.AddHoldInvoice(request)
+
+        hold_payment["invoice"] = response.payment_request
+        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
+        hold_payment["preimage"] = preimage.hex()
+        hold_payment["payment_hash"] = payreq_decoded.payment_hash
+        hold_payment["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        hold_payment["expires_at"] = hold_payment["created_at"] + timedelta(
+            seconds=payreq_decoded.expiry
+        )
+        hold_payment["cltv_expiry"] = cltv_expiry_blocks
+
+        return hold_payment
+
+    @classmethod
+    def validate_hold_invoice_locked(cls, lnpayment):
+        """Checks if hold invoice is locked"""
+        from api.models import LNPayment
+
+        request = invoicesrpc.LookupInvoiceMsg(
+            payment_hash=bytes.fromhex(lnpayment.payment_hash)
+        )
+        response = cls.invoicesstub.LookupInvoiceV2(request)
+
+        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
+        # time has passed (but these are 15% padded at the moment). Should catch it
+        # and report back that the invoice has expired (better robustness)
+        if response.state == lnrpc.Invoice.InvoiceState.OPEN:  # OPEN
+            pass
+        if response.state == lnrpc.Invoice.InvoiceState.SETTLED:  # SETTLED
+            pass
+        if response.state == lnrpc.Invoice.InvoiceState.CANCELED:  # CANCELED
+            pass
+        if response.state == lnrpc.Invoice.InvoiceState.ACCEPTED:  # ACCEPTED (LOCKED)
+            lnpayment.expiry_height = response.htlcs[0].expiry_height
+            lnpayment.status = LNPayment.Status.LOCKED
+            lnpayment.save(update_fields=["expiry_height", "status"])
+            return True
+
+    @classmethod
+    def lookup_invoice_status(cls, lnpayment):
+        """
+        Returns the status (as LNpayment.Status) of the given payment_hash
+        If unchanged, returns the previous status
+        """
+        from api.models import LNPayment
+
+        status = lnpayment.status
+        expiry_height = 0
+
+        lnd_response_state_to_lnpayment_status = {
+            0: LNPayment.Status.INVGEN,  # OPEN
+            1: LNPayment.Status.SETLED,  # SETTLED
+            2: LNPayment.Status.CANCEL,  # CANCELED
+            3: LNPayment.Status.LOCKED,  # ACCEPTED
+        }
+
+        try:
+            # this is similar to LNNnode.validate_hold_invoice_locked
+            request = invoicesrpc.LookupInvoiceMsg(
+                payment_hash=bytes.fromhex(lnpayment.payment_hash)
+            )
+            response = cls.invoicesstub.LookupInvoiceV2(request)
+
+            status = lnd_response_state_to_lnpayment_status[response.state]
+
+            # get expiry height
+            if hasattr(response, "htlcs"):
+                try:
+                    for htlc in response.htlcs:
+                        expiry_height = max(expiry_height, htlc.expiry_height)
+                except Exception:
+                    pass
+
+        except Exception as e:
+            # If it fails at finding the invoice: it has been canceled.
+            # In RoboSats DB we make a distinction between CANCELED and returned (LND does not)
+            if "unable to locate invoice" in str(e):
+                print(str(e))
+                status = LNPayment.Status.CANCEL
+
+            # LND restarted.
+            if "wallet locked, unlock it" in str(e):
+                print(str(timezone.now()) + " :: Wallet Locked")
+
+            # Other write to logs
+            else:
+                print(str(e))
+
+        return status, expiry_height
+
+    @classmethod
+    def resetmc(cls):
+        request = routerrpc.ResetMissionControlRequest()
+        _ = cls.routerstub.ResetMissionControl(request)
+        return True
+
+    @classmethod
+    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
+        """Checks if the submited LN invoice comforms to expectations"""
+
+        payout = {
+            "valid": False,
+            "context": None,
+            "description": None,
+            "payment_hash": None,
+            "created_at": None,
+            "expires_at": None,
+        }
+
+        try:
+            payreq_decoded = cls.decode_payreq(invoice)
+        except Exception:
+            payout["context"] = {
+                "bad_invoice": "Does not look like a valid lightning invoice"
+            }
+            return payout
+
+        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
+        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
+        route_hints = payreq_decoded.route_hints
+
+        # Max amount RoboSats will pay for routing
+        if routing_budget_ppm == 0:
+            max_routing_fee_sats = max(
+                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        else:
+            max_routing_fee_sats = int(
+                float(num_satoshis) * float(routing_budget_ppm) / 1_000_000
+            )
+
+        if route_hints:
+            routes_cost = []
+            # For every hinted route...
+            for hinted_route in route_hints:
+                route_cost = 0
+                # ...add up the cost of every hinted hop...
+                for hop_hint in hinted_route.hop_hints:
+                    route_cost += hop_hint.fee_base_msat / 1000
+                    route_cost += (
+                        hop_hint.fee_proportional_millionths * num_satoshis / 1_000_000
+                    )
+
+                # ...and store the cost of the route to the array
+                routes_cost.append(route_cost)
+
+            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
+            if min(routes_cost) >= max_routing_fee_sats:
+                payout["context"] = {
+                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
+                }
+                return payout
+
+        if payreq_decoded.num_satoshis == 0:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has no explicit amount"
+            }
+            return payout
+
+        if not payreq_decoded.num_satoshis == num_satoshis:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided is not for "
+                + "{:,}".format(num_satoshis)
+                + " Sats"
+            }
+            return payout
+
+        payout["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        payout["expires_at"] = payout["created_at"] + timedelta(
+            seconds=payreq_decoded.expiry
+        )
+
+        if payout["expires_at"] < timezone.now():
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has already expired"
+            }
+            return payout
+
+        payout["valid"] = True
+        payout["description"] = payreq_decoded.description
+        payout["payment_hash"] = payreq_decoded.payment_hash
+
+        return payout
+
+    @classmethod
+    def pay_invoice(cls, lnpayment):
+        """Sends sats. Used for rewards payouts"""
+        from api.models import LNPayment
+
+        fee_limit_sat = int(
+            max(
+                lnpayment.num_satoshis
+                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        )  # 200 ppm or 10 sats
+        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
+        request = routerrpc.SendPaymentRequest(
+            payment_request=lnpayment.invoice,
+            fee_limit_sat=fee_limit_sat,
+            timeout_seconds=timeout_seconds,
+        )
+
+        for response in cls.routerstub.SendPaymentV2(request):
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.UNKNOWN
+            ):  # Status 0 'UNKNOWN'
+                # Not sure when this status happens
+                pass
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT
+            ):  # Status 1 'IN_FLIGHT'
+                pass
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.FAILED
+            ):  # Status 3 'FAILED'
+                """0	Payment isn't failed (yet).
+                1	There are more routes to try, but the payment timeout was exceeded.
+                2	All possible routes were tried and failed permanently. Or were no routes to the destination at all.
+                3	A non-recoverable error has occured.
+                4	Payment details incorrect (unknown hash, invalid amt or invalid final cltv delta)
+                5	Insufficient local balance.
+                """
+                failure_reason = cls.payment_failure_context[response.failure_reason]
+                lnpayment.failure_reason = response.failure_reason
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.save(update_fields=["failure_reason", "status"])
+                return False, failure_reason
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED
+            ):  # STATUS 'SUCCEEDED'
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.fee_msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save(update_fields=["fee", "status", "preimage"])
+                return True, None
+
+        return False
+
+    @classmethod
+    def follow_send_payment(cls, lnpayment, fee_limit_sat, timeout_seconds):
+        """
+        Sends sats to buyer, continuous update.
+        Has a lot of boilerplate to correctly handle every possible condition and failure case.
+        """
+        from api.models import LNPayment, Order
+
+        hash = lnpayment.payment_hash
+
+        request = routerrpc.SendPaymentRequest(
+            payment_request=lnpayment.invoice,
+            fee_limit_sat=fee_limit_sat,
+            timeout_seconds=timeout_seconds,
+            allow_self_payment=True,
+        )
+
+        order = lnpayment.order_paid_LN
+        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
+            print(f"Order: {order.id} Payout is larger than collateral !?")
+            return
+
+        def handle_response(response, was_in_transit=False):
+            lnpayment.status = LNPayment.Status.FLIGHT
+            lnpayment.in_flight = True
+            lnpayment.save(update_fields=["in_flight", "status"])
+            order.status = Order.Status.PAY
+            order.save(update_fields=["status"])
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.UNKNOWN
+            ):  # Status 0 'UNKNOWN'
+                # Not sure when this status happens
+                print(f"Order: {order.id} UNKNOWN. Hash {hash}")
+                lnpayment.in_flight = False
+                lnpayment.save(update_fields=["in_flight"])
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT
+            ):  # Status 1 'IN_FLIGHT'
+                print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
+
+                # If payment was already "payment is in transition" we do not
+                # want to spawn a new thread every 3 minutes to check on it.
+                # in case this thread dies, let's move the last_routing_time
+                # 20 minutes in the future so another thread spawns.
+                if was_in_transit:
+                    lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
+                    lnpayment.save(update_fields=["last_routing_time"])
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.FAILED
+            ):  # Status 3 'FAILED'
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.last_routing_time = timezone.now()
+                lnpayment.routing_attempts += 1
+                lnpayment.failure_reason = response.failure_reason
+                lnpayment.in_flight = False
+                if lnpayment.routing_attempts > 2:
+                    lnpayment.status = LNPayment.Status.EXPIRE
+                    lnpayment.routing_attempts = 0
+                lnpayment.save(
+                    update_fields=[
+                        "status",
+                        "last_routing_time",
+                        "routing_attempts",
+                        "failure_reason",
+                        "in_flight",
+                    ]
+                )
+
+                order.status = Order.Status.FAI
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.FAI)
+                )
+                order.save(update_fields=["status", "expires_at"])
+                print(
+                    f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[response.failure_reason]}"
+                )
+                return {
+                    "succeded": False,
+                    "context": f"payment failure reason: {cls.payment_failure_context[response.failure_reason]}",
+                }
+
+            if (
+                response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED
+            ):  # Status 2 'SUCCEEDED'
+                print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.fee_msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save(update_fields=["status", "fee", "preimage"])
+
+                order.status = Order.Status.SUC
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.SUC)
+                )
+                order.save(update_fields=["status", "expires_at"])
+
+                results = {"succeded": True}
+                return results
+
+        try:
+            for response in cls.routerstub.SendPaymentV2(request):
+
+                handle_response(response)
+
+        except Exception as e:
+
+            if "invoice expired" in str(e):
+                print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
+                # An expired invoice can already be in-flight. Check.
+                try:
+                    request = routerrpc.TrackPaymentRequest(
+                        payment_hash=bytes.fromhex(hash)
+                    )
+
+                    for response in cls.routerstub.TrackPaymentV2(request):
+                        handle_response(response, was_in_transit=True)
+
+                except Exception as e:
+                    if "payment isn't initiated" in str(e):
+                        print(
+                            f"Order: {order.id}. The expired invoice had not been initiated. Hash: {hash}"
+                        )
+
+                        lnpayment.status = LNPayment.Status.EXPIRE
+                        lnpayment.last_routing_time = timezone.now()
+                        lnpayment.in_flight = False
+                        lnpayment.save(
+                            update_fields=["status", "last_routing_time", "in_flight"]
+                        )
+
+                        order.status = Order.Status.FAI
+                        order.expires_at = timezone.now() + timedelta(
+                            seconds=order.t_to_expire(Order.Status.FAI)
+                        )
+                        order.save(update_fields=["status", "expires_at"])
+
+                        results = {
+                            "succeded": False,
+                            "context": "The payout invoice has expired",
+                        }
+                        return results
+
+            elif "payment is in transition" in str(e):
+                print(f"Order: {order.id} ALREADY IN TRANSITION. Hash: {hash}.")
+
+                request = routerrpc.TrackPaymentRequest(
+                    payment_hash=bytes.fromhex(hash)
+                )
+
+                for response in cls.routerstub.TrackPaymentV2(request):
+                    handle_response(response, was_in_transit=True)
+
+            elif "invoice is already paid" in str(e):
+                print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
+
+                request = routerrpc.TrackPaymentRequest(
+                    payment_hash=bytes.fromhex(hash)
+                )
+
+                for response in cls.routerstub.TrackPaymentV2(request):
+                    handle_response(response)
+
+            else:
+                print(str(e))
+
+    @classmethod
+    def send_keysend(
+        cls, target_pubkey, message, num_satoshis, routing_budget_sats, timeout, sign
+    ):
+        # Thank you @cryptosharks131 / lndg for the inspiration
+        # Source https://github.com/cryptosharks131/lndg/blob/master/keysend.py
+
+        from api.models import LNPayment
+
+        ALLOW_SELF_KEYSEND = config("ALLOW_SELF_KEYSEND", cast=bool, default=False)
+        keysend_payment = {}
+        keysend_payment["created_at"] = timezone.now()
+        keysend_payment["expires_at"] = timezone.now()
+        try:
+            secret = secrets.token_bytes(32)
+            hashed_secret = hashlib.sha256(secret).hexdigest()
+            custom_records = [
+                (5482373484, secret),
+            ]
+            keysend_payment["preimage"] = secret.hex()
+            keysend_payment["payment_hash"] = hashed_secret
+
+            msg = str(message)
+
+            if len(msg) > 0:
+                custom_records.append(
+                    (34349334, bytes.fromhex(msg.encode("utf-8").hex()))
+                )
+                if sign:
+                    self_pubkey = cls.lightningstub.GetInfo(
+                        lnrpc.GetInfoRequest()
+                    ).identity_pubkey
+                    timestamp = struct.pack(">i", int(time.time()))
+                    signature = cls.signerstub.SignMessage(
+                        signerrpc.SignMessageReq(
+                            msg=(
+                                bytes.fromhex(self_pubkey)
+                                + bytes.fromhex(target_pubkey)
+                                + timestamp
+                                + bytes.fromhex(msg.encode("utf-8").hex())
+                            ),
+                            key_loc=signerrpc.KeyLocator(key_family=6, key_index=0),
+                        )
+                    ).signature
+                    custom_records.append((34349337, signature))
+                    custom_records.append((34349339, bytes.fromhex(self_pubkey)))
+                    custom_records.append((34349343, timestamp))
+
+            request = routerrpc.SendPaymentRequest(
+                dest=bytes.fromhex(target_pubkey),
+                dest_custom_records=custom_records,
+                fee_limit_sat=routing_budget_sats,
+                timeout_seconds=timeout,
+                amt=num_satoshis,
+                payment_hash=bytes.fromhex(hashed_secret),
+                allow_self_payment=ALLOW_SELF_KEYSEND,
+            )
+            for response in cls.routerstub.SendPaymentV2(request):
+                if response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT:
+                    keysend_payment["status"] = LNPayment.Status.FLIGHT
+                if response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED:
+                    keysend_payment["fee"] = float(response.fee_msat) / 1000
+                    keysend_payment["status"] = LNPayment.Status.SUCCED
+                if response.status == lnrpc.Payment.PaymentStatus.FAILED:
+                    keysend_payment["status"] = LNPayment.Status.FAILRO
+                    keysend_payment["failure_reason"] = response.failure_reason
+                if response.status == lnrpc.Payment.PaymentStatus.UNKNOWN:
+                    print("Unknown Error")
+        except Exception as e:
+            if "self-payments not allowed" in str(e):
+                print("Self keysend is not allowed")
+            else:
+                print("Error while sending keysend payment! Error: " + str(e))
+
+        return True, keysend_payment
+
+    @classmethod
+    def double_check_htlc_is_settled(cls, payment_hash):
+        """Just as it sounds. Better safe than sorry!"""
+        request = invoicesrpc.LookupInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.invoicesstub.LookupInvoiceV2(request)
+
+        return (
+            response.state == lnrpc.Invoice.InvoiceState.SETTLED
+        )  # LND states: 0 OPEN, 1 SETTLED, 3 ACCEPTED, GRPC_ERROR status 5 when CANCELED/returned

api/lightning/node.py

@@ -1,718 +1,16 @@
-import hashlib
-import os
-import secrets
-import struct
-import time
-from base64 import b64decode
-from datetime import datetime, timedelta
-
-import grpc
-import ring
 from decouple import config
-from django.utils import timezone
 
-from . import invoices_pb2 as invoicesrpc
+LN_vendor = config("LNVENDOR", cast=str, default="LND")
-from . import invoices_pb2_grpc as invoicesstub
-from . import lightning_pb2 as lnrpc
-from . import lightning_pb2_grpc as lightningstub
-from . import router_pb2 as routerrpc
-from . import router_pb2_grpc as routerstub
-from . import signer_pb2 as signerrpc
-from . import signer_pb2_grpc as signerstub
-from . import verrpc_pb2 as verrpc
-from . import verrpc_pb2_grpc as verstub
 
-#######
+if LN_vendor == "LND":
-# Works with LND (c-lightning in the future for multi-vendor resilience)
+    from api.lightning.lnd import LNDNode
-#######
 
-# Read tls.cert from file or .env variable string encoded as base64
+    LNNode = LNDNode
-try:
+elif LN_vendor == "CLN":
-    with open(os.path.join(config("LND_DIR"), "tls.cert"), "rb") as f:
+    from api.lightning.cln import CLNNode
-        CERT = f.read()
-except Exception:
-    CERT = b64decode(config("LND_CERT_BASE64"))
 
-# Read macaroon from file or .env variable string encoded as base64
+    LNNode = CLNNode
-try:
+else:
-    with open(os.path.join(config("LND_DIR"), config("MACAROON_path")), "rb") as f:
+    raise ValueError(
-        MACAROON = f.read()
+        f'Invalid Lightning Node vendor: {LN_vendor}. Must be either "LND" or "CLN"'
-except Exception:
+    )
-    MACAROON = b64decode(config("LND_MACAROON_BASE64"))
-
-LND_GRPC_HOST = config("LND_GRPC_HOST")
-DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
-MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000)
-
-
-class LNNode:
-
-    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
-
-    def metadata_callback(context, callback):
-        callback([("macaroon", MACAROON.hex())], None)
-
-    ssl_creds = grpc.ssl_channel_credentials(CERT)
-    auth_creds = grpc.metadata_call_credentials(metadata_callback)
-    combined_creds = grpc.composite_channel_credentials(ssl_creds, auth_creds)
-    channel = grpc.secure_channel(LND_GRPC_HOST, combined_creds)
-
-    lightningstub = lightningstub.LightningStub(channel)
-    invoicesstub = invoicesstub.InvoicesStub(channel)
-    routerstub = routerstub.RouterStub(channel)
-    signerstub = signerstub.SignerStub(channel)
-    verstub = verstub.VersionerStub(channel)
-
-    payment_failure_context = {
-        0: "Payment isn't failed (yet)",
-        1: "There are more routes to try, but the payment timeout was exceeded.",
-        2: "All possible routes were tried and failed permanently. Or were no routes to the destination at all.",
-        3: "A non-recoverable error has occured.",
-        4: "Payment details incorrect (unknown hash, invalid amt or invalid final cltv delta)",
-        5: "Insufficient local balance.",
-    }
-
-    @classmethod
-    def get_version(cls):
-        try:
-            request = verrpc.VersionRequest()
-            response = cls.verstub.GetVersion(request)
-            return "v" + response.version
-        except Exception as e:
-            print(e)
-            return None
-
-    @classmethod
-    def decode_payreq(cls, invoice):
-        """Decodes a lightning payment request (invoice)"""
-        request = lnrpc.PayReqString(pay_req=invoice)
-        response = cls.lightningstub.DecodePayReq(request)
-        return response
-
-    @classmethod
-    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
-        """Returns estimated fee for onchain payouts"""
-
-        # We assume segwit. Use hardcoded address as shortcut so there is no need of user inputs yet.
-        request = lnrpc.EstimateFeeRequest(
-            AddrToAmount={"bc1qgxwaqe4m9mypd7ltww53yv3lyxhcfnhzzvy5j3": amount_sats},
-            target_conf=target_conf,
-            min_confs=min_confs,
-            spend_unconfirmed=False,
-        )
-
-        response = cls.lightningstub.EstimateFee(request)
-
-        return {
-            "mining_fee_sats": response.fee_sat,
-            "mining_fee_rate": response.sat_per_vbyte,
-        }
-
-    wallet_balance_cache = {}
-
-    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
-    @classmethod
-    def wallet_balance(cls):
-        """Returns onchain balance"""
-        request = lnrpc.WalletBalanceRequest()
-        response = cls.lightningstub.WalletBalance(request)
-
-        return {
-            "total_balance": response.total_balance,
-            "confirmed_balance": response.confirmed_balance,
-            "unconfirmed_balance": response.unconfirmed_balance,
-        }
-
-    channel_balance_cache = {}
-
-    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
-    @classmethod
-    def channel_balance(cls):
-        """Returns channels balance"""
-        request = lnrpc.ChannelBalanceRequest()
-        response = cls.lightningstub.ChannelBalance(request)
-
-        return {
-            "local_balance": response.local_balance.sat,
-            "remote_balance": response.remote_balance.sat,
-            "unsettled_local_balance": response.unsettled_local_balance.sat,
-            "unsettled_remote_balance": response.unsettled_remote_balance.sat,
-        }
-
-    @classmethod
-    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
-        """Send onchain transaction for buyer payouts"""
-
-        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
-            return False
-
-        request = lnrpc.SendCoinsRequest(
-            addr=onchainpayment.address,
-            amount=int(onchainpayment.sent_satoshis),
-            sat_per_vbyte=int(onchainpayment.mining_fee_rate),
-            label=str("Payout order #" + str(onchainpayment.order_paid_TX.id)),
-            spend_unconfirmed=config("SPEND_UNCONFIRMED", default=False, cast=bool),
-        )
-
-        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
-        delay = (
-            secrets.randbelow(2**256) / (2**256) * 10
-        )  # Random uniform 0 to 5 secs with good entropy
-        time.sleep(3 + delay)
-
-        if onchainpayment.status == queue_code:
-            # Changing the state to "MEMPO" should be atomic with SendCoins.
-            onchainpayment.status = on_mempool_code
-            onchainpayment.save(update_fields=["status"])
-            response = cls.lightningstub.SendCoins(request)
-
-            if response.txid:
-                onchainpayment.txid = response.txid
-                onchainpayment.broadcasted = True
-            onchainpayment.save(update_fields=["txid", "broadcasted"])
-            return True
-
-        elif onchainpayment.status == on_mempool_code:
-            # Bug, double payment attempted
-            return True
-
-    @classmethod
-    def cancel_return_hold_invoice(cls, payment_hash):
-        """Cancels or returns a hold invoice"""
-        request = invoicesrpc.CancelInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
-        response = cls.invoicesstub.CancelInvoice(request)
-        # Fix this: tricky because canceling sucessfully an invoice has no response. TODO
-        return str(response) == ""  # True if no response, false otherwise.
-
-    @classmethod
-    def settle_hold_invoice(cls, preimage):
-        """settles a hold invoice"""
-        request = invoicesrpc.SettleInvoiceMsg(preimage=bytes.fromhex(preimage))
-        response = cls.invoicesstub.SettleInvoice(request)
-        # Fix this: tricky because settling sucessfully an invoice has None response. TODO
-        return str(response) == ""  # True if no response, false otherwise.
-
-    @classmethod
-    def gen_hold_invoice(
-        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks
-    ):
-        """Generates hold invoice"""
-
-        hold_payment = {}
-        # The preimage is a random hash of 256 bits entropy
-        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
-
-        # Its hash is used to generate the hold invoice
-        r_hash = hashlib.sha256(preimage).digest()
-
-        request = invoicesrpc.AddHoldInvoiceRequest(
-            memo=description,
-            value=num_satoshis,
-            hash=r_hash,
-            expiry=int(
-                invoice_expiry * 1.5
-            ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
-            cltv_expiry=cltv_expiry_blocks,
-        )
-        response = cls.invoicesstub.AddHoldInvoice(request)
-
-        hold_payment["invoice"] = response.payment_request
-        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
-        hold_payment["preimage"] = preimage.hex()
-        hold_payment["payment_hash"] = payreq_decoded.payment_hash
-        hold_payment["created_at"] = timezone.make_aware(
-            datetime.fromtimestamp(payreq_decoded.timestamp)
-        )
-        hold_payment["expires_at"] = hold_payment["created_at"] + timedelta(
-            seconds=payreq_decoded.expiry
-        )
-        hold_payment["cltv_expiry"] = cltv_expiry_blocks
-
-        return hold_payment
-
-    @classmethod
-    def validate_hold_invoice_locked(cls, lnpayment):
-        """Checks if hold invoice is locked"""
-        from api.models import LNPayment
-
-        request = invoicesrpc.LookupInvoiceMsg(
-            payment_hash=bytes.fromhex(lnpayment.payment_hash)
-        )
-        response = cls.invoicesstub.LookupInvoiceV2(request)
-
-        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
-        # time has passed (but these are 15% padded at the moment). Should catch it
-        # and report back that the invoice has expired (better robustness)
-        if response.state == lnrpc.Invoice.InvoiceState.OPEN:  # OPEN
-            pass
-        if response.state == lnrpc.Invoice.InvoiceState.SETTLED:  # SETTLED
-            pass
-        if response.state == lnrpc.Invoice.InvoiceState.CANCELED:  # CANCELED
-            pass
-        if response.state == lnrpc.Invoice.InvoiceState.ACCEPTED:  # ACCEPTED (LOCKED)
-            lnpayment.expiry_height = response.htlcs[0].expiry_height
-            lnpayment.status = LNPayment.Status.LOCKED
-            lnpayment.save(update_fields=["expiry_height", "status"])
-            return True
-
-    @classmethod
-    def lookup_invoice_status(cls, lnpayment):
-        """
-        Returns the status (as LNpayment.Status) of the given payment_hash
-        If unchanged, returns the previous status
-        """
-        from api.models import LNPayment
-
-        status = lnpayment.status
-        expiry_height = 0
-
-        lnd_response_state_to_lnpayment_status = {
-            0: LNPayment.Status.INVGEN,  # OPEN
-            1: LNPayment.Status.SETLED,  # SETTLED
-            2: LNPayment.Status.CANCEL,  # CANCELED
-            3: LNPayment.Status.LOCKED,  # ACCEPTED
-        }
-
-        try:
-            # this is similar to LNNnode.validate_hold_invoice_locked
-            request = invoicesrpc.LookupInvoiceMsg(
-                payment_hash=bytes.fromhex(lnpayment.payment_hash)
-            )
-            response = cls.invoicesstub.LookupInvoiceV2(request)
-
-            status = lnd_response_state_to_lnpayment_status[response.state]
-
-            # get expiry height
-            if hasattr(response, "htlcs"):
-                try:
-                    for htlc in response.htlcs:
-                        expiry_height = max(expiry_height, htlc.expiry_height)
-                except Exception:
-                    pass
-
-        except Exception as e:
-            # If it fails at finding the invoice: it has been canceled.
-            # In RoboSats DB we make a distinction between CANCELED and returned (LND does not)
-            if "unable to locate invoice" in str(e):
-                print(str(e))
-                status = LNPayment.Status.CANCEL
-
-            # LND restarted.
-            if "wallet locked, unlock it" in str(e):
-                print(str(timezone.now()) + " :: Wallet Locked")
-
-            # Other write to logs
-            else:
-                print(str(e))
-
-        return status, expiry_height
-
-    @classmethod
-    def resetmc(cls):
-        request = routerrpc.ResetMissionControlRequest()
-        _ = cls.routerstub.ResetMissionControl(request)
-        return True
-
-    @classmethod
-    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
-        """Checks if the submited LN invoice comforms to expectations"""
-
-        payout = {
-            "valid": False,
-            "context": None,
-            "description": None,
-            "payment_hash": None,
-            "created_at": None,
-            "expires_at": None,
-        }
-
-        try:
-            payreq_decoded = cls.decode_payreq(invoice)
-        except Exception:
-            payout["context"] = {
-                "bad_invoice": "Does not look like a valid lightning invoice"
-            }
-            return payout
-
-        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
-        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
-        route_hints = payreq_decoded.route_hints
-
-        # Max amount RoboSats will pay for routing
-        if routing_budget_ppm == 0:
-            max_routing_fee_sats = max(
-                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
-                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
-            )
-        else:
-            max_routing_fee_sats = int(
-                float(num_satoshis) * float(routing_budget_ppm) / 1_000_000
-            )
-
-        if route_hints:
-            routes_cost = []
-            # For every hinted route...
-            for hinted_route in route_hints:
-                route_cost = 0
-                # ...add up the cost of every hinted hop...
-                for hop_hint in hinted_route.hop_hints:
-                    route_cost += hop_hint.fee_base_msat / 1000
-                    route_cost += (
-                        hop_hint.fee_proportional_millionths * num_satoshis / 1_000_000
-                    )
-
-                # ...and store the cost of the route to the array
-                routes_cost.append(route_cost)
-
-            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
-            if min(routes_cost) >= max_routing_fee_sats:
-                payout["context"] = {
-                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
-                }
-                return payout
-
-        if payreq_decoded.num_satoshis == 0:
-            payout["context"] = {
-                "bad_invoice": "The invoice provided has no explicit amount"
-            }
-            return payout
-
-        if not payreq_decoded.num_satoshis == num_satoshis:
-            payout["context"] = {
-                "bad_invoice": "The invoice provided is not for "
-                + "{:,}".format(num_satoshis)
-                + " Sats"
-            }
-            return payout
-
-        payout["created_at"] = timezone.make_aware(
-            datetime.fromtimestamp(payreq_decoded.timestamp)
-        )
-        payout["expires_at"] = payout["created_at"] + timedelta(
-            seconds=payreq_decoded.expiry
-        )
-
-        if payout["expires_at"] < timezone.now():
-            payout["context"] = {
-                "bad_invoice": "The invoice provided has already expired"
-            }
-            return payout
-
-        payout["valid"] = True
-        payout["description"] = payreq_decoded.description
-        payout["payment_hash"] = payreq_decoded.payment_hash
-
-        return payout
-
-    @classmethod
-    def pay_invoice(cls, lnpayment):
-        """Sends sats. Used for rewards payouts"""
-        from api.models import LNPayment
-
-        fee_limit_sat = int(
-            max(
-                lnpayment.num_satoshis
-                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
-                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
-            )
-        )  # 200 ppm or 10 sats
-        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
-        request = routerrpc.SendPaymentRequest(
-            payment_request=lnpayment.invoice,
-            fee_limit_sat=fee_limit_sat,
-            timeout_seconds=timeout_seconds,
-        )
-
-        for response in cls.routerstub.SendPaymentV2(request):
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.UNKNOWN
-            ):  # Status 0 'UNKNOWN'
-                # Not sure when this status happens
-                pass
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT
-            ):  # Status 1 'IN_FLIGHT'
-                pass
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.FAILED
-            ):  # Status 3 'FAILED'
-                """0	Payment isn't failed (yet).
-                1	There are more routes to try, but the payment timeout was exceeded.
-                2	All possible routes were tried and failed permanently. Or were no routes to the destination at all.
-                3	A non-recoverable error has occured.
-                4	Payment details incorrect (unknown hash, invalid amt or invalid final cltv delta)
-                5	Insufficient local balance.
-                """
-                failure_reason = cls.payment_failure_context[response.failure_reason]
-                lnpayment.failure_reason = response.failure_reason
-                lnpayment.status = LNPayment.Status.FAILRO
-                lnpayment.save(update_fields=["failure_reason", "status"])
-                return False, failure_reason
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED
-            ):  # STATUS 'SUCCEEDED'
-                lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.fee_msat) / 1000
-                lnpayment.preimage = response.payment_preimage
-                lnpayment.save(update_fields=["fee", "status", "preimage"])
-                return True, None
-
-        return False
-
-    @classmethod
-    def follow_send_payment(cls, lnpayment, fee_limit_sat, timeout_seconds):
-        """
-        Sends sats to buyer, continuous update.
-        Has a lot of boilerplate to correctly handle every possible condition and failure case.
-        """
-        from api.models import LNPayment, Order
-
-        hash = lnpayment.payment_hash
-
-        request = routerrpc.SendPaymentRequest(
-            payment_request=lnpayment.invoice,
-            fee_limit_sat=fee_limit_sat,
-            timeout_seconds=timeout_seconds,
-            allow_self_payment=True,
-        )
-
-        order = lnpayment.order_paid_LN
-        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
-            print(f"Order: {order.id} Payout is larger than collateral !?")
-            return
-
-        def handle_response(response, was_in_transit=False):
-            lnpayment.status = LNPayment.Status.FLIGHT
-            lnpayment.in_flight = True
-            lnpayment.save(update_fields=["in_flight", "status"])
-            order.status = Order.Status.PAY
-            order.save(update_fields=["status"])
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.UNKNOWN
-            ):  # Status 0 'UNKNOWN'
-                # Not sure when this status happens
-                print(f"Order: {order.id} UNKNOWN. Hash {hash}")
-                lnpayment.in_flight = False
-                lnpayment.save(update_fields=["in_flight"])
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT
-            ):  # Status 1 'IN_FLIGHT'
-                print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
-
-                # If payment was already "payment is in transition" we do not
-                # want to spawn a new thread every 3 minutes to check on it.
-                # in case this thread dies, let's move the last_routing_time
-                # 20 minutes in the future so another thread spawns.
-                if was_in_transit:
-                    lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
-                    lnpayment.save(update_fields=["last_routing_time"])
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.FAILED
-            ):  # Status 3 'FAILED'
-                lnpayment.status = LNPayment.Status.FAILRO
-                lnpayment.last_routing_time = timezone.now()
-                lnpayment.routing_attempts += 1
-                lnpayment.failure_reason = response.failure_reason
-                lnpayment.in_flight = False
-                if lnpayment.routing_attempts > 2:
-                    lnpayment.status = LNPayment.Status.EXPIRE
-                    lnpayment.routing_attempts = 0
-                lnpayment.save(
-                    update_fields=[
-                        "status",
-                        "last_routing_time",
-                        "routing_attempts",
-                        "failure_reason",
-                        "in_flight",
-                    ]
-                )
-
-                order.status = Order.Status.FAI
-                order.expires_at = timezone.now() + timedelta(
-                    seconds=order.t_to_expire(Order.Status.FAI)
-                )
-                order.save(update_fields=["status", "expires_at"])
-                print(
-                    f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[response.failure_reason]}"
-                )
-                return {
-                    "succeded": False,
-                    "context": f"payment failure reason: {cls.payment_failure_context[response.failure_reason]}",
-                }
-
-            if (
-                response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED
-            ):  # Status 2 'SUCCEEDED'
-                print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
-                lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.fee_msat) / 1000
-                lnpayment.preimage = response.payment_preimage
-                lnpayment.save(update_fields=["status", "fee", "preimage"])
-
-                order.status = Order.Status.SUC
-                order.expires_at = timezone.now() + timedelta(
-                    seconds=order.t_to_expire(Order.Status.SUC)
-                )
-                order.save(update_fields=["status", "expires_at"])
-
-                results = {"succeded": True}
-                return results
-
-        try:
-            for response in cls.routerstub.SendPaymentV2(request):
-
-                handle_response(response)
-
-        except Exception as e:
-
-            if "invoice expired" in str(e):
-                print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
-                # An expired invoice can already be in-flight. Check.
-                try:
-                    request = routerrpc.TrackPaymentRequest(
-                        payment_hash=bytes.fromhex(hash)
-                    )
-
-                    for response in cls.routerstub.TrackPaymentV2(request):
-                        handle_response(response, was_in_transit=True)
-
-                except Exception as e:
-                    if "payment isn't initiated" in str(e):
-                        print(
-                            f"Order: {order.id}. The expired invoice had not been initiated. Hash: {hash}"
-                        )
-
-                        lnpayment.status = LNPayment.Status.EXPIRE
-                        lnpayment.last_routing_time = timezone.now()
-                        lnpayment.in_flight = False
-                        lnpayment.save(
-                            update_fields=["status", "last_routing_time", "in_flight"]
-                        )
-
-                        order.status = Order.Status.FAI
-                        order.expires_at = timezone.now() + timedelta(
-                            seconds=order.t_to_expire(Order.Status.FAI)
-                        )
-                        order.save(update_fields=["status", "expires_at"])
-
-                        results = {
-                            "succeded": False,
-                            "context": "The payout invoice has expired",
-                        }
-                        return results
-
-            elif "payment is in transition" in str(e):
-                print(f"Order: {order.id} ALREADY IN TRANSITION. Hash: {hash}.")
-
-                request = routerrpc.TrackPaymentRequest(
-                    payment_hash=bytes.fromhex(hash)
-                )
-
-                for response in cls.routerstub.TrackPaymentV2(request):
-                    handle_response(response, was_in_transit=True)
-
-            elif "invoice is already paid" in str(e):
-                print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
-
-                request = routerrpc.TrackPaymentRequest(
-                    payment_hash=bytes.fromhex(hash)
-                )
-
-                for response in cls.routerstub.TrackPaymentV2(request):
-                    handle_response(response)
-
-            else:
-                print(str(e))
-
-    @classmethod
-    def send_keysend(
-        cls, target_pubkey, message, num_satoshis, routing_budget_sats, timeout, sign
-    ):
-        # Thank you @cryptosharks131 / lndg for the inspiration
-        # Source https://github.com/cryptosharks131/lndg/blob/master/keysend.py
-
-        from api.models import LNPayment
-
-        ALLOW_SELF_KEYSEND = config("ALLOW_SELF_KEYSEND", cast=bool, default=False)
-        keysend_payment = {}
-        keysend_payment["created_at"] = timezone.now()
-        keysend_payment["expires_at"] = timezone.now()
-        try:
-            secret = secrets.token_bytes(32)
-            hashed_secret = hashlib.sha256(secret).hexdigest()
-            custom_records = [
-                (5482373484, secret),
-            ]
-            keysend_payment["preimage"] = secret.hex()
-            keysend_payment["payment_hash"] = hashed_secret
-
-            msg = str(message)
-
-            if len(msg) > 0:
-                custom_records.append(
-                    (34349334, bytes.fromhex(msg.encode("utf-8").hex()))
-                )
-                if sign:
-                    self_pubkey = cls.lightningstub.GetInfo(
-                        lnrpc.GetInfoRequest()
-                    ).identity_pubkey
-                    timestamp = struct.pack(">i", int(time.time()))
-                    signature = cls.signerstub.SignMessage(
-                        signerrpc.SignMessageReq(
-                            msg=(
-                                bytes.fromhex(self_pubkey)
-                                + bytes.fromhex(target_pubkey)
-                                + timestamp
-                                + bytes.fromhex(msg.encode("utf-8").hex())
-                            ),
-                            key_loc=signerrpc.KeyLocator(key_family=6, key_index=0),
-                        )
-                    ).signature
-                    custom_records.append((34349337, signature))
-                    custom_records.append((34349339, bytes.fromhex(self_pubkey)))
-                    custom_records.append((34349343, timestamp))
-
-            request = routerrpc.SendPaymentRequest(
-                dest=bytes.fromhex(target_pubkey),
-                dest_custom_records=custom_records,
-                fee_limit_sat=routing_budget_sats,
-                timeout_seconds=timeout,
-                amt=num_satoshis,
-                payment_hash=bytes.fromhex(hashed_secret),
-                allow_self_payment=ALLOW_SELF_KEYSEND,
-            )
-            for response in cls.routerstub.SendPaymentV2(request):
-                if response.status == lnrpc.Payment.PaymentStatus.IN_FLIGHT:
-                    keysend_payment["status"] = LNPayment.Status.FLIGHT
-                if response.status == lnrpc.Payment.PaymentStatus.SUCCEEDED:
-                    keysend_payment["fee"] = float(response.fee_msat) / 1000
-                    keysend_payment["status"] = LNPayment.Status.SUCCED
-                if response.status == lnrpc.Payment.PaymentStatus.FAILED:
-                    keysend_payment["status"] = LNPayment.Status.FAILRO
-                    keysend_payment["failure_reason"] = response.failure_reason
-                if response.status == lnrpc.Payment.PaymentStatus.UNKNOWN:
-                    print("Unknown Error")
-        except Exception as e:
-            if "self-payments not allowed" in str(e):
-                print("Self keysend is not allowed")
-            else:
-                print("Error while sending keysend payment! Error: " + str(e))
-
-        return True, keysend_payment
-
-    @classmethod
-    def double_check_htlc_is_settled(cls, payment_hash):
-        """Just as it sounds. Better safe than sorry!"""
-        request = invoicesrpc.LookupInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
-        response = cls.invoicesstub.LookupInvoiceV2(request)
-
-        return (
-            response.state == lnrpc.Invoice.InvoiceState.SETTLED
-        )  # LND states: 0 OPEN, 1 SETTLED, 3 ACCEPTED, GRPC_ERROR status 5 when CANCELED/returned

api/logics.py

@@ -559,7 +559,8 @@ class Logics:
         # Compute a safer available  onchain liquidity: (confirmed_utxos - reserve - pending_outgoing_txs))
         # Accounts for already committed outgoing TX for previous users.
         confirmed = onchain_payment.balance.onchain_confirmed
-        reserve = 300_000  # We assume a reserve of 300K Sats (3 times higher than LND's default anchor reserve)
+        # We assume a reserve of 300K Sats (3 times higher than LND's default anchor reserve)
+        reserve = 300_000
         pending_txs = OnchainPayment.objects.filter(
             status__in=[OnchainPayment.Status.VALID, OnchainPayment.Status.QUEUE]
         ).aggregate(Sum("num_satoshis"))["num_satoshis__sum"]
@@ -790,7 +791,8 @@ class Logics:
             concept=LNPayment.Concepts.PAYBUYER,
             type=LNPayment.Types.NORM,
             sender=User.objects.get(username=ESCROW_USERNAME),
-            order_paid_LN=order,  # In case this user has other payouts, update the one related to this order.
+            # In case this user has other payouts, update the one related to this order.
+            order_paid_LN=order,
             receiver=user,
             routing_budget_ppm=routing_budget_ppm,
             routing_budget_sats=routing_budget_sats,
@@ -1097,6 +1099,9 @@ class Logics:
                 description,
                 invoice_expiry=order.t_to_expire(Order.Status.WFB),
                 cltv_expiry_blocks=cls.compute_cltv_expiry_blocks(order, "maker_bond"),
+                order_id=order.id,
+                lnpayment_concept=LNPayment.Concepts.MAKEBOND.label,
+                time=int(timezone.now().timestamp()),
             )
         except Exception as e:
             print(str(e))
@@ -1208,6 +1213,9 @@ class Logics:
                 description,
                 invoice_expiry=order.t_to_expire(Order.Status.TAK),
                 cltv_expiry_blocks=cls.compute_cltv_expiry_blocks(order, "taker_bond"),
+                order_id=order.id,
+                lnpayment_concept=LNPayment.Concepts.TAKEBOND.label,
+                time=int(timezone.now().timestamp()),
             )
 
         except Exception as e:
@@ -1298,6 +1306,9 @@ class Logics:
                 cltv_expiry_blocks=cls.compute_cltv_expiry_blocks(
                     order, "trade_escrow"
                 ),
+                order_id=order.id,
+                lnpayment_concept=LNPayment.Concepts.TRESCROW.label,
+                time=int(timezone.now().timestamp()),
             )
 
         except Exception as e:

api/utils.py

@@ -145,10 +145,20 @@ lnd_version_cache = {}
 @ring.dict(lnd_version_cache, expire=3600)
 def get_lnd_version():
 
-    from api.lightning.node import LNNode
+    from api.lightning.lnd import LNDNode
 
-    print(LNNode.get_version())
+    return LNDNode.get_version()
-    return LNNode.get_version()
+
+
+cln_version_cache = {}
+
+
+@ring.dict(cln_version_cache, expire=3600)
+def get_cln_version():
+
+    from api.lightning.cln import CLNNode
+
+    return CLNNode.get_version()
 
 
 robosats_commit_cache = {}

api/views.py

@@ -54,6 +54,7 @@ from api.serializers import (
 from api.utils import (
     compute_avg_premium,
     compute_premium_percentile,
+    get_cln_version,
     get_lnd_version,
     get_robosats_commit,
     validate_pgp_keys,
@@ -991,6 +992,7 @@ class InfoView(ListAPIView):
         context["last_day_volume"] = round(total_volume, 8)
         context["lifetime_volume"] = round(lifetime_volume, 8)
         context["lnd_version"] = get_lnd_version()
+        context["cln_version"] = get_cln_version()
         context["robosats_running_commit_hash"] = get_robosats_commit()
         context["version"] = settings.VERSION
         context["alternative_site"] = config("ALTERNATIVE_SITE")

docker-compose.yml

@@ -34,6 +34,7 @@ services:
     volumes:
       - .:/usr/src/robosats
       - ./node/lnd:/lnd
+      - ./node/cln:/cln
     network_mode: service:tor
     command: python3 -u manage.py runserver 0.0.0.0:8000
 
@@ -69,6 +70,7 @@ services:
     volumes:
       - .:/usr/src/robosats
       - ./node/lnd:/lnd
+      - ./node/cln:/cln
     network_mode: service:tor
 
   follow-invoices:
@@ -84,6 +86,7 @@ services:
     volumes:
       - .:/usr/src/robosats
       - ./node/lnd:/lnd
+      - ./node/cln:/cln
     network_mode: service:tor
 
   telegram-watcher:
@@ -96,6 +99,7 @@ services:
     volumes:
       - .:/usr/src/robosats
       - ./node/lnd:/lnd
+      - ./node/cln:/cln
     network_mode: service:tor
 
   celery-worker:
@@ -108,6 +112,7 @@ services:
     volumes:
       - .:/usr/src/robosats
       - ./node/lnd:/lnd
+      - ./node/cln:/cln
     command: celery -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000
     depends_on:
       - redis
@@ -123,6 +128,8 @@ services:
     command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
     volumes:
       - .:/usr/src/robosats
+      - ./node/lnd:/lnd
+      - ./node/cln:/cln
     depends_on:
       - redis
     network_mode: service:tor
@@ -169,6 +176,22 @@ services:
       LND_REST_PORT: 8080
       AUTO_UNLOCK_PWD: ${AUTO_UNLOCK_PWD}
 
+  cln:
+    build: ./docker/cln
+    restart: always
+    network_mode: service:tor
+    container_name: cln-dev
+    depends_on:
+      - tor
+      - bitcoind
+      # - postgres-cln
+    volumes:
+      - ./node/tor/data:/var/lib/tor
+      - ./node/tor/config:/etc/tor
+      - ./node/cln:/root/.lightning
+      - ./node/bitcoin:/root/.bitcoin
+    command: lightningd
+
   bitcoind:
     build: ./docker/bitcoind
     container_name: btc-dev
@@ -194,5 +217,19 @@ services:
     volumes:
       - ./node/db:/var/lib/postgresql/data
 
+  # # Postgresql for CLN
+  # postgres-cln:
+  #   image: postgres:14.2-alpine
+  #   container_name: cln-sql-dev
+  #   restart: always
+  #   environment:
+  #     PGUSER: user
+  #     PGDATABASE: cln
+  #     POSTGRES_PASSWORD: pass
+  #     PGPORT: 5433
+  #   network_mode: service:tor
+  #   volumes:
+  #     - ./node/cln-db:/var/lib/postgresql/data
+
 volumes:
   redisdata:

docker/cln/Dockerfile

@@ -0,0 +1,151 @@
+# Forked of https://github.com/ElementsProject/lightning/blob/2c9b043be97ee4aeca1334d29c2f0ad99da69d34/Dockerfile
+# Changes over base core-lightning Dockerfile:
+# Adds hodlvoice grpc plugin
+# ARG DEVELOPER=0
+
+# This dockerfile is meant to compile a core-lightning x64 image
+# It is using multi stage build:
+# * downloader: Download bitcoin and qemu binaries needed for core-lightning
+# * builder: Compile core-lightning dependencies, then core-lightning itself with static linking
+# * final: Copy the binaries required at runtime
+# The resulting image uploaded to dockerhub will only contain what is needed for runtime.
+# From the root of the repository, run "docker build -t yourimage:yourtag ."
+FROM debian:bullseye-slim as downloader
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex \
+	&& apt-get update \
+	&& apt-get install -qq --no-install-recommends ca-certificates dirmngr wget
+
+WORKDIR /opt
+
+RUN wget -qO /opt/tini "https://github.com/krallin/tini/releases/download/v0.18.0/tini" \
+    && echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /opt/tini" | sha256sum -c - \
+    && chmod +x /opt/tini
+
+ARG BITCOIN_VERSION=24.0.1
+ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
+ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL
+ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS
+
+RUN mkdir /opt/bitcoin && cd /opt/bitcoin \
+    && wget -qO $BITCOIN_TARBALL "$BITCOIN_URL" \
+    && wget -qO bitcoin "$BITCOIN_ASC_URL" \
+    && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \
+    && sha256sum -c SHA256SUMS \
+    && BD=bitcoin-$BITCOIN_VERSION/bin \
+    && tar -xzvf $BITCOIN_TARBALL $BD/bitcoin-cli --strip-components=1 \
+    && rm $BITCOIN_TARBALL
+
+FROM debian:bullseye-slim as builder
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG LIGHTNINGD_VERSION=v23.05
+
+RUN apt-get update -qq && \
+    apt-get install -qq -y --no-install-recommends \
+        autoconf \
+        automake \
+        build-essential \
+        ca-certificates \
+        curl \
+        dirmngr \
+        gettext \
+        git \
+        gnupg \
+        libpq-dev \
+        libtool \
+        libffi-dev \
+        protobuf-compiler \
+        python3 \
+        python3-dev \
+        python3-mako \
+        python3-pip \
+        python3-venv \
+        python3-setuptools \
+        wget
+
+# RUN apt-get install -y --no-install-recommends \
+#     postgresql-common \
+#     postgresql-14 \
+#     libpq-dev=14.* \
+#     && rm -rf /var/lib/apt/lists/*
+
+RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz \
+    && tar xvf zlib-1.2.13.tar.gz \
+    && cd zlib-1.2.13 \
+    && ./configure \
+    && make \
+    && make install && cd .. && \
+    rm zlib-1.2.13.tar.gz && \
+    rm -rf zlib-1.2.13
+
+RUN apt-get install -y --no-install-recommends unzip tclsh \
+    && wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip \
+    && unzip sqlite-src-3290000.zip \
+    && cd sqlite-src-3290000 \
+    && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension \
+    && make \
+    && make install && cd .. && rm sqlite-src-3290000.zip && rm -rf sqlite-src-3290000
+
+RUN wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
+    && tar xvf gmp-6.1.2.tar.xz \
+    && cd gmp-6.1.2 \
+    && ./configure --disable-assembly \
+    && make \
+    && make install && cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
+
+ENV RUST_PROFILE=release
+ENV PATH=$PATH:/root/.cargo/bin/
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN rustup toolchain install stable --component rustfmt --allow-downgrade
+
+WORKDIR /opt/lightningd
+# Clone git repo into /tmp/lightning
+RUN git clone --recursive --branch $LIGHTNINGD_VERSION https://github.com/ElementsProject/lightning.git /tmp/lightning
+RUN git clone --recursive /tmp/lightning . && \
+    git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
+
+RUN git clone --recursive --branch hodlvoice https://github.com/daywalker90/lightning.git /tmp/hodlvoice
+RUN cd /tmp/hodlvoice/plugins/grpc-plugin \
+    && cargo build --release
+
+ENV PYTHON_VERSION=3
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+    && pip3 install -U pip \
+    && pip3 install -U wheel \
+    && /root/.local/bin/poetry install
+
+RUN ./configure --prefix=/tmp/lightning_install --enable-static && \
+    make DEVELOPER=${DEVELOPER} && \
+    /root/.local/bin/poetry run make install
+
+FROM debian:bullseye-slim as final
+
+COPY --from=downloader /opt/tini /usr/bin/tini
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      socat \
+      inotify-tools \
+      python3 \
+      python3-pip \
+      libpq5 && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV LIGHTNINGD_DATA=/root/.lightning
+ENV LIGHTNINGD_RPC_PORT=9835
+ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
+
+RUN mkdir $LIGHTNINGD_DATA && \
+    touch $LIGHTNINGD_DATA/config
+VOLUME [ "/root/.lightning" ]
+COPY --from=builder /tmp/lightning_install/ /usr/local/
+COPY --from=builder /tmp/hodlvoice/target/release/cln-grpc-hodl /tmp/cln-grpc-hodl
+COPY --from=downloader /opt/bitcoin/bin /usr/bin
+COPY config /tmp/config
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+EXPOSE 9735 9835
+ENTRYPOINT  [ "/usr/bin/tini", "-g", "--", "./entrypoint.sh" ]

docker/cln/config

@@ -0,0 +1,9 @@
+network=testnet
+proxy=127.0.0.1:9050
+bind-addr=127.0.0.1:9736
+addr=statictor:127.0.0.1:9051
+grpc-port=9999
+always-use-proxy=true
+important-plugin=/root/.lightning/plugins/cln-grpc-hodl
+# wallet=postgres://user:pass@localhost:5433/cln
+# bookkeeper-db=postgres://user:pass@localhost:5433/cln

docker/cln/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+: "${EXPOSE_TCP:=false}"
+
+networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}"
+
+if [ "$EXPOSE_TCP" == "true" ]; then
+    set -m
+    lightningd "$@" &
+
+    echo "Core-Lightning starting"
+    while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \
+    < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor)
+    echo "Core-Lightning started"
+    echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT"
+
+    socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" &
+    fg %-
+else
+    # Always copy the cln-grpc-hodl plugin into the plugins directory on start up
+    mkdir -p /root/.lightning/plugins
+    cp /tmp/cln-grpc-hodl /root/.lightning/plugins/cln-grpc-hodl
+    if [ ! -f /root/.lightning/config ]; then
+        cp /tmp/config /root/.lightning/config
+    fi
+    exec "$@"
+fi

frontend/src/components/Dialogs/Stats.tsx

@@ -66,12 +66,23 @@ const StatsDialog = ({ open = false, onClose, info }: Props): JSX.Element => {
 
           <Divider />
 
-          <ListItem>
+          {info.lnd_version ? (
-            <ListItemIcon>
+            <ListItem>
-              <BoltIcon />
+              <ListItemIcon>
-            </ListItemIcon>
+                <BoltIcon />
-            <ListItemText primary={info.lnd_version} secondary={t('LND version')} />
+              </ListItemIcon>
-          </ListItem>
+              <ListItemText primary={info.lnd_version} secondary={t('LND version')} />
+            </ListItem>
+          ) : null}
+
+          {info.lnd_version ? (
+            <ListItem>
+              <ListItemIcon>
+                <BoltIcon />
+              </ListItemIcon>
+              <ListItemText primary={info.cln_version} secondary={t('CLN version')} />
+            </ListItem>
+          ) : null}
 
           <Divider />
 

frontend/src/models/Info.model.ts

@@ -8,7 +8,8 @@ export interface Info {
   last_day_nonkyc_btc_premium: number;
   last_day_volume: number;
   lifetime_volume: number;
-  lnd_version: string;
+  lnd_version?: string;
+  cln_version?: string;
   robosats_running_commit_hash: string;
   alternative_site: string;
   alternative_name: string;
@@ -35,7 +36,8 @@ export const defaultInfo: Info = {
   last_day_nonkyc_btc_premium: 0,
   last_day_volume: 0,
   lifetime_volume: 0,
-  lnd_version: 'v0.0.0-beta',
+  lnd_version: '0.0.0-beta',
+  cln_version: '0.0.0',
   robosats_running_commit_hash: '000000000000000',
   alternative_site: 'RoboSats6tkf3eva7x2voqso3a5wcorsnw34jveyxfqi2fu7oyheasid.onion',
   alternative_name: 'RoboSats Mainnet',

scripts/generate_grpc.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# generate grpc definitions
+# generate LND grpc definitions
 cd api/lightning
 [ -d googleapis ] || git clone https://github.com/googleapis/googleapis.git googleapis
 
@@ -24,10 +24,16 @@ python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_pyt
 curl -o verrpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/verrpc/verrpc.proto
 python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. verrpc.proto
 
+# generate CLN grpc definitions
+curl -o node.proto -s https://raw.githubusercontent.com/daywalker90/lightning/hodlvoice/cln-grpc/proto/node.proto
+curl -o primitives.proto -s https://raw.githubusercontent.com/daywalker90/lightning/hodlvoice/cln-grpc/proto/primitives.proto
+python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. node.proto primitives.proto
+
 # delete googleapis
 rm -r googleapis
 
 # patch generated files relative imports
+# LND
 sed -i 's/^import .*_pb2 as/from . \0/' router_pb2.py
 sed -i 's/^import .*_pb2 as/from . \0/' signer_pb2.py
 sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2.py
@@ -38,8 +44,12 @@ sed -i 's/^import .*_pb2 as/from . \0/' lightning_pb2_grpc.py
 sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2_grpc.py
 sed -i 's/^import .*_pb2 as/from . \0/' verrpc_pb2_grpc.py
 
+# CLN
+sed -i 's/^import .*_pb2 as/from . \0/' node_pb2.py
+sed -i 's/^import .*_pb2 as/from . \0/' node_pb2_grpc.py
+
 # On development environments the local volume will be mounted over these files. We copy pb2 and grpc files to /tmp/.
 # This way, we can find if these files are missing with our entrypoint.sh and copy them into the volume.
 
 cp -r *_pb2.py /tmp/
-cp -r *_grpc.py /tmp/
+cp -r *_grpc.py /tmp/