Use new relay

Conflicts:
	compose/relay/sync_external.sh
	compose/strfry/sync.sh

compose/docker-compose.yml

@@ -144,7 +144,9 @@ services:
     network_mode: service:tor
 
   strfry:
-    build: ./strfry
+    build:
+      context: ./strfry
+      dockerfile: old.Dockerfile
     container_name: strfry${SUFFIX}
     restart: always
     volumes:
@@ -153,6 +155,19 @@ services:
       - ${STRFRY_DATA}/db:/app/strfry-db:rw
     network_mode: service:tor
 
+  relay:
+    build:
+      context: ./relay
+      dockerfile: Dockerfile
+    container_name: relay${SUFFIX}
+    restart: always
+    volumes:
+      - ${RELAY_URLS_EXTERNAL}:/app/external_urls.txt:ro
+      - ${RELAY_URLS_FEDERATION}:/app/federation_urls.txt:ro
+      - ${RELAY_CONF}:/etc/strfry.conf:ro
+      - ${RELAY_DATA}/db:/app/strfry-db:rw
+    network_mode: service:tor
+
 # Example simple backup service (copy/paste to attached storage locations)
   # backup:
   #   build: ./backup

compose/env-sample/lndtn/compose.env

@@ -32,6 +32,11 @@ STRFRY_CONF='./env-sample/lndtn/strfry.conf'
 STRFRY_URLS='./strfry/tn.onion_urls.txt'
 STRFRY_DATA='/custom_path/testnet/strfry'
 
+RELAY_CONF='./env-sample/lndtn/relay.strfry.conf'
+RELAY_URLS_EXTERNAL='./strfry/tn.external_urls.txt'
+RELAY_URLS_FEDERATION='./strfry/tn.federation_urls.txt'
+RELAY_DATA='/custom_path/testnet/relay'
+
 # Port and number of HTTP server workers for the robosats backend
 WEB_LOCAL_PORT=8001
 GUNICORN_WORKERS=2

compose/env-sample/lndtn/relay.strfry.conf

@@ -0,0 +1,138 @@
+##
+## Default strfry config
+##
+
+# Directory that contains the strfry LMDB database (restart required)
+db = "/app/strfry-db/"
+
+dbParams {
+    # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required)
+    maxreaders = 256
+
+    # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required)
+    mapsize = 10995116277760
+
+    # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required)
+    noReadAhead = false
+}
+
+events {
+    # Maximum size of normalised JSON, in bytes
+    maxEventSize = 65536
+
+    # Events newer than this will be rejected
+    rejectEventsNewerThanSeconds = 900
+
+    # Events older than this will be rejected
+    rejectEventsOlderThanSeconds = 94608000
+
+    # Ephemeral events older than this will be rejected
+    rejectEphemeralEventsOlderThanSeconds = 60
+
+    # Ephemeral events will be deleted from the DB when older than this
+    ephemeralEventsLifetimeSeconds = 300
+
+    # Maximum number of tags allowed
+    maxNumTags = 2000
+
+    # Maximum size for tag values, in bytes
+    maxTagValSize = 1024
+}
+
+relay {
+    # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required)
+    bind = "0.0.0.0"
+
+    # Port to open for the nostr websocket protocol (restart required)
+    port = 7778
+
+    # Set OS-limit on maximum number of open files/sockets (if 0, don't attempt to set) (restart required)
+    nofiles = 524288
+
+    # HTTP header that contains the client's real IP, before reverse proxying (ie x-real-ip) (MUST be all lower-case)
+    realIpHeader = ""
+
+    info {
+        # NIP-11: Name of this server. Short/descriptive (< 30 characters)
+        name = "Robosats"
+
+        # NIP-11: Detailed information about relay, free-form
+        description = "Federation cache system."
+
+        # NIP-11: Administrative nostr pubkey, for contact purposes
+        pubkey = ""
+
+        # NIP-11: Alternative administrative contact (email, website, etc)
+        contact = ""
+    }
+
+    # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required)
+    maxWebsocketPayloadSize = 131072
+
+    # Websocket-level PING message frequency (should be less than any reverse proxy idle timeouts) (restart required)
+    autoPingSeconds = 55
+
+    # If TCP keep-alive should be enabled (detect dropped connections to upstream reverse proxy)
+    enableTcpKeepalive = false
+
+    # How much uninterrupted CPU time a REQ query should get during its DB scan
+    queryTimesliceBudgetMicroseconds = 10000
+
+    # Maximum records that can be returned per filter
+    maxFilterLimit = 500
+
+    # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time
+    maxSubsPerConnection = 3
+
+    writePolicy {
+        # If non-empty, path to an executable script that implements the writePolicy plugin logic
+        plugin = ""
+    }
+
+    compression {
+        # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required)
+        enabled = true
+
+        # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required)
+        slidingWindow = false
+    }
+
+    logging {
+        # Dump all incoming messages
+        dumpInAll = false
+
+        # Dump all incoming EVENT messages
+        dumpInEvents = false
+
+        # Dump all incoming REQ/CLOSE messages
+        dumpInReqs = false
+
+        # Log performance metrics for initial REQ database scans
+        dbScanPerf = false
+
+        # Log reason for invalid event rejection? Can be disabled to silence excessive logging
+        invalidEvents = true
+    }
+
+    numThreads {
+        # Ingester threads: route incoming requests, validate events/sigs (restart required)
+        ingester = 3
+
+        # reqWorker threads: Handle initial DB scan for events (restart required)
+        reqWorker = 3
+
+        # reqMonitor threads: Handle filtering of new events (restart required)
+        reqMonitor = 3
+
+        # negentropy threads: Handle negentropy protocol messages (restart required)
+        negentropy = 2
+    }
+
+    negentropy {
+        # Support negentropy protocol messages
+        enabled = true
+
+        # Maximum records that sync will process before returning an error
+        maxSyncEvents = 1000000
+    }
+}

compose/nginx/mn.conf.d/local.conf

@@ -95,6 +95,14 @@ server {
         proxy_set_header Host $host;
     }
 
+    location /relay {
+        proxy_pass http://127.0.0.1:7778;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_set_header Host $host;
+    }
+
     location = /favicon.ico { access_log off; log_not_found off; }
 
 }

compose/relay/Dockerfile

@@ -0,0 +1,43 @@
+FROM ubuntu:jammy
+ENV TZ=Europe/London
+
+RUN apt update && apt install -y --no-install-recommends \
+    git g++ make pkg-config libtool ca-certificates \
+    libssl-dev zlib1g-dev liblmdb-dev libflatbuffers-dev \
+    libsecp256k1-dev libzstd-dev
+
+# setup app
+RUN git clone --branch 1.0.4 --single-branch https://github.com/hoytech/strfry /app
+
+WORKDIR /app
+
+RUN git submodule update --init
+RUN make setup-golpe
+RUN make clean
+RUN make -j4
+
+RUN apt update && apt install -y --no-install-recommends \
+    liblmdb0 libflatbuffers1 libsecp256k1-0 libb2-1 libzstd1 torsocks cron\
+    && rm -rf /var/lib/apt/lists/*
+
+RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf
+RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf
+
+# Setting up crontab
+COPY crontab /etc/cron.d/crontab
+RUN chmod 0644 /etc/cron.d/crontab
+RUN crontab /etc/cron.d/crontab
+
+# Setting up entrypoints
+COPY sync_external.sh /etc/strfry/sync_external.sh
+COPY sync_federation.sh /etc/strfry/sync_federation.sh
+COPY entrypoint.sh /etc/strfry/entrypoint.sh
+
+RUN chmod +x /etc/strfry/entrypoint.sh
+RUN chmod +x /etc/strfry/sync_external.sh
+RUN chmod +x /etc/strfry/sync_federation.sh
+
+#Setting up logs
+RUN touch /var/log/cron.log && chmod 0644 /var/log/cron.log
+
+ENTRYPOINT ["/etc/strfry/entrypoint.sh"]

compose/relay/README.md

@@ -0,0 +1,5 @@
+## Synchronization
+
+To keep a healthy network, make sure your relay sinchronizes with at least 2 clearnet relays (implementing strfry). 
+
+If they are different from other coordinators, better.

compose/relay/crontab

@@ -0,0 +1,25 @@
+# Edit this file to introduce tasks to be run by cron.
+#
+# Each task to run has to be defined through a single line
+# indicating with different fields when the task will be run
+# and what command to run for the task
+#
+# To define the time you can provide concrete values for
+# minute (m), hour (h), day of month (dom), month (mon),
+# and day of week (dow) or use '*' in these fields (for 'any').
+#
+# Notice that tasks will be started based on the cron's system
+# daemon's notion of time and timezones.
+#
+# Output of the crontab jobs (including errors) is sent through
+# email to the user the crontab file belongs to (unless redirected).
+#
+# For example, you can run a backup of all your user accounts
+# at 5 a.m every week with:
+# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
+#
+# For more information see the manual pages of crontab(5) and cron(8)
+#
+# m h  dom mon dow   command
+*/2 * * * * torsocks /etc/strfry/sync_federation.sh >> /var/log/cron.log 2>&1
+*/5 * * * * torsocks /etc/strfry/sync_external.sh >> /var/log/cron.log 2>&1

compose/relay/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cron -f -l 8 & tail -f /var/log/cron.log & /app/strfry relay

compose/relay/mn.external_urls.txt

@@ -0,0 +1,3 @@
+wss://nostr.satstralia.com
+wss://relay.damus.io
+wss://freelay.sovbit.host

compose/relay/mn.federation_urls.txt

@@ -0,0 +1,4 @@
+ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/relay
+ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/relay
+ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/relay
+ws://otmoonrndnrddqdlhu6b36heunmbyw3cgvadqo2oqeau3656wfv7fwad.onion/relay

compose/relay/sync_external.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+filters_external='{"kinds":[38383]}'
+timeout_duration="15s"
+
+while IFS= read -r line; do
+  timeout "$timeout_duration" /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters_external" --dir both
+done < /app/external_urls.txt

compose/relay/sync_federation.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+filters_federation='{"kinds":[38383, 31986, 1059]}'
+timeout_duration="15s"
+
+while IFS= read -r line; do
+  timeout "$timeout_duration" /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters_federation" --dir both
+done < /app/federation_urls.txt

compose/relay/tn.external_urls.txt

@@ -0,0 +1,3 @@
+wss://nostr.satstralia.com
+wss://relay.damus.io
+wss://freelay.sovbit.host

compose/relay/tn.federation_urls.txt

@@ -0,0 +1,4 @@
+ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/relay
+ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/relay
+ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/relay
+ws://otmtestgbj3kqo3nre6oksusuqfb4ids5zg2y5z2qza2jogeu67stwid.onion/relay

compose/strfry/Dockerfile

@@ -7,7 +7,7 @@ RUN apt update && apt install -y --no-install-recommends \
     libsecp256k1-dev libzstd-dev
 
 # setup app
-RUN git clone --branch 1.0.0 --single-branch https://github.com/hoytech/strfry /app
+RUN git clone https://github.com/KoalaSat/strfry /app
 
 WORKDIR /app
 
@@ -29,11 +29,13 @@ RUN chmod 0644 /etc/cron.d/crontab
 RUN crontab /etc/cron.d/crontab
 
 # Setting up entrypoints
-COPY sync.sh /etc/strfry/sync.sh
+COPY sync_external.sh /etc/strfry/sync_external.sh
+COPY sync_federation.sh /etc/strfry/sync_federation.sh
 COPY entrypoint.sh /etc/strfry/entrypoint.sh
 
 RUN chmod +x /etc/strfry/entrypoint.sh
-RUN chmod +x /etc/strfry/sync.sh
+RUN chmod +x /etc/strfry/sync_external.sh
+RUN chmod +x /etc/strfry/sync_federation.sh
 
 #Setting up logs
 RUN touch /var/log/cron.log && chmod 0644 /var/log/cron.log

compose/strfry/crontab

@@ -21,4 +21,5 @@
 # For more information see the manual pages of crontab(5) and cron(8)
 #
 # m h  dom mon dow   command
-*/1 * * * * torsocks /etc/strfry/sync.sh >> /var/log/cron.log 2>&1
+*/2 * * * * torsocks /etc/strfry/sync_federation.sh >> /var/log/cron.log 2>&1
+*/5 * * * * torsocks /etc/strfry/sync_external.sh >> /var/log/cron.log 2>&1

compose/strfry/sync.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-filters='{"kinds":[38383, 31986, 1059]}'
-
-while IFS= read -r line; do
-  /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters" --dir both
-done < /app/onion_urls.txt

compose/strfry/sync_external.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+filters_external='{"kinds":[38383]}'
+timeout_duration="15s"
+
+while IFS= read -r line; do
+  timeout "$timeout_duration" /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters_external" --dir both
+done < /app/external_urls.txt
+

compose/strfry/sync_federation.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+filters_federation='{"kinds":[38383, 31986, 1059]}'
+timeout_duration="15s"
+
+while IFS= read -r line; do
+  timeout "$timeout_duration" /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters_federation" --dir both
+done < /app/federation_urls.txt