From 3e460bb2558bb6d2b7f4acabd74b5d237bd3438d Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 1 Jul 2024 18:00:17 +0200 Subject: [PATCH 01/33] Use nostr as cache system --- api/logics.py | 4 +- api/nostr.py | 47 ++++++++++ api/tasks.py | 19 ++++ docker-compose.yml | 9 ++ docker-tests.yml | 11 ++- nodeapp/rnostr/config/rnostr.toml | 145 ++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/utils/trade.py | 3 +- 8 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 api/nostr.py create mode 100644 nodeapp/rnostr/config/rnostr.toml diff --git a/api/logics.py b/api/logics.py index 7049e21c..da562bc4 100644 --- a/api/logics.py +++ b/api/logics.py @@ -8,7 +8,7 @@ from django.utils import timezone from api.lightning.node import LNNode from api.models import Currency, LNPayment, MarketTick, OnchainPayment, Order -from api.tasks import send_devfund_donation, send_notification +from api.tasks import send_devfund_donation, send_notification, send_order_nostr_event from api.utils import get_minning_fee, validate_onchain_address, location_country from chat.models import Message @@ -1208,6 +1208,8 @@ class Logics: order.save() # update all fields + send_order_nostr_event.delay(order_id=order.id, message="new") + order.log(f"Order({order.id},{str(order)}) is public in the order book") return diff --git a/api/nostr.py b/api/nostr.py new file mode 100644 index 00000000..9e31d27a --- /dev/null +++ b/api/nostr.py @@ -0,0 +1,47 @@ +import time +import pygeohash +from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Filter +from api.models import Order +from decouple import config + + +class Nostr: + """Simple nostr events manager to be used as a cache system for clients""" + + async def send_new_order_event(self, order): + """Creates the event and sends it to the coordinator relay""" + # Initialize with coordinator Keys + keys = Keys.generate() + signer = NostrSigner.keys(keys) + client = Client(signer) + + # Add relays and connect + await client.add_relays(["ws://localhost:888"]) + await client.connect() + + event = EventBuilder(38383, "", generate_tags(order)).to_event(keys) + output = await client.send_event(event) + + print(f"Nostr event sent: {output}") + + def generate_tags(self, order): + return [ + ["d", order.id], + ["name", order.maker.robot_name], + ["k", order.type.lower()], + ["f", order.currency], + ["s", Order.Status(order.status).label], + ["amt", order.last_satoshis], + ["fa", order.amount], + ["pm", order.payment_method.split(" ")], + ["premium", order.premium_percentile], + ["source", f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}"], + ["expiration", order.expires_at.timestamp()], + ["y", "robosats"], + ["coordinator", config("COORDINATOR_ALIAS", cast=str)] + ["z", "order"], + ["n", order.network], + ["layer", "lightning"], + ["g", pygeohash.encode(order.latitude, order.longitude)], + ["bond", order.bond] + ] diff --git a/api/tasks.py b/api/tasks.py index 656feb89..10379eac 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,3 +1,4 @@ +import asyncio from celery import shared_task from celery.exceptions import SoftTimeLimitExceeded @@ -251,6 +252,24 @@ def cache_market(): return +@shared_task(name="", ignore_result=True, time_limit=120) +def send_order_nostr_event(order_id=None, message=None): + if order_id: + from api.models import Order + from api.nostr import Nostr + + order = Order.objects.get(id=order_id) + + nostr = Nostr() + if message == "new": + coroutine = nostr.send_new_order_event(order) + + if coroutine: + loop = asyncio.get_event_loop() + loop.run_until_complete(coroutine) + + return + @shared_task(name="send_notification", ignore_result=True, time_limit=120) def send_notification(order_id=None, chat_message_id=None, message=None): if order_id: diff --git a/docker-compose.yml b/docker-compose.yml index c8334bdb..1542f8dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,6 +163,7 @@ services: ports: - 8000:8000 # dev frontend build - 12596:12596 # umbrel frontend + - 888:888 # nostr lnd: build: ./docker/lnd @@ -226,6 +227,14 @@ services: volumes: - ./node/db:/var/lib/postgresql/data + rnostr: + build: https://github.com/rnostr/rnostr.git + container_name: rnostr-dev + restart: unless-stopped + volumes: + - ./nodeapp/rnostr/config/rnostr.toml:/rnostr/config/rnostr.toml:r + network_mode: service:tor + # # Postgresql for CLN # postgres-cln: # image: postgres:14.2-alpine diff --git a/docker-tests.yml b/docker-tests.yml index 975747e8..202f46dd 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -182,7 +182,7 @@ services: # celery-worker: # image: backend-image # pull_policy: never - # container_name: celery-worker + # container_name: test-celery-worker # restart: always # environment: # DEVELOPMENT: True @@ -205,6 +205,15 @@ services: # - redis # network_mode: service:bitcoind + rnostr: + build: https://github.com/rnostr/rnostr.git + container_name: test-rnostr + restart: unless-stopped + volumes: + - ./nodeapp/rnostr/config/rnostr.toml:/rnostr/config/rnostr.toml:r + network_mode: service:bitcoind + + volumes: redisdata: bitcoin: diff --git a/nodeapp/rnostr/config/rnostr.toml b/nodeapp/rnostr/config/rnostr.toml new file mode 100644 index 00000000..1d3e3dbc --- /dev/null +++ b/nodeapp/rnostr/config/rnostr.toml @@ -0,0 +1,145 @@ +# Configuration +# All duration format reference https://docs.rs/duration-str/latest/duration_str/ +# +# config relay information +[information] +name = "rnostr" +description = "A high-performance and scalable nostr relay written in Rust." +software = "https://github.com/rnostr/rnostr" +# pubkey = "" +# contact = "" + +# config data path +[data] +# the data path (restart required) +# the events db path is $path/events +path = "./data" + +# Query filter timeout time, default no timeout. +db_query_timeout = "100ms" + +# config network +[network] +# Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) +host = "127.0.0.1" +# Listen port (restart required) +port = 888 + +# real ip header (default empty) +# ie: cf-connecting-ip, x-real-ip, x-forwarded-for +# real_ip_header = "x-forwarded-for" + +# redirect to other site when user access the http index page +# index_redirect_to = "https://example.com" + +# heartbeat timeout (default 120 seconds, must bigger than heartbeat interval) +# How long before lack of client response causes a timeout +# heartbeat_timeout = "2m" + +# heartbeat interval (default 60 seconds) +# How often heartbeat pings are sent +# heartbeat_interval = "1m" + +# config thread (restart required) +[thread] +# number of http server threads (restart required) +# default 0 will use the num of cpus +# http = 0 + +# number of read event threads (restart required) +# default 0 will use the num of cpus +# reader = 0 + +[limitation] +# this is the maximum number of bytes for incoming JSON. default 512K +max_message_length = 524288 +# total number of subscriptions that may be active on a single websocket connection to this relay. default 20 +max_subscriptions = 1 +# maximum number of filter values in each subscription. default 10 +max_filters = 10 +# the relay server will clamp each filter's limit value to this number. This means the client won't be able to get more than this number of events from a single subscription filter. default 300 +max_limit = 300 +# maximum length of subscription id as a string. default 100 +max_subid_length = 100 +# for authors and ids filters which are to match against a hex prefix, you must provide at least this many hex digits in the prefix. default 10 +min_prefix = 10 +# in any event, this is the maximum number of elements in the tags list. default 5000 +max_event_tags = 15 +# Events older than this will be rejected. default 3 years +max_event_time_older_than_now = 94608000 +# Events newer than this will be rejected. default 15 minutes +max_event_time_newer_than_now = 900 + +# Metrics extension, get the metrics data from https://example.com/metrics?auth=auth_key +[metrics] +enabled = false +# change the auth key +auth = "auth_key" + +# Auth extension +[auth] +enabled = false + +# # Authenticate the command 'REQ' get event, subscribe filter +# [auth.req] +# # only the list IP are allowed to req +# ip_whitelist = ["127.0.0.1"] +# # only the list IP are denied to req +# ip_blacklist = ["127.0.0.1"] +# # Restrict on nip42 verified pubkey, so client needs to implement nip42 and authenticate success +# pubkey_whitelist = ["xxxxxx"] +# pubkey_blacklist = ["xxxx"] + +# # Authenticate the command 'EVENT' write event +# [auth.event] +# ip_whitelist = ["127.0.0.1"] +# ip_blacklist = ["127.0.0.1"] +# # Restrict on nip42 verified pubkey, so client needs to implement nip42 and authenticate success +# pubkey_whitelist = ["xxxxxx"] +# pubkey_blacklist = ["xxxx"] +# # Restrict on event author pubkey, No need nip42 authentication +# event_pubkey_whitelist = ["xxxxxx"] +# event_pubkey_blacklist = ["xxxx"] + +# IP Rate limiter extension +[rate_limiter] +enabled = false + +# # interval at second for clearing invalid data to free up memory. +# # 0 will be converted to default 60 seconds +# clear_interval = "60s" + +# # rate limiter ruler list when write event per user client IP +# [[rate_limiter.event]] +# # name of rate limiter, used by metrics +# name = "all" +# # description will notice the user when rate limiter exceeded +# description = "allow only ten events per minute" +# period = "1m" +# limit = 10 + +# # only limit for kinds +# # support kind list: [1, 2, 3] +# # kind ranges included(start) to excluded(end): [[0, 10000], [30000, 40000]] +# # mixed: [1, 2, [30000, 40000]] +# kinds = [[0, 40000]] + +# # skip when ip in whitelist +# ip_whitelist = ["127.0.0.1"] + +# [[rate_limiter.event]] +# name = "kind 10000" +# description = "allow only five write events per minute when event kind between 0 to 10000" +# period = "60s" +# limit = 5 +# kinds = [[0, 10000]] + +# NIP-45 Count extension +# use carefully. see README.md#count +[count] +enabled = false + +# NIP-50 Search extension +# use carefully. see README.md#search +[search] +enabled = false diff --git a/requirements.txt b/requirements.txt index 2b979cec..6e7ae047 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ drf-spectacular==0.27.2 drf-spectacular-sidecar==2024.7.1 django-cors-headers==4.4.0 base91==1.0.1 +nostr-sdk==0.32.2 diff --git a/tests/utils/trade.py b/tests/utils/trade.py index 39bcda0a..7ac981b7 100644 --- a/tests/utils/trade.py +++ b/tests/utils/trade.py @@ -5,7 +5,7 @@ from django.urls import reverse from api.management.commands.clean_orders import Command as CleanOrders from api.management.commands.follow_invoices import Command as FollowInvoices from api.models import Order -from api.tasks import follow_send_payment, send_notification +from api.tasks import follow_send_payment, send_notification, send_order_nostr_event from tests.utils.node import ( add_invoice, create_address, @@ -156,6 +156,7 @@ class Trade: wait_nodes_sync() @patch("api.tasks.send_notification.delay", send_notification) + @patch("api.tasks.send_order_nostr_event.delay", send_order_nostr_event) def publish_order(self): # Maker's first order fetch. Should trigger maker bond hold invoice generation. self.get_order() From 97bb2dc7770fddd3bbe5810a9e67c5a63131bfa4 Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 1 Jul 2024 18:43:35 +0200 Subject: [PATCH 02/33] Prepare for other events --- api/nostr.py | 18 +++++++++--------- api/tasks.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index 9e31d27a..6d14b24c 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -1,6 +1,5 @@ -import time import pygeohash -from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Filter +from nostr_sdk import Keys, Client, EventBuilder, NostrSigner from api.models import Order from decouple import config @@ -18,10 +17,9 @@ class Nostr: # Add relays and connect await client.add_relays(["ws://localhost:888"]) await client.connect() - - event = EventBuilder(38383, "", generate_tags(order)).to_event(keys) - output = await client.send_event(event) + event = EventBuilder(38383, "", self.generate_tags(order)).to_event(keys) + output = await client.send_event(event) print(f"Nostr event sent: {output}") def generate_tags(self, order): @@ -35,13 +33,15 @@ class Nostr: ["fa", order.amount], ["pm", order.payment_method.split(" ")], ["premium", order.premium_percentile], - ["source", f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}"], + [ + "source", + f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", + ], ["expiration", order.expires_at.timestamp()], ["y", "robosats"], - ["coordinator", config("COORDINATOR_ALIAS", cast=str)] - ["z", "order"], + ["coordinator", config("COORDINATOR_ALIAS", cast=str)]["z", "order"], ["n", order.network], ["layer", "lightning"], ["g", pygeohash.encode(order.latitude, order.longitude)], - ["bond", order.bond] + ["bond", order.bond], ] diff --git a/api/tasks.py b/api/tasks.py index 10379eac..fa1d44b7 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -263,13 +263,13 @@ def send_order_nostr_event(order_id=None, message=None): nostr = Nostr() if message == "new": coroutine = nostr.send_new_order_event(order) - if coroutine: loop = asyncio.get_event_loop() loop.run_until_complete(coroutine) return + @shared_task(name="send_notification", ignore_result=True, time_limit=120) def send_notification(order_id=None, chat_message_id=None, message=None): if order_id: From bdc10eb289adfa5f5ac0345801993925a907c0e1 Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 15 Jul 2024 10:19:53 +0200 Subject: [PATCH 03/33] Switch to strfry --- api/nostr.py | 2 +- docker-compose.yml | 10 +-- docker-tests.yml | 14 +-- nodeapp/rnostr/config/rnostr.toml | 145 ------------------------------ nodeapp/strfry/config/strfry.conf | 138 ++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 158 deletions(-) delete mode 100644 nodeapp/rnostr/config/rnostr.toml create mode 100644 nodeapp/strfry/config/strfry.conf diff --git a/api/nostr.py b/api/nostr.py index 6d14b24c..02effa65 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -15,7 +15,7 @@ class Nostr: client = Client(signer) # Add relays and connect - await client.add_relays(["ws://localhost:888"]) + await client.add_relays(["ws://localhost:7777"]) await client.connect() event = EventBuilder(38383, "", self.generate_tags(order)).to_event(keys) diff --git a/docker-compose.yml b/docker-compose.yml index 1542f8dc..df78c7d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,7 +163,7 @@ services: ports: - 8000:8000 # dev frontend build - 12596:12596 # umbrel frontend - - 888:888 # nostr + - 7777:7777 # nostr lnd: build: ./docker/lnd @@ -227,12 +227,12 @@ services: volumes: - ./node/db:/var/lib/postgresql/data - rnostr: - build: https://github.com/rnostr/rnostr.git - container_name: rnostr-dev + strfry: + build: https://github.com/hoytech/strfry.git + container_name: strfry-dev restart: unless-stopped volumes: - - ./nodeapp/rnostr/config/rnostr.toml:/rnostr/config/rnostr.toml:r + - ./nodeapp/strfry/config/strfry.conf:/app/strfry.conf:r network_mode: service:tor # # Postgresql for CLN diff --git a/docker-tests.yml b/docker-tests.yml index 202f46dd..41d3b5aa 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -205,13 +205,13 @@ services: # - redis # network_mode: service:bitcoind - rnostr: - build: https://github.com/rnostr/rnostr.git - container_name: test-rnostr - restart: unless-stopped - volumes: - - ./nodeapp/rnostr/config/rnostr.toml:/rnostr/config/rnostr.toml:r - network_mode: service:bitcoind + # strfry: + # build: https://github.com/hoytech/strfry.git + # container_name: test-strfry + # restart: unless-stopped + # volumes: + # - ./nodeapp/strfry/config/strfry.conf:/strfry/strfry.conf:r + # network_mode: service:bitcoind volumes: diff --git a/nodeapp/rnostr/config/rnostr.toml b/nodeapp/rnostr/config/rnostr.toml deleted file mode 100644 index 1d3e3dbc..00000000 --- a/nodeapp/rnostr/config/rnostr.toml +++ /dev/null @@ -1,145 +0,0 @@ -# Configuration -# All duration format reference https://docs.rs/duration-str/latest/duration_str/ -# -# config relay information -[information] -name = "rnostr" -description = "A high-performance and scalable nostr relay written in Rust." -software = "https://github.com/rnostr/rnostr" -# pubkey = "" -# contact = "" - -# config data path -[data] -# the data path (restart required) -# the events db path is $path/events -path = "./data" - -# Query filter timeout time, default no timeout. -db_query_timeout = "100ms" - -# config network -[network] -# Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) -host = "127.0.0.1" -# Listen port (restart required) -port = 888 - -# real ip header (default empty) -# ie: cf-connecting-ip, x-real-ip, x-forwarded-for -# real_ip_header = "x-forwarded-for" - -# redirect to other site when user access the http index page -# index_redirect_to = "https://example.com" - -# heartbeat timeout (default 120 seconds, must bigger than heartbeat interval) -# How long before lack of client response causes a timeout -# heartbeat_timeout = "2m" - -# heartbeat interval (default 60 seconds) -# How often heartbeat pings are sent -# heartbeat_interval = "1m" - -# config thread (restart required) -[thread] -# number of http server threads (restart required) -# default 0 will use the num of cpus -# http = 0 - -# number of read event threads (restart required) -# default 0 will use the num of cpus -# reader = 0 - -[limitation] -# this is the maximum number of bytes for incoming JSON. default 512K -max_message_length = 524288 -# total number of subscriptions that may be active on a single websocket connection to this relay. default 20 -max_subscriptions = 1 -# maximum number of filter values in each subscription. default 10 -max_filters = 10 -# the relay server will clamp each filter's limit value to this number. This means the client won't be able to get more than this number of events from a single subscription filter. default 300 -max_limit = 300 -# maximum length of subscription id as a string. default 100 -max_subid_length = 100 -# for authors and ids filters which are to match against a hex prefix, you must provide at least this many hex digits in the prefix. default 10 -min_prefix = 10 -# in any event, this is the maximum number of elements in the tags list. default 5000 -max_event_tags = 15 -# Events older than this will be rejected. default 3 years -max_event_time_older_than_now = 94608000 -# Events newer than this will be rejected. default 15 minutes -max_event_time_newer_than_now = 900 - -# Metrics extension, get the metrics data from https://example.com/metrics?auth=auth_key -[metrics] -enabled = false -# change the auth key -auth = "auth_key" - -# Auth extension -[auth] -enabled = false - -# # Authenticate the command 'REQ' get event, subscribe filter -# [auth.req] -# # only the list IP are allowed to req -# ip_whitelist = ["127.0.0.1"] -# # only the list IP are denied to req -# ip_blacklist = ["127.0.0.1"] -# # Restrict on nip42 verified pubkey, so client needs to implement nip42 and authenticate success -# pubkey_whitelist = ["xxxxxx"] -# pubkey_blacklist = ["xxxx"] - -# # Authenticate the command 'EVENT' write event -# [auth.event] -# ip_whitelist = ["127.0.0.1"] -# ip_blacklist = ["127.0.0.1"] -# # Restrict on nip42 verified pubkey, so client needs to implement nip42 and authenticate success -# pubkey_whitelist = ["xxxxxx"] -# pubkey_blacklist = ["xxxx"] -# # Restrict on event author pubkey, No need nip42 authentication -# event_pubkey_whitelist = ["xxxxxx"] -# event_pubkey_blacklist = ["xxxx"] - -# IP Rate limiter extension -[rate_limiter] -enabled = false - -# # interval at second for clearing invalid data to free up memory. -# # 0 will be converted to default 60 seconds -# clear_interval = "60s" - -# # rate limiter ruler list when write event per user client IP -# [[rate_limiter.event]] -# # name of rate limiter, used by metrics -# name = "all" -# # description will notice the user when rate limiter exceeded -# description = "allow only ten events per minute" -# period = "1m" -# limit = 10 - -# # only limit for kinds -# # support kind list: [1, 2, 3] -# # kind ranges included(start) to excluded(end): [[0, 10000], [30000, 40000]] -# # mixed: [1, 2, [30000, 40000]] -# kinds = [[0, 40000]] - -# # skip when ip in whitelist -# ip_whitelist = ["127.0.0.1"] - -# [[rate_limiter.event]] -# name = "kind 10000" -# description = "allow only five write events per minute when event kind between 0 to 10000" -# period = "60s" -# limit = 5 -# kinds = [[0, 10000]] - -# NIP-45 Count extension -# use carefully. see README.md#count -[count] -enabled = false - -# NIP-50 Search extension -# use carefully. see README.md#search -[search] -enabled = false diff --git a/nodeapp/strfry/config/strfry.conf b/nodeapp/strfry/config/strfry.conf new file mode 100644 index 00000000..3bdcf584 --- /dev/null +++ b/nodeapp/strfry/config/strfry.conf @@ -0,0 +1,138 @@ +## +## Default strfry config +## + +# Directory that contains the strfry LMDB database (restart required) +db = "./strfry-db/" + +dbParams { + # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) + maxreaders = 256 + + # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) + mapsize = 10995116277760 + + # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required) + noReadAhead = false +} + +events { + # Maximum size of normalised JSON, in bytes + maxEventSize = 65536 + + # Events newer than this will be rejected + rejectEventsNewerThanSeconds = 900 + + # Events older than this will be rejected + rejectEventsOlderThanSeconds = 94608000 + + # Ephemeral events older than this will be rejected + rejectEphemeralEventsOlderThanSeconds = 60 + + # Ephemeral events will be deleted from the DB when older than this + ephemeralEventsLifetimeSeconds = 300 + + # Maximum number of tags allowed + maxNumTags = 2000 + + # Maximum size for tag values, in bytes + maxTagValSize = 1024 +} + +relay { + # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) + bind = "127.0.0.1" + + # Port to open for the nostr websocket protocol (restart required) + port = 7777 + + # Set OS-limit on maximum number of open files/sockets (if 0, don't attempt to set) (restart required) + nofiles = 1000000 + + # HTTP header that contains the client's real IP, before reverse proxying (ie x-real-ip) (MUST be all lower-case) + realIpHeader = "" + + info { + # NIP-11: Name of this server. Short/descriptive (< 30 characters) + name = "Robosats" + + # NIP-11: Detailed information about relay, free-form + description = "Federation cache system." + + # NIP-11: Administrative nostr pubkey, for contact purposes + pubkey = "" + + # NIP-11: Alternative administrative contact (email, website, etc) + contact = "" + } + + # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required) + maxWebsocketPayloadSize = 131072 + + # Websocket-level PING message frequency (should be less than any reverse proxy idle timeouts) (restart required) + autoPingSeconds = 55 + + # If TCP keep-alive should be enabled (detect dropped connections to upstream reverse proxy) + enableTcpKeepalive = false + + # How much uninterrupted CPU time a REQ query should get during its DB scan + queryTimesliceBudgetMicroseconds = 10000 + + # Maximum records that can be returned per filter + maxFilterLimit = 500 + + # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time + maxSubsPerConnection = 20 + + writePolicy { + # If non-empty, path to an executable script that implements the writePolicy plugin logic + plugin = "" + } + + compression { + # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required) + enabled = true + + # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required) + slidingWindow = true + } + + logging { + # Dump all incoming messages + dumpInAll = false + + # Dump all incoming EVENT messages + dumpInEvents = false + + # Dump all incoming REQ/CLOSE messages + dumpInReqs = false + + # Log performance metrics for initial REQ database scans + dbScanPerf = false + + # Log reason for invalid event rejection? Can be disabled to silence excessive logging + invalidEvents = true + } + + numThreads { + # Ingester threads: route incoming requests, validate events/sigs (restart required) + ingester = 3 + + # reqWorker threads: Handle initial DB scan for events (restart required) + reqWorker = 3 + + # reqMonitor threads: Handle filtering of new events (restart required) + reqMonitor = 3 + + # negentropy threads: Handle negentropy protocol messages (restart required) + negentropy = 2 + } + + negentropy { + # Support negentropy protocol messages + enabled = true + + # Maximum records that sync will process before returning an error + maxSyncEvents = 1000000 + } +} From cbb063f06d908b30fda110a29c6ee03ce60e1a94 Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 15 Jul 2024 12:33:09 +0200 Subject: [PATCH 04/33] NIP corrections --- api/logics.py | 49 ++++++++++++++++++++++++++++++++++---------- api/nostr.py | 21 +++++++++++++++---- api/tasks.py | 5 ++--- docker-tests.yml | 16 +++++++-------- requirements.txt | 1 + tests/utils/trade.py | 4 ++-- 6 files changed, 68 insertions(+), 28 deletions(-) diff --git a/api/logics.py b/api/logics.py index da562bc4..8bec242a 100644 --- a/api/logics.py +++ b/api/logics.py @@ -8,7 +8,7 @@ from django.utils import timezone from api.lightning.node import LNNode from api.models import Currency, LNPayment, MarketTick, OnchainPayment, Order -from api.tasks import send_devfund_donation, send_notification, send_order_nostr_event +from api.tasks import send_devfund_donation, send_notification, nostr_send_order_event from api.utils import get_minning_fee, validate_onchain_address, location_country from chat.models import Message @@ -704,9 +704,9 @@ class Logics: if context["invoice_amount"] < MIN_SWAP_AMOUNT: context["swap_allowed"] = False - context[ - "swap_failure_reason" - ] = f"Order amount is smaller than the minimum swap available of {MIN_SWAP_AMOUNT} Sats" + context["swap_failure_reason"] = ( + f"Order amount is smaller than the minimum swap available of {MIN_SWAP_AMOUNT} Sats" + ) order.log( f"Onchain payment option was not offered: amount is smaller than the minimum swap available of {MIN_SWAP_AMOUNT} Sats", level="WARN", @@ -714,9 +714,9 @@ class Logics: return True, context elif context["invoice_amount"] > MAX_SWAP_AMOUNT: context["swap_allowed"] = False - context[ - "swap_failure_reason" - ] = f"Order amount is bigger than the maximum swap available of {MAX_SWAP_AMOUNT} Sats" + context["swap_failure_reason"] = ( + f"Order amount is bigger than the maximum swap available of {MAX_SWAP_AMOUNT} Sats" + ) order.log( f"Onchain payment option was not offered: amount is bigger than the maximum swap available of {MAX_SWAP_AMOUNT} Sats", level="WARN", @@ -741,9 +741,9 @@ class Logics: ) if not valid: context["swap_allowed"] = False - context[ - "swap_failure_reason" - ] = "Not enough onchain liquidity available to offer a swap" + context["swap_failure_reason"] = ( + "Not enough onchain liquidity available to offer a swap" + ) order.log( "Onchain payment option was not offered: onchain liquidity available to offer a swap", level="WARN", @@ -1019,6 +1019,8 @@ class Logics: order.log("Order expired while waiting for maker bond") order.log("Maker bond was cancelled") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 2.a) When maker cancels after bond @@ -1039,6 +1041,8 @@ class Logics: order.log("Order cancelled by maker while public or paused") order.log("Maker bond was unlocked") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 2.b) When maker cancels after bond and before taker bond is locked @@ -1058,6 +1062,8 @@ class Logics: order.log("Maker bond was unlocked") order.log("Taker bond was cancelled") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 3) When taker cancels before bond @@ -1070,6 +1076,8 @@ class Logics: order.log("Taker cancelled before locking the bond") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 4) When taker or maker cancel after bond (before escrow) @@ -1099,6 +1107,8 @@ class Logics: order.log("Maker bond was settled") order.log("Taker bond was unlocked") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 4.b) When taker cancel after bond (before escrow) @@ -1121,6 +1131,8 @@ class Logics: order.log("Taker bond was settled") order.log("Maker bond was unlocked") + nostr_send_order_event.delay(order_id=order.id) + return True, None # 5) When trade collateral has been posted (after escrow) @@ -1136,6 +1148,9 @@ class Logics: order.log( f"Taker Robot({user.robot.id},{user.username}) accepted the collaborative cancellation" ) + + nostr_send_order_event.delay(order_id=order.id) + return True, None # if the taker had asked, and now the maker does: cancel order, return everything @@ -1144,6 +1159,9 @@ class Logics: order.log( f"Maker Robot({user.robot.id},{user.username}) accepted the collaborative cancellation" ) + + nostr_send_order_event.delay(order_id=order.id) + return True, None # Otherwise just make true the asked for cancel flags @@ -1181,6 +1199,8 @@ class Logics: order.update_status(Order.Status.CCA) send_notification.delay(order_id=order.id, message="collaborative_cancelled") + nostr_send_order_event.delay(order_id=order.id) + order.log("Order was collaboratively cancelled") order.log("Maker bond was unlocked") order.log("Taker bond was unlocked") @@ -1208,7 +1228,7 @@ class Logics: order.save() # update all fields - send_order_nostr_event.delay(order_id=order.id, message="new") + nostr_send_order_event.delay(order_id=order.id) order.log(f"Order({order.id},{str(order)}) is public in the order book") return @@ -1352,6 +1372,9 @@ class Logics: except Exception: pass send_notification.delay(order_id=order.id, message="order_taken_confirmed") + + nostr_send_order_event.delay(order_id=order.id) + order.log( f"Contract formalized. Maker: Robot({order.maker.robot.id},{order.maker}). Taker: Robot({order.taker.robot.id},{order.taker}). API median price {order.currency.exchange_rate} {dict(Currency.currency_choices)[order.currency.currency]}/BTC. Premium is {order.premium}%. Contract size {order.last_satoshis} Sats" ) @@ -1743,11 +1766,15 @@ class Logics: order.log( f"Robot({user.robot.id},{user.username}) paused the public order" ) + + nostr_send_order_event.delay(order_id=order.id) elif order.status == Order.Status.PAU: order.update_status(Order.Status.PUB) order.log( f"Robot({user.robot.id},{user.username}) made public the paused order" ) + + nostr_send_order_event.delay(order_id=order.id) else: order.log( f"Robot({user.robot.id},{user.username}) tried to pause/unpause an order that was not public or paused", diff --git a/api/nostr.py b/api/nostr.py index 02effa65..4ebb5460 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -1,4 +1,6 @@ import pygeohash +import hashlib +import uuid from nostr_sdk import Keys, Client, EventBuilder, NostrSigner from api.models import Order from decouple import config @@ -7,7 +9,7 @@ from decouple import config class Nostr: """Simple nostr events manager to be used as a cache system for clients""" - async def send_new_order_event(self, order): + async def send_order_event(self, order): """Creates the event and sends it to the coordinator relay""" # Initialize with coordinator Keys keys = Keys.generate() @@ -23,12 +25,16 @@ class Nostr: print(f"Nostr event sent: {output}") def generate_tags(self, order): + hashed_id = hashlib.md5( + f"{config("COORDINATOR_ALIAS", cast=str)}{order.id}".encode("utf-8") + ).hexdigest() + return [ - ["d", order.id], + ["d", uuid.UUID(hashed_id)], ["name", order.maker.robot_name], ["k", order.type.lower()], ["f", order.currency], - ["s", Order.Status(order.status).label], + ["s", self.get_status_tag(order)], ["amt", order.last_satoshis], ["fa", order.amount], ["pm", order.payment_method.split(" ")], @@ -39,9 +45,16 @@ class Nostr: ], ["expiration", order.expires_at.timestamp()], ["y", "robosats"], - ["coordinator", config("COORDINATOR_ALIAS", cast=str)]["z", "order"], ["n", order.network], ["layer", "lightning"], ["g", pygeohash.encode(order.latitude, order.longitude)], ["bond", order.bond], + ["z", "order"], + ["coordinator", config("COORDINATOR_ALIAS", cast=str)], ] + + def get_status_tag(self, order): + if order.status == Order.Status.PUB: + return "pending" + else: + return "canceled" diff --git a/api/tasks.py b/api/tasks.py index fa1d44b7..a87c1f00 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -253,7 +253,7 @@ def cache_market(): @shared_task(name="", ignore_result=True, time_limit=120) -def send_order_nostr_event(order_id=None, message=None): +def nostr_send_order_event(order_id=None): if order_id: from api.models import Order from api.nostr import Nostr @@ -261,8 +261,7 @@ def send_order_nostr_event(order_id=None, message=None): order = Order.objects.get(id=order_id) nostr = Nostr() - if message == "new": - coroutine = nostr.send_new_order_event(order) + coroutine = nostr.send_order_event(order) if coroutine: loop = asyncio.get_event_loop() loop.run_until_complete(coroutine) diff --git a/docker-tests.yml b/docker-tests.yml index 41d3b5aa..21de001f 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -205,14 +205,14 @@ services: # - redis # network_mode: service:bitcoind - # strfry: - # build: https://github.com/hoytech/strfry.git - # container_name: test-strfry - # restart: unless-stopped - # volumes: - # - ./nodeapp/strfry/config/strfry.conf:/strfry/strfry.conf:r - # network_mode: service:bitcoind - + strfry: + image: dockurr/strfry:0.9.6 + container_name: test-strfry + restart: unless-stopped + volumes: + - ./nodeapp/strfry/db:/app/strfry-db + - ./nodeapp/strfry/config/strfry.conf:/etc/strfry.conf:r + network_mode: service:bitcoind volumes: redisdata: diff --git a/requirements.txt b/requirements.txt index 6e7ae047..e6200b67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ drf-spectacular-sidecar==2024.7.1 django-cors-headers==4.4.0 base91==1.0.1 nostr-sdk==0.32.2 +pygeohash==1.2.0 diff --git a/tests/utils/trade.py b/tests/utils/trade.py index 7ac981b7..c6303b0a 100644 --- a/tests/utils/trade.py +++ b/tests/utils/trade.py @@ -5,7 +5,7 @@ from django.urls import reverse from api.management.commands.clean_orders import Command as CleanOrders from api.management.commands.follow_invoices import Command as FollowInvoices from api.models import Order -from api.tasks import follow_send_payment, send_notification, send_order_nostr_event +from api.tasks import follow_send_payment, send_notification, nostr_send_order_event from tests.utils.node import ( add_invoice, create_address, @@ -156,7 +156,7 @@ class Trade: wait_nodes_sync() @patch("api.tasks.send_notification.delay", send_notification) - @patch("api.tasks.send_order_nostr_event.delay", send_order_nostr_event) + @patch("api.tasks.nostr_send_order_event.delay", nostr_send_order_event) def publish_order(self): # Maker's first order fetch. Should trigger maker bond hold invoice generation. self.get_order() From 6635722a96615ca903fe3d19ca688faa6b7ea448 Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 15 Jul 2024 16:36:30 +0200 Subject: [PATCH 05/33] Addapt Docker --- api/nostr.py | 3 + api/tasks.py | 6 +- docker-compose.yml | 6 +- docker-tests.yml | 5 +- docker/strfry/config/strfry.conf | 138 ++++++++++++++++++ nodeapp/coordinators/exp/locations.conf | 8 + nodeapp/coordinators/lake/locations.conf | 8 + .../coordinators/satstralia/locations.conf | 8 + nodeapp/coordinators/temple/locations.conf | 8 + nodeapp/coordinators/veneto/locations.conf | 8 + nodeapp/nginx.conf | 8 + {nodeapp => tests}/strfry/config/strfry.conf | 4 +- web/nginx.conf | 8 + 13 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 docker/strfry/config/strfry.conf rename {nodeapp => tests}/strfry/config/strfry.conf (98%) diff --git a/api/nostr.py b/api/nostr.py index 4ebb5460..b1774279 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -11,6 +11,9 @@ class Nostr: async def send_order_event(self, order): """Creates the event and sends it to the coordinator relay""" + + print("Sending nostr event") + # Initialize with coordinator Keys keys = Keys.generate() signer = NostrSigner.keys(keys) diff --git a/api/tasks.py b/api/tasks.py index a87c1f00..5715c981 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,4 +1,3 @@ -import asyncio from celery import shared_task from celery.exceptions import SoftTimeLimitExceeded @@ -261,10 +260,7 @@ def nostr_send_order_event(order_id=None): order = Order.objects.get(id=order_id) nostr = Nostr() - coroutine = nostr.send_order_event(order) - if coroutine: - loop = asyncio.get_event_loop() - loop.run_until_complete(coroutine) + nostr.send_order_event(order) return diff --git a/docker-compose.yml b/docker-compose.yml index df78c7d5..afabaeb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,7 +163,6 @@ services: ports: - 8000:8000 # dev frontend build - 12596:12596 # umbrel frontend - - 7777:7777 # nostr lnd: build: ./docker/lnd @@ -228,11 +227,12 @@ services: - ./node/db:/var/lib/postgresql/data strfry: - build: https://github.com/hoytech/strfry.git + image: dockurr/strfry:0.9.6 container_name: strfry-dev restart: unless-stopped volumes: - - ./nodeapp/strfry/config/strfry.conf:/app/strfry.conf:r + - ./node/strfry/db:/app/strfry-db + - ./docker/strfry/config/strfry.conf:/etc/strfry.conf:r network_mode: service:tor # # Postgresql for CLN diff --git a/docker-tests.yml b/docker-tests.yml index 21de001f..fc6c88be 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -27,6 +27,7 @@ services: - "9998:9998" - "5432:5432" - "6379:6379" + - "7777:7777" volumes: - bitcoin:/bitcoin/.bitcoin/ - ./tests/bitcoind/entrypoint.sh:/entrypoint.sh @@ -210,8 +211,8 @@ services: container_name: test-strfry restart: unless-stopped volumes: - - ./nodeapp/strfry/db:/app/strfry-db - - ./nodeapp/strfry/config/strfry.conf:/etc/strfry.conf:r + - ./tests/strfry/db:/app/strfry-db + - ./tests/strfry/config/strfry.conf:/etc/strfry.conf:r network_mode: service:bitcoind volumes: diff --git a/docker/strfry/config/strfry.conf b/docker/strfry/config/strfry.conf new file mode 100644 index 00000000..fea0d885 --- /dev/null +++ b/docker/strfry/config/strfry.conf @@ -0,0 +1,138 @@ +## +## Default strfry config +## + +# Directory that contains the strfry LMDB database (restart required) +db = "./strfry-db/" + +dbParams { + # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) + maxreaders = 256 + + # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) + mapsize = 10995116277760 + + # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required) + noReadAhead = false +} + +events { + # Maximum size of normalised JSON, in bytes + maxEventSize = 65536 + + # Events newer than this will be rejected + rejectEventsNewerThanSeconds = 900 + + # Events older than this will be rejected + rejectEventsOlderThanSeconds = 94608000 + + # Ephemeral events older than this will be rejected + rejectEphemeralEventsOlderThanSeconds = 60 + + # Ephemeral events will be deleted from the DB when older than this + ephemeralEventsLifetimeSeconds = 300 + + # Maximum number of tags allowed + maxNumTags = 2000 + + # Maximum size for tag values, in bytes + maxTagValSize = 1024 +} + +relay { + # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) + bind = "0.0.0.0" + + # Port to open for the nostr websocket protocol (restart required) + port = 7777 + + # Set OS-limit on maximum number of open files/sockets (if 0, don't attempt to set) (restart required) + nofiles = 1000000 + + # HTTP header that contains the client's real IP, before reverse proxying (ie x-real-ip) (MUST be all lower-case) + realIpHeader = "" + + info { + # NIP-11: Name of this server. Short/descriptive (< 30 characters) + name = "Robosats" + + # NIP-11: Detailed information about relay, free-form + description = "Federation cache system." + + # NIP-11: Administrative nostr pubkey, for contact purposes + pubkey = "" + + # NIP-11: Alternative administrative contact (email, website, etc) + contact = "" + } + + # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required) + maxWebsocketPayloadSize = 131072 + + # Websocket-level PING message frequency (should be less than any reverse proxy idle timeouts) (restart required) + autoPingSeconds = 55 + + # If TCP keep-alive should be enabled (detect dropped connections to upstream reverse proxy) + enableTcpKeepalive = false + + # How much uninterrupted CPU time a REQ query should get during its DB scan + queryTimesliceBudgetMicroseconds = 10000 + + # Maximum records that can be returned per filter + maxFilterLimit = 500 + + # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time + maxSubsPerConnection = 3 + + writePolicy { + # If non-empty, path to an executable script that implements the writePolicy plugin logic + plugin = "" + } + + compression { + # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required) + enabled = true + + # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required) + slidingWindow = false + } + + logging { + # Dump all incoming messages + dumpInAll = false + + # Dump all incoming EVENT messages + dumpInEvents = false + + # Dump all incoming REQ/CLOSE messages + dumpInReqs = false + + # Log performance metrics for initial REQ database scans + dbScanPerf = false + + # Log reason for invalid event rejection? Can be disabled to silence excessive logging + invalidEvents = true + } + + numThreads { + # Ingester threads: route incoming requests, validate events/sigs (restart required) + ingester = 3 + + # reqWorker threads: Handle initial DB scan for events (restart required) + reqWorker = 3 + + # reqMonitor threads: Handle filtering of new events (restart required) + reqMonitor = 3 + + # negentropy threads: Handle negentropy protocol messages (restart required) + negentropy = 2 + } + + negentropy { + # Support negentropy protocol messages + enabled = true + + # Maximum records that sync will process before returning an error + maxSyncEvents = 1000000 + } +} diff --git a/nodeapp/coordinators/exp/locations.conf b/nodeapp/coordinators/exp/locations.conf index 7be66e85..6a31d01c 100644 --- a/nodeapp/coordinators/exp/locations.conf +++ b/nodeapp/coordinators/exp/locations.conf @@ -37,6 +37,14 @@ location /mainnet/exp/ws/ { proxy_set_header Host $host; } +location /mainnet/exp/nostr/ { + proxy_pass http://mainnet_exp/nostr/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; +} + # Experimental Coordinator Testnet Locations location /test/exp/static/assets/avatars/ { proxy_pass http://testnet_exp/static/assets/avatars/; diff --git a/nodeapp/coordinators/lake/locations.conf b/nodeapp/coordinators/lake/locations.conf index d45ea193..8c91b44a 100644 --- a/nodeapp/coordinators/lake/locations.conf +++ b/nodeapp/coordinators/lake/locations.conf @@ -37,6 +37,14 @@ location /mainnet/lake/ws/ { proxy_set_header Host $host; } +location /mainnet/lake/nostr/ { + proxy_pass http://mainnet_lake/nostr/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; +} + # TheBigLake Coordinator Testnet Locations location /test/lake/static/assets/avatars/ { proxy_pass http://testnet_lake/static/assets/avatars/; diff --git a/nodeapp/coordinators/satstralia/locations.conf b/nodeapp/coordinators/satstralia/locations.conf index 828960f8..4d17e5da 100644 --- a/nodeapp/coordinators/satstralia/locations.conf +++ b/nodeapp/coordinators/satstralia/locations.conf @@ -37,6 +37,14 @@ location /mainnet/satstralia/ws/ { proxy_set_header Host $host; } +location /mainnet/satstralia/nostr/ { + proxy_pass http://mainnet_satstralia/nostr/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; +} + # Satstralia Coordinator Testnet Locations location /test/satstralia/static/assets/avatars/ { proxy_pass http://testnet_satstralia/static/assets/avatars/; diff --git a/nodeapp/coordinators/temple/locations.conf b/nodeapp/coordinators/temple/locations.conf index 6339b5ce..6f28cd9c 100644 --- a/nodeapp/coordinators/temple/locations.conf +++ b/nodeapp/coordinators/temple/locations.conf @@ -37,6 +37,14 @@ location /mainnet/temple/ws/ { proxy_set_header Host $host; } +location /mainnet/temple/nostr/ { + proxy_pass http://mainnet_temple/nostr/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; +} + # Temple of Sats Coordinator Testnet Locations location /test/temple/static/assets/avatars/ { proxy_pass http://testnet_temple/static/assets/avatars/; diff --git a/nodeapp/coordinators/veneto/locations.conf b/nodeapp/coordinators/veneto/locations.conf index 366dee58..d5b38738 100644 --- a/nodeapp/coordinators/veneto/locations.conf +++ b/nodeapp/coordinators/veneto/locations.conf @@ -37,6 +37,14 @@ location /mainnet/veneto/ws/ { proxy_set_header Host $host; } +location /mainnet/veneto/nostr/ { + proxy_pass http://mainnet_veneto/nostr/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; +} + # BitcoinVeneto Coordinator Testnet Locations location /test/veneto/static/assets/avatars/ { proxy_pass http://testnet_veneto/static/assets/avatars/; diff --git a/nodeapp/nginx.conf b/nodeapp/nginx.conf index 6b0eedd5..a36ae165 100644 --- a/nodeapp/nginx.conf +++ b/nodeapp/nginx.conf @@ -64,6 +64,14 @@ http { autoindex on; } + location /nostr { + proxy_pass http://127.0.0.1:7777; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + } + location = /favicon.ico { alias /usr/src/robosats/static/assets/images/favicon-96x96.png; } diff --git a/nodeapp/strfry/config/strfry.conf b/tests/strfry/config/strfry.conf similarity index 98% rename from nodeapp/strfry/config/strfry.conf rename to tests/strfry/config/strfry.conf index 3bdcf584..53c7c14c 100644 --- a/nodeapp/strfry/config/strfry.conf +++ b/tests/strfry/config/strfry.conf @@ -82,7 +82,7 @@ relay { maxFilterLimit = 500 # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time - maxSubsPerConnection = 20 + maxSubsPerConnection = 3 writePolicy { # If non-empty, path to an executable script that implements the writePolicy plugin logic @@ -94,7 +94,7 @@ relay { enabled = true # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required) - slidingWindow = true + slidingWindow = false } logging { diff --git a/web/nginx.conf b/web/nginx.conf index 8fd1a981..b78e2e9f 100644 --- a/web/nginx.conf +++ b/web/nginx.conf @@ -52,6 +52,14 @@ http { alias /serve_misc/; } + location /nostr { + proxy_pass http://127.0.0.1:7777; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + } + location = /favicon.ico { alias /usr/src/robosats/static/assets/images/favicon-96x96.png; } From a6c213084652586efe5f525996ce20149b7af4b9 Mon Sep 17 00:00:00 2001 From: koalasat Date: Tue, 16 Jul 2024 11:11:08 +0200 Subject: [PATCH 06/33] Testing --- .env-sample | 3 +++ api/nostr.py | 11 +++++++---- tests/strfry/config/strfry.conf | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.env-sample b/.env-sample index 66841738..4898512c 100644 --- a/.env-sample +++ b/.env-sample @@ -175,3 +175,6 @@ SLASHED_BOND_REWARD_SPLIT = 0.5 # Username for HTLCs escrows ESCROW_USERNAME = 'admin' + +#Social +NOSTR_NSEC = 'nsec1vxhs2zc4kqe0dhz4z2gfrdyjsrwf8pg3neeqx6w4nl8djfzdp0dqwd6rxh' diff --git a/api/nostr.py b/api/nostr.py index b1774279..feb0d634 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -12,10 +12,13 @@ class Nostr: async def send_order_event(self, order): """Creates the event and sends it to the coordinator relay""" + if config("NOSTR_NSEC", cast=str, default="") == "": + return + print("Sending nostr event") # Initialize with coordinator Keys - keys = Keys.generate() + keys = Keys.parse(config("NOSTR_NSEC", cast=str)) signer = NostrSigner.keys(keys) client = Client(signer) @@ -24,6 +27,7 @@ class Nostr: await client.connect() event = EventBuilder(38383, "", self.generate_tags(order)).to_event(keys) + event.custom_created_at(order.created_at.timestamp()) output = await client.send_event(event) print(f"Nostr event sent: {output}") @@ -38,7 +42,7 @@ class Nostr: ["k", order.type.lower()], ["f", order.currency], ["s", self.get_status_tag(order)], - ["amt", order.last_satoshis], + ["amt", "0"], ["fa", order.amount], ["pm", order.payment_method.split(" ")], ["premium", order.premium_percentile], @@ -47,13 +51,12 @@ class Nostr: f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", ], ["expiration", order.expires_at.timestamp()], - ["y", "robosats"], + ["y", "robosats", config("COORDINATOR_ALIAS", cast=str)], ["n", order.network], ["layer", "lightning"], ["g", pygeohash.encode(order.latitude, order.longitude)], ["bond", order.bond], ["z", "order"], - ["coordinator", config("COORDINATOR_ALIAS", cast=str)], ] def get_status_tag(self, order): diff --git a/tests/strfry/config/strfry.conf b/tests/strfry/config/strfry.conf index 53c7c14c..fea0d885 100644 --- a/tests/strfry/config/strfry.conf +++ b/tests/strfry/config/strfry.conf @@ -41,7 +41,7 @@ events { relay { # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) - bind = "127.0.0.1" + bind = "0.0.0.0" # Port to open for the nostr websocket protocol (restart required) port = 7777 From c597cc8129fa838af27240621d5ca150e533de91 Mon Sep 17 00:00:00 2001 From: koalasat Date: Tue, 16 Jul 2024 13:16:48 +0200 Subject: [PATCH 07/33] Better tags --- api/nostr.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index feb0d634..266382d2 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -53,7 +53,7 @@ class Nostr: ["expiration", order.expires_at.timestamp()], ["y", "robosats", config("COORDINATOR_ALIAS", cast=str)], ["n", order.network], - ["layer", "lightning"], + ["layer", self.get_layer_tag(order)], ["g", pygeohash.encode(order.latitude, order.longitude)], ["bond", order.bond], ["z", "order"], @@ -63,4 +63,10 @@ class Nostr: if order.status == Order.Status.PUB: return "pending" else: - return "canceled" + return "success" + + def get_layer_tag(self, order): + if order.type == Order.Types.SELL: + return ["onchain", "lightning"] + else: + return ["lightning"] From c1e9da5c6f602f8fbdd4161bd11c03690d1b2c15 Mon Sep 17 00:00:00 2001 From: koalasat Date: Thu, 18 Jul 2024 01:33:47 +0200 Subject: [PATCH 08/33] Test entrypoint --- docker-compose.yml | 5 +++-- docker/strfry/Dockerfile | 11 +++++++++++ docker/strfry/crontab | 24 ++++++++++++++++++++++++ docker/strfry/entrypoint.sh | 3 +++ docker/strfry/{config => }/strfry.conf | 0 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 docker/strfry/Dockerfile create mode 100644 docker/strfry/crontab create mode 100755 docker/strfry/entrypoint.sh rename docker/strfry/{config => }/strfry.conf (100%) diff --git a/docker-compose.yml b/docker-compose.yml index afabaeb9..472fc72c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -227,12 +227,13 @@ services: - ./node/db:/var/lib/postgresql/data strfry: - image: dockurr/strfry:0.9.6 + build: ./docker/strfry + command: /app/strfry.sh container_name: strfry-dev restart: unless-stopped volumes: - ./node/strfry/db:/app/strfry-db - - ./docker/strfry/config/strfry.conf:/etc/strfry.conf:r + - ./docker/strfry:/etc/strfry network_mode: service:tor # # Postgresql for CLN diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile new file mode 100644 index 00000000..eb0bb4a2 --- /dev/null +++ b/docker/strfry/Dockerfile @@ -0,0 +1,11 @@ +FROM dockurr/strfry:0.9.6 + +USER root + +RUN apk add --no-cache dcron + +RUN crontab -u root /etc/crontab + +ENTRYPOINT ["/etc/entrypoint.sh"] + +CMD ["$@"] \ No newline at end of file diff --git a/docker/strfry/crontab b/docker/strfry/crontab new file mode 100644 index 00000000..ea94fd2a --- /dev/null +++ b/docker/strfry/crontab @@ -0,0 +1,24 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +*/1 * * * * bash '/app/strfry sync wss://nostr.satstralia.com --dir both >> /var/log/cron.log 2>&1' \ No newline at end of file diff --git a/docker/strfry/entrypoint.sh b/docker/strfry/entrypoint.sh new file mode 100755 index 00000000..5ea34c77 --- /dev/null +++ b/docker/strfry/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +crond -f -l 8 \ No newline at end of file diff --git a/docker/strfry/config/strfry.conf b/docker/strfry/strfry.conf similarity index 100% rename from docker/strfry/config/strfry.conf rename to docker/strfry/strfry.conf From b7f9f1827e912d7214197e07b0cd55b9a2f9b0dc Mon Sep 17 00:00:00 2001 From: koalasat Date: Fri, 19 Jul 2024 12:51:06 +0200 Subject: [PATCH 09/33] Working --- docker-compose.yml | 4 ++-- docker/strfry/Dockerfile | 10 +++++----- docker/strfry/crontab | 2 +- docker/strfry/entrypoint.sh | 4 +++- docker/strfry/strfry.conf | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 472fc72c..bd70fa70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -228,12 +228,12 @@ services: strfry: build: ./docker/strfry - command: /app/strfry.sh container_name: strfry-dev restart: unless-stopped volumes: + - ./docker/strfry/strfry.conf:/app/strfry.conf - ./node/strfry/db:/app/strfry-db - - ./docker/strfry:/etc/strfry + - ./node/strfry/log/:/var/log/ network_mode: service:tor # # Postgresql for CLN diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index eb0bb4a2..159b6e70 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -1,11 +1,11 @@ FROM dockurr/strfry:0.9.6 -USER root +RUN apk add --no-cache dcron curl iproute2 -RUN apk add --no-cache dcron +ADD . /app -RUN crontab -u root /etc/crontab +COPY entrypoint.sh /etc/strfry/entrypoint.sh -ENTRYPOINT ["/etc/entrypoint.sh"] +RUN chmod +x /etc/strfry/entrypoint.sh -CMD ["$@"] \ No newline at end of file +ENTRYPOINT ["/etc/strfry/entrypoint.sh"] diff --git a/docker/strfry/crontab b/docker/strfry/crontab index ea94fd2a..2c38b4c7 100644 --- a/docker/strfry/crontab +++ b/docker/strfry/crontab @@ -21,4 +21,4 @@ # For more information see the manual pages of crontab(5) and cron(8) # # m h dom mon dow command -*/1 * * * * bash '/app/strfry sync wss://nostr.satstralia.com --dir both >> /var/log/cron.log 2>&1' \ No newline at end of file +*/1 * * * * /app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --dir both >> /var/log/cron.log 2>&1 diff --git a/docker/strfry/entrypoint.sh b/docker/strfry/entrypoint.sh index 5ea34c77..724e755d 100755 --- a/docker/strfry/entrypoint.sh +++ b/docker/strfry/entrypoint.sh @@ -1,3 +1,5 @@ #!/bin/sh -crond -f -l 8 \ No newline at end of file +crontab /app/crontab + +crond -f -l 8 & /app/strfry.sh \ No newline at end of file diff --git a/docker/strfry/strfry.conf b/docker/strfry/strfry.conf index fea0d885..01fb191a 100644 --- a/docker/strfry/strfry.conf +++ b/docker/strfry/strfry.conf @@ -3,7 +3,7 @@ ## # Directory that contains the strfry LMDB database (restart required) -db = "./strfry-db/" +db = "/app/strfry-db/" dbParams { # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) From 334587b212c5a1072e6163ee12b87a5db36f5409 Mon Sep 17 00:00:00 2001 From: koalasat Date: Fri, 19 Jul 2024 17:45:09 +0200 Subject: [PATCH 10/33] Sync filtering --- docker/strfry/crontab | 2 +- docker/strfry/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/strfry/crontab b/docker/strfry/crontab index 2c38b4c7..6e7fef56 100644 --- a/docker/strfry/crontab +++ b/docker/strfry/crontab @@ -21,4 +21,4 @@ # For more information see the manual pages of crontab(5) and cron(8) # # m h dom mon dow command -*/1 * * * * /app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --dir both >> /var/log/cron.log 2>&1 +*/1 * * * * /app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 diff --git a/docker/strfry/entrypoint.sh b/docker/strfry/entrypoint.sh index 724e755d..b949e23d 100755 --- a/docker/strfry/entrypoint.sh +++ b/docker/strfry/entrypoint.sh @@ -2,4 +2,4 @@ crontab /app/crontab -crond -f -l 8 & /app/strfry.sh \ No newline at end of file +crond -f -l 8 & /app/strfry.sh From 7accfcf9a91a2534516737aaf7ec913d95c358e3 Mon Sep 17 00:00:00 2001 From: koalasat Date: Sat, 20 Jul 2024 15:03:04 +0200 Subject: [PATCH 11/33] Torify strfry sync --- docker-compose.yml | 1 + docker/strfry/Dockerfile | 4 +++- docker/strfry/crontab | 2 +- docker/strfry/sync.sh | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 docker/strfry/sync.sh diff --git a/docker-compose.yml b/docker-compose.yml index bd70fa70..78ad512a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,6 +232,7 @@ services: restart: unless-stopped volumes: - ./docker/strfry/strfry.conf:/app/strfry.conf + - ./docker/strfry/sync.sh:/app/sync.sh - ./node/strfry/db:/app/strfry-db - ./node/strfry/log/:/var/log/ network_mode: service:tor diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index 159b6e70..5d42b72d 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -1,6 +1,8 @@ FROM dockurr/strfry:0.9.6 -RUN apk add --no-cache dcron curl iproute2 +RUN apk add --no-cache dcron torsocks + +RUN echo "TorAddress 127.0.0.1:9050" > /etc/tor/torsocks.conf ADD . /app diff --git a/docker/strfry/crontab b/docker/strfry/crontab index 6e7fef56..ec45be69 100644 --- a/docker/strfry/crontab +++ b/docker/strfry/crontab @@ -21,4 +21,4 @@ # For more information see the manual pages of crontab(5) and cron(8) # # m h dom mon dow command -*/1 * * * * /app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +*/1 * * * * torsocks /app/sync.sh diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh new file mode 100755 index 00000000..b3d1a892 --- /dev/null +++ b/docker/strfry/sync.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 From be3bd80a6af92641383990e558e419d84aa0825c Mon Sep 17 00:00:00 2001 From: koalasat Date: Sat, 20 Jul 2024 15:03:34 +0200 Subject: [PATCH 12/33] CR --- web/nginx.conf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/web/nginx.conf b/web/nginx.conf index b78e2e9f..8fd1a981 100644 --- a/web/nginx.conf +++ b/web/nginx.conf @@ -52,14 +52,6 @@ http { alias /serve_misc/; } - location /nostr { - proxy_pass http://127.0.0.1:7777; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - } - location = /favicon.ico { alias /usr/src/robosats/static/assets/images/favicon-96x96.png; } From 9b7240e3dd1c75cba8026bf35f0c645a99003b98 Mon Sep 17 00:00:00 2001 From: koalasat Date: Sun, 21 Jul 2024 14:29:48 +0200 Subject: [PATCH 13/33] Real onions --- docker/strfry/sync.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index b3d1a892..b0b67d72 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -1,3 +1,6 @@ #!/bin/sh -/app/strfry --config /app/strfry.conf sync wss://nostr.satstralia.com --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 From c28ba4afc841e99791cb394fb513131003a275b8 Mon Sep 17 00:00:00 2001 From: koalasat Date: Sun, 21 Jul 2024 19:59:30 +0200 Subject: [PATCH 14/33] Some NIP fixes --- api/nostr.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index 266382d2..dbc97b6f 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -36,7 +36,7 @@ class Nostr: f"{config("COORDINATOR_ALIAS", cast=str)}{order.id}".encode("utf-8") ).hexdigest() - return [ + tags = [ ["d", uuid.UUID(hashed_id)], ["name", order.maker.robot_name], ["k", order.type.lower()], @@ -45,7 +45,7 @@ class Nostr: ["amt", "0"], ["fa", order.amount], ["pm", order.payment_method.split(" ")], - ["premium", order.premium_percentile], + ["premium", order.premium_percentile * 100], [ "source", f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", @@ -54,11 +54,15 @@ class Nostr: ["y", "robosats", config("COORDINATOR_ALIAS", cast=str)], ["n", order.network], ["layer", self.get_layer_tag(order)], - ["g", pygeohash.encode(order.latitude, order.longitude)], ["bond", order.bond], ["z", "order"], ] + if order.latitude and order.longitude: + tags.extend([["g", pygeohash.encode(order.latitude, order.longitude)]]) + + return tags + def get_status_tag(self, order): if order.status == Order.Status.PUB: return "pending" From ae4502903ce80ffd35dca026c2bfaa31631845a3 Mon Sep 17 00:00:00 2001 From: koalasat Date: Mon, 22 Jul 2024 14:54:03 +0200 Subject: [PATCH 15/33] Filter as variable in sync.sh --- docker/strfry/Dockerfile | 3 ++- docker/strfry/sync.sh | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index 5d42b72d..ef70fccd 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -2,7 +2,8 @@ FROM dockurr/strfry:0.9.6 RUN apk add --no-cache dcron torsocks -RUN echo "TorAddress 127.0.0.1:9050" > /etc/tor/torsocks.conf +RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf +RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf ADD . /app diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index b0b67d72..efd16094 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -1,6 +1,8 @@ #!/bin/sh -/app/strfry --config /app/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter '{"kinds":[38383]}' --dir both >> /var/log/cron.log 2>&1 +filters='{"kinds":[38383]}' + +/app/strfry --config /app/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /app/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 From ae18acdb905049fb1d4c4253c361e837d2086b4c Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 17:08:28 +0200 Subject: [PATCH 16/33] Fix unmutable conf --- docker/strfry/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index ef70fccd..7cdebe1f 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -5,8 +5,6 @@ RUN apk add --no-cache dcron torsocks RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf -ADD . /app - COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh From 7d0b3bace914773a7d686083a2ac9ba6f0c3d55c Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 17:18:13 +0200 Subject: [PATCH 17/33] Fix unmutable conf 2 --- docker/strfry/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index 7cdebe1f..c7c606ad 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -5,6 +5,7 @@ RUN apk add --no-cache dcron torsocks RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf +COPY crontab /app/crontab COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh From fda906bd9247d72dd318c84c7517a8ddf7f613c3 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 17:42:20 +0200 Subject: [PATCH 18/33] Fix unmutable conf 3 --- docker-compose.yml | 2 +- docker/strfry/sync.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 78ad512a..9778af91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -231,7 +231,7 @@ services: container_name: strfry-dev restart: unless-stopped volumes: - - ./docker/strfry/strfry.conf:/app/strfry.conf + - ./docker/strfry/strfry.conf:/etc/strfry.conf - ./docker/strfry/sync.sh:/app/sync.sh - ./node/strfry/db:/app/strfry-db - ./node/strfry/log/:/var/log/ diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index efd16094..debbd265 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -2,7 +2,7 @@ filters='{"kinds":[38383]}' -/app/strfry --config /app/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /app/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 From 73189af6afc96015a4b0873eb493140b51055cbf Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 18:09:12 +0200 Subject: [PATCH 19/33] Tesnet sync --- docker/strfry/sync.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index debbd265..dc9400f7 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -2,7 +2,4 @@ filters='{"kinds":[38383]}' -/app/strfry --config /etc/strfry.conf sync ws://ngdk7ocdzmz5kzsysa3om6du7ycj2evxp2f2olfkyq37htx3gllwp2yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://satstraoq35jffvkgpfoqld32nzw2siuvowanruindbfojowpwsjdgad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://4t4jxmivv6uqej6xzx2jx3fxh75gtt65v3szjoqmc4ugdlhipzdat6yd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://mmhaqzuirth5rx7gl24d4773lknltjhik57k7ahec5iefktezv4b3uid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 From c0d08726f9dd191a57d65189dd5779dae349896b Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 18:19:21 +0200 Subject: [PATCH 20/33] Tesnet onions --- docker/strfry/sync.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index dc9400f7..85634e20 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -3,3 +3,6 @@ filters='{"kinds":[38383]}' /app/strfry --config /etc/strfry.conf sync ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 From ab1a488a4b997b94bbc11024560209deb74b7454 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 18:33:16 +0200 Subject: [PATCH 21/33] Docker write access --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9778af91..ad71a465 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -231,10 +231,10 @@ services: container_name: strfry-dev restart: unless-stopped volumes: - - ./docker/strfry/strfry.conf:/etc/strfry.conf - - ./docker/strfry/sync.sh:/app/sync.sh - - ./node/strfry/db:/app/strfry-db - - ./node/strfry/log/:/var/log/ + - ./docker/strfry/strfry.conf:/etc/strfry.conf:ro + - ./docker/strfry/sync.sh:/app/sync.sh:ro + - ./node/strfry/db:/app/strfry-db:rw + - ./node/strfry/log/:/var/log:rw network_mode: service:tor # # Postgresql for CLN From 507082d86568b3d6be14d115d1c47ee48b6a2a7c Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 18:56:54 +0200 Subject: [PATCH 22/33] Logs to docker --- docker-compose.yml | 1 - docker/strfry/Dockerfile | 7 +++++-- docker/strfry/entrypoint.sh | 2 -- docker/strfry/sync.sh | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ad71a465..d2ad509e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,7 +234,6 @@ services: - ./docker/strfry/strfry.conf:/etc/strfry.conf:ro - ./docker/strfry/sync.sh:/app/sync.sh:ro - ./node/strfry/db:/app/strfry-db:rw - - ./node/strfry/log/:/var/log:rw network_mode: service:tor # # Postgresql for CLN diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index c7c606ad..578d0f53 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -1,11 +1,14 @@ FROM dockurr/strfry:0.9.6 -RUN apk add --no-cache dcron torsocks +RUN apk add --no-cache torsocks RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf -COPY crontab /app/crontab +# Setting up crontab +COPY crontab /tmp/crontab +RUN cat /tmp/crontab > /etc/crontabs/root + COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh diff --git a/docker/strfry/entrypoint.sh b/docker/strfry/entrypoint.sh index b949e23d..2ebe6b38 100755 --- a/docker/strfry/entrypoint.sh +++ b/docker/strfry/entrypoint.sh @@ -1,5 +1,3 @@ #!/bin/sh -crontab /app/crontab - crond -f -l 8 & /app/strfry.sh diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index 85634e20..d79feb21 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -2,7 +2,7 @@ filters='{"kinds":[38383]}' -/app/strfry --config /etc/strfry.conf sync ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 -/app/strfry --config /etc/strfry.conf sync ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/nostr --filter "$filters" --dir both >> /var/log/cron.log 2>&1 +/app/strfry --config /etc/strfry.conf sync ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr --filter "$filters" --dir both +/app/strfry --config /etc/strfry.conf sync ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/nostr --filter "$filters" --dir both +/app/strfry --config /etc/strfry.conf sync ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/nostr --filter "$filters" --dir both +/app/strfry --config /etc/strfry.conf sync ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/nostr --filter "$filters" --dir both From e3e530c08b7859dc6771f6cfc3d37bed2dae3b1e Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 19:20:13 +0200 Subject: [PATCH 23/33] executable sync --- docker-compose.yml | 2 +- docker-tests.yml | 11 ++++++----- docker/strfry/Dockerfile | 1 + docker/strfry/crontab | 2 +- docker/strfry/onion_urls.txt | 4 ++++ docker/strfry/sync.sh | 7 +++---- 6 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 docker/strfry/onion_urls.txt diff --git a/docker-compose.yml b/docker-compose.yml index d2ad509e..18f369b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,7 +232,7 @@ services: restart: unless-stopped volumes: - ./docker/strfry/strfry.conf:/etc/strfry.conf:ro - - ./docker/strfry/sync.sh:/app/sync.sh:ro + - ./docker/strfry/onion_urls.txt:/app/onion_urls.txt:ro - ./node/strfry/db:/app/strfry-db:rw network_mode: service:tor diff --git a/docker-tests.yml b/docker-tests.yml index fc6c88be..06dbd939 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -207,13 +207,14 @@ services: # network_mode: service:bitcoind strfry: - image: dockurr/strfry:0.9.6 - container_name: test-strfry + build: ./docker/strfry + container_name: strfry-dev restart: unless-stopped volumes: - - ./tests/strfry/db:/app/strfry-db - - ./tests/strfry/config/strfry.conf:/etc/strfry.conf:r - network_mode: service:bitcoind + - ./test/strfry/strfry.conf:/etc/strfry.conf:ro + - ./test/strfry/onion_urls.txt:/etc/strfry/onion_urls.txt:ro + - ./test/strfry/db:/app/strfry-db:rw + network_mode: service:tor volumes: redisdata: diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index 578d0f53..dd0612a8 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -9,6 +9,7 @@ RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf COPY crontab /tmp/crontab RUN cat /tmp/crontab > /etc/crontabs/root +COPY sync.sh /etc/strfry/sync.sh COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh diff --git a/docker/strfry/crontab b/docker/strfry/crontab index ec45be69..71a96d9a 100644 --- a/docker/strfry/crontab +++ b/docker/strfry/crontab @@ -21,4 +21,4 @@ # For more information see the manual pages of crontab(5) and cron(8) # # m h dom mon dow command -*/1 * * * * torsocks /app/sync.sh +*/1 * * * * torsocks /etc/strfry/sync.sh diff --git a/docker/strfry/onion_urls.txt b/docker/strfry/onion_urls.txt new file mode 100644 index 00000000..2c81ddc1 --- /dev/null +++ b/docker/strfry/onion_urls.txt @@ -0,0 +1,4 @@ +testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion +jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion +ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion +wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion \ No newline at end of file diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index d79feb21..338fb53d 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -2,7 +2,6 @@ filters='{"kinds":[38383]}' -/app/strfry --config /etc/strfry.conf sync ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr --filter "$filters" --dir both -/app/strfry --config /etc/strfry.conf sync ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/nostr --filter "$filters" --dir both -/app/strfry --config /etc/strfry.conf sync ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/nostr --filter "$filters" --dir both -/app/strfry --config /etc/strfry.conf sync ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/nostr --filter "$filters" --dir both +while IFS= read -r line; do + /app/strfry --config /etc/strfry.conf sync ws://${line}/nostr --filter "$filters" --dir both +done < /app/onion_urls.txt From dd35a3e69f3697de819154f6a8ed27973cc505d3 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 24 Jul 2024 19:27:44 +0200 Subject: [PATCH 24/33] executable sync --- docker/strfry/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index dd0612a8..83f8760a 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -13,5 +13,6 @@ COPY sync.sh /etc/strfry/sync.sh COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh +RUN chmod +x /etc/strfry/sync.sh ENTRYPOINT ["/etc/strfry/entrypoint.sh"] From 19fc1f08a814acbe308b2e00348e46d784df0c20 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 7 Aug 2024 00:21:56 +0200 Subject: [PATCH 25/33] Remove test dockers --- api/tasks.py | 3 +- docker-tests.yml | 10 --- tests/strfry/config/strfry.conf | 138 -------------------------------- 3 files changed, 2 insertions(+), 149 deletions(-) delete mode 100644 tests/strfry/config/strfry.conf diff --git a/api/tasks.py b/api/tasks.py index 5715c981..824bd9e6 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,3 +1,4 @@ +import asyncio from celery import shared_task from celery.exceptions import SoftTimeLimitExceeded @@ -260,7 +261,7 @@ def nostr_send_order_event(order_id=None): order = Order.objects.get(id=order_id) nostr = Nostr() - nostr.send_order_event(order) + asyncio.run(nostr.send_order_event(order)) return diff --git a/docker-tests.yml b/docker-tests.yml index 06dbd939..07010344 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -206,16 +206,6 @@ services: # - redis # network_mode: service:bitcoind - strfry: - build: ./docker/strfry - container_name: strfry-dev - restart: unless-stopped - volumes: - - ./test/strfry/strfry.conf:/etc/strfry.conf:ro - - ./test/strfry/onion_urls.txt:/etc/strfry/onion_urls.txt:ro - - ./test/strfry/db:/app/strfry-db:rw - network_mode: service:tor - volumes: redisdata: bitcoin: diff --git a/tests/strfry/config/strfry.conf b/tests/strfry/config/strfry.conf deleted file mode 100644 index fea0d885..00000000 --- a/tests/strfry/config/strfry.conf +++ /dev/null @@ -1,138 +0,0 @@ -## -## Default strfry config -## - -# Directory that contains the strfry LMDB database (restart required) -db = "./strfry-db/" - -dbParams { - # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) - maxreaders = 256 - - # Size of mmap() to use when loading LMDB (default is 10TB, does *not* correspond to disk-space used) (restart required) - mapsize = 10995116277760 - - # Disables read-ahead when accessing the LMDB mapping. Reduces IO activity when DB size is larger than RAM. (restart required) - noReadAhead = false -} - -events { - # Maximum size of normalised JSON, in bytes - maxEventSize = 65536 - - # Events newer than this will be rejected - rejectEventsNewerThanSeconds = 900 - - # Events older than this will be rejected - rejectEventsOlderThanSeconds = 94608000 - - # Ephemeral events older than this will be rejected - rejectEphemeralEventsOlderThanSeconds = 60 - - # Ephemeral events will be deleted from the DB when older than this - ephemeralEventsLifetimeSeconds = 300 - - # Maximum number of tags allowed - maxNumTags = 2000 - - # Maximum size for tag values, in bytes - maxTagValSize = 1024 -} - -relay { - # Interface to listen on. Use 0.0.0.0 to listen on all interfaces (restart required) - bind = "0.0.0.0" - - # Port to open for the nostr websocket protocol (restart required) - port = 7777 - - # Set OS-limit on maximum number of open files/sockets (if 0, don't attempt to set) (restart required) - nofiles = 1000000 - - # HTTP header that contains the client's real IP, before reverse proxying (ie x-real-ip) (MUST be all lower-case) - realIpHeader = "" - - info { - # NIP-11: Name of this server. Short/descriptive (< 30 characters) - name = "Robosats" - - # NIP-11: Detailed information about relay, free-form - description = "Federation cache system." - - # NIP-11: Administrative nostr pubkey, for contact purposes - pubkey = "" - - # NIP-11: Alternative administrative contact (email, website, etc) - contact = "" - } - - # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required) - maxWebsocketPayloadSize = 131072 - - # Websocket-level PING message frequency (should be less than any reverse proxy idle timeouts) (restart required) - autoPingSeconds = 55 - - # If TCP keep-alive should be enabled (detect dropped connections to upstream reverse proxy) - enableTcpKeepalive = false - - # How much uninterrupted CPU time a REQ query should get during its DB scan - queryTimesliceBudgetMicroseconds = 10000 - - # Maximum records that can be returned per filter - maxFilterLimit = 500 - - # Maximum number of subscriptions (concurrent REQs) a connection can have open at any time - maxSubsPerConnection = 3 - - writePolicy { - # If non-empty, path to an executable script that implements the writePolicy plugin logic - plugin = "" - } - - compression { - # Use permessage-deflate compression if supported by client. Reduces bandwidth, but slight increase in CPU (restart required) - enabled = true - - # Maintain a sliding window buffer for each connection. Improves compression, but uses more memory (restart required) - slidingWindow = false - } - - logging { - # Dump all incoming messages - dumpInAll = false - - # Dump all incoming EVENT messages - dumpInEvents = false - - # Dump all incoming REQ/CLOSE messages - dumpInReqs = false - - # Log performance metrics for initial REQ database scans - dbScanPerf = false - - # Log reason for invalid event rejection? Can be disabled to silence excessive logging - invalidEvents = true - } - - numThreads { - # Ingester threads: route incoming requests, validate events/sigs (restart required) - ingester = 3 - - # reqWorker threads: Handle initial DB scan for events (restart required) - reqWorker = 3 - - # reqMonitor threads: Handle filtering of new events (restart required) - reqMonitor = 3 - - # negentropy threads: Handle negentropy protocol messages (restart required) - negentropy = 2 - } - - negentropy { - # Support negentropy protocol messages - enabled = true - - # Maximum records that sync will process before returning an error - maxSyncEvents = 1000000 - } -} From af3a03c891a30b388a628172d2ebf7274ea79c36 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 7 Aug 2024 10:13:48 +0200 Subject: [PATCH 26/33] Async to sync --- api/nostr.py | 54 ++++++++++++++++++++++++++++++-------------- api/tasks.py | 4 ++-- requirements.txt | 1 + tests/utils/trade.py | 3 +-- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index dbc97b6f..472fb1af 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -1,7 +1,9 @@ import pygeohash import hashlib import uuid -from nostr_sdk import Keys, Client, EventBuilder, NostrSigner + +from asgiref.sync import sync_to_async +from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Kind, Tag from api.models import Order from decouple import config @@ -26,38 +28,56 @@ class Nostr: await client.add_relays(["ws://localhost:7777"]) await client.connect() - event = EventBuilder(38383, "", self.generate_tags(order)).to_event(keys) - event.custom_created_at(order.created_at.timestamp()) + robot_name = await self.get_robot_name(order) + currency = await self.get_robot_currency(order) + + event = ( + EventBuilder( + Kind(38383), + "", + Tag.parse(self.generate_tags(order, robot_name, currency)), + ) + .custom_created_at(order.created_at.timestamp()) + .to_event(keys) + ) output = await client.send_event(event) print(f"Nostr event sent: {output}") - def generate_tags(self, order): + @sync_to_async + def get_robot_name(self, order): + return order.maker.username + + @sync_to_async + def get_robot_currency(self, order): + return str(order.currency) + + def generate_tags(self, order, robot_name, currency): hashed_id = hashlib.md5( f"{config("COORDINATOR_ALIAS", cast=str)}{order.id}".encode("utf-8") ).hexdigest() tags = [ - ["d", uuid.UUID(hashed_id)], - ["name", order.maker.robot_name], - ["k", order.type.lower()], - ["f", order.currency], + ["d", str(uuid.UUID(hashed_id))], + ["name", robot_name], + ["k", "sell" if order.type == Order.Types.SELL else "buy"], + ["f", currency], ["s", self.get_status_tag(order)], ["amt", "0"], - ["fa", order.amount], - ["pm", order.payment_method.split(" ")], - ["premium", order.premium_percentile * 100], + ["fa", str(order.amount)], + ["pm"] + order.payment_method.split(" "), + ["premium", str(order.premium)], [ "source", - f"{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", + f"http://{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", ], - ["expiration", order.expires_at.timestamp()], + ["expiration", int(order.expires_at.timestamp())], ["y", "robosats", config("COORDINATOR_ALIAS", cast=str)], - ["n", order.network], - ["layer", self.get_layer_tag(order)], - ["bond", order.bond], + ["n", str(config("NETWORK"))], + ["layer"] + self.get_layer_tag(order), + ["bond", str(order.bond_size)], ["z", "order"], ] - + print(tags) if order.latitude and order.longitude: tags.extend([["g", pygeohash.encode(order.latitude, order.longitude)]]) diff --git a/api/tasks.py b/api/tasks.py index 824bd9e6..416a5847 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,4 +1,4 @@ -import asyncio +from asgiref.sync import async_to_sync from celery import shared_task from celery.exceptions import SoftTimeLimitExceeded @@ -261,7 +261,7 @@ def nostr_send_order_event(order_id=None): order = Order.objects.get(id=order_id) nostr = Nostr() - asyncio.run(nostr.send_order_event(order)) + async_to_sync(nostr.send_order_event)(order) return diff --git a/requirements.txt b/requirements.txt index e6200b67..00ca5d63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ django-cors-headers==4.4.0 base91==1.0.1 nostr-sdk==0.32.2 pygeohash==1.2.0 +asgiref == 3.8.1 diff --git a/tests/utils/trade.py b/tests/utils/trade.py index c6303b0a..39bcda0a 100644 --- a/tests/utils/trade.py +++ b/tests/utils/trade.py @@ -5,7 +5,7 @@ from django.urls import reverse from api.management.commands.clean_orders import Command as CleanOrders from api.management.commands.follow_invoices import Command as FollowInvoices from api.models import Order -from api.tasks import follow_send_payment, send_notification, nostr_send_order_event +from api.tasks import follow_send_payment, send_notification from tests.utils.node import ( add_invoice, create_address, @@ -156,7 +156,6 @@ class Trade: wait_nodes_sync() @patch("api.tasks.send_notification.delay", send_notification) - @patch("api.tasks.nostr_send_order_event.delay", nostr_send_order_event) def publish_order(self): # Maker's first order fetch. Should trigger maker bond hold invoice generation. self.get_order() From 032a48a4b2305ba7b62997d57bda82784c0f9060 Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 7 Aug 2024 22:49:16 +0200 Subject: [PATCH 27/33] Review nostr tags --- api/nostr.py | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index 472fb1af..a06573b3 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -33,9 +33,7 @@ class Nostr: event = ( EventBuilder( - Kind(38383), - "", - Tag.parse(self.generate_tags(order, robot_name, currency)), + Kind(38383), "", self.generate_tags(order, robot_name, currency) ) .custom_created_at(order.created_at.timestamp()) .to_event(keys) @@ -57,29 +55,33 @@ class Nostr: ).hexdigest() tags = [ - ["d", str(uuid.UUID(hashed_id))], - ["name", robot_name], - ["k", "sell" if order.type == Order.Types.SELL else "buy"], - ["f", currency], - ["s", self.get_status_tag(order)], - ["amt", "0"], - ["fa", str(order.amount)], - ["pm"] + order.payment_method.split(" "), - ["premium", str(order.premium)], - [ - "source", - f"http://{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", - ], - ["expiration", int(order.expires_at.timestamp())], - ["y", "robosats", config("COORDINATOR_ALIAS", cast=str)], - ["n", str(config("NETWORK"))], - ["layer"] + self.get_layer_tag(order), - ["bond", str(order.bond_size)], - ["z", "order"], + Tag.parse(["d", str(uuid.UUID(hashed_id))]), + Tag.parse(["name", robot_name]), + Tag.parse(["k", "sell" if order.type == Order.Types.SELL else "buy"]), + Tag.parse(["f", currency]), + Tag.parse(["s", self.get_status_tag(order)]), + Tag.parse(["amt", "0"]), + Tag.parse(["fa", str(order.amount)]), + Tag.parse(["pm"] + order.payment_method.split(" ")), + Tag.parse(["premium", str(order.premium)]), + Tag.parse( + [ + "source", + f"http://{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", + ] + ), + Tag.parse(["expiration", int(order.expires_at.timestamp())]), + Tag.parse(["y", "robosats", config("COORDINATOR_ALIAS", cast=str)]), + Tag.parse(["n", str(config("NETWORK"))]), + Tag.parse(["layer"] + self.get_layer_tag(order)), + Tag.parse(["bond", str(order.bond_size)]), + Tag.parse(["z", "order"]), ] print(tags) if order.latitude and order.longitude: - tags.extend([["g", pygeohash.encode(order.latitude, order.longitude)]]) + tags.extend( + [Tag.parse(["g", pygeohash.encode(order.latitude, order.longitude)])] + ) return tags From 1c7b2a892abe490fd8fea1162bf6fc414853b00b Mon Sep 17 00:00:00 2001 From: koalasat Date: Wed, 7 Aug 2024 23:03:07 +0200 Subject: [PATCH 28/33] Fix integer tag --- api/nostr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index a06573b3..b061b5d7 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -35,7 +35,7 @@ class Nostr: EventBuilder( Kind(38383), "", self.generate_tags(order, robot_name, currency) ) - .custom_created_at(order.created_at.timestamp()) + .custom_created_at(int(order.created_at.timestamp())) .to_event(keys) ) output = await client.send_event(event) @@ -70,14 +70,14 @@ class Nostr: f"http://{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", ] ), - Tag.parse(["expiration", int(order.expires_at.timestamp())]), + Tag.parse(["expiration", str(int(order.expires_at.timestamp()))]), Tag.parse(["y", "robosats", config("COORDINATOR_ALIAS", cast=str)]), Tag.parse(["n", str(config("NETWORK"))]), Tag.parse(["layer"] + self.get_layer_tag(order)), Tag.parse(["bond", str(order.bond_size)]), Tag.parse(["z", "order"]), ] - print(tags) + if order.latitude and order.longitude: tags.extend( [Tag.parse(["g", pygeohash.encode(order.latitude, order.longitude)])] From cad1a865140f4b5fa0bb365f59419173c7854722 Mon Sep 17 00:00:00 2001 From: koalasat Date: Thu, 8 Aug 2024 00:19:51 +0200 Subject: [PATCH 29/33] Fix integer tag --- api/nostr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index b061b5d7..7665217e 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -3,7 +3,7 @@ import hashlib import uuid from asgiref.sync import sync_to_async -from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Kind, Tag +from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Kind, Tag, Timestamp from api.models import Order from decouple import config @@ -30,12 +30,13 @@ class Nostr: robot_name = await self.get_robot_name(order) currency = await self.get_robot_currency(order) + created_at = Timestamp.from_secs(int(order.created_at.timestamp())) event = ( EventBuilder( Kind(38383), "", self.generate_tags(order, robot_name, currency) ) - .custom_created_at(int(order.created_at.timestamp())) + .custom_created_at(created_at) .to_event(keys) ) output = await client.send_event(event) From 3433f4544d2f07c8ca75d84c1fbefb4fa52ee21f Mon Sep 17 00:00:00 2001 From: koalasat Date: Thu, 8 Aug 2024 00:28:38 +0200 Subject: [PATCH 30/33] Fix url tag --- api/nostr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index 7665217e..f477ff9d 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -39,8 +39,8 @@ class Nostr: .custom_created_at(created_at) .to_event(keys) ) - output = await client.send_event(event) - print(f"Nostr event sent: {output}") + await client.send_event(event) + print(f"Nostr event sent: {event.as_json()}") @sync_to_async def get_robot_name(self, order): @@ -68,11 +68,11 @@ class Nostr: Tag.parse( [ "source", - f"http://{config("HOST_NAME")}/{config("COORDINATOR_ALIAS")}/order/{order.id}", + f"http://{config("HOST_NAME")}/order/{config("COORDINATOR_ALIAS", cast=str).lower()}/{order.id}", ] ), Tag.parse(["expiration", str(int(order.expires_at.timestamp()))]), - Tag.parse(["y", "robosats", config("COORDINATOR_ALIAS", cast=str)]), + Tag.parse(["y", "robosats", config("COORDINATOR_ALIAS", cast=str).lower()]), Tag.parse(["n", str(config("NETWORK"))]), Tag.parse(["layer"] + self.get_layer_tag(order)), Tag.parse(["bond", str(order.bond_size)]), From d79f27b5be50488aa8fa3879426bce4033f9b7a8 Mon Sep 17 00:00:00 2001 From: koalasat Date: Thu, 8 Aug 2024 00:45:42 +0200 Subject: [PATCH 31/33] Remove created at --- api/nostr.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index f477ff9d..e9fa94a9 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -3,7 +3,7 @@ import hashlib import uuid from asgiref.sync import sync_to_async -from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Kind, Tag, Timestamp +from nostr_sdk import Keys, Client, EventBuilder, NostrSigner, Kind, Tag from api.models import Order from decouple import config @@ -30,15 +30,10 @@ class Nostr: robot_name = await self.get_robot_name(order) currency = await self.get_robot_currency(order) - created_at = Timestamp.from_secs(int(order.created_at.timestamp())) - event = ( - EventBuilder( - Kind(38383), "", self.generate_tags(order, robot_name, currency) - ) - .custom_created_at(created_at) - .to_event(keys) - ) + event = EventBuilder( + Kind(38383), "", self.generate_tags(order, robot_name, currency) + ).to_event(keys) await client.send_event(event) print(f"Nostr event sent: {event.as_json()}") From 7ef8a1b33a0cc9254d3de5ca9b648cbf0b2df1bf Mon Sep 17 00:00:00 2001 From: koalasat Date: Fri, 9 Aug 2024 10:57:19 +0200 Subject: [PATCH 32/33] strfry bug free version --- api/nostr.py | 6 +++++- docker/strfry/Dockerfile | 31 +++++++++++++++++++++++++++---- docker/strfry/crontab | 2 +- docker/strfry/entrypoint.sh | 2 +- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/api/nostr.py b/api/nostr.py index e9fa94a9..1027c13f 100644 --- a/api/nostr.py +++ b/api/nostr.py @@ -57,7 +57,11 @@ class Nostr: Tag.parse(["f", currency]), Tag.parse(["s", self.get_status_tag(order)]), Tag.parse(["amt", "0"]), - Tag.parse(["fa", str(order.amount)]), + Tag.parse( + ["fa"] + [str(order.amount)] + if not order.has_range + else [str(order.min_amount), str(order.max_amount)] + ), Tag.parse(["pm"] + order.payment_method.split(" ")), Tag.parse(["premium", str(order.premium)]), Tag.parse( diff --git a/docker/strfry/Dockerfile b/docker/strfry/Dockerfile index 83f8760a..86a2b083 100644 --- a/docker/strfry/Dockerfile +++ b/docker/strfry/Dockerfile @@ -1,18 +1,41 @@ -FROM dockurr/strfry:0.9.6 +FROM ubuntu:jammy +ENV TZ=Europe/London -RUN apk add --no-cache torsocks +RUN apt update && apt install -y --no-install-recommends \ + git g++ make pkg-config libtool ca-certificates \ + libssl-dev zlib1g-dev liblmdb-dev libflatbuffers-dev \ + libsecp256k1-dev libzstd-dev + +# setup app +RUN git clone https://github.com/KoalaSat/strfry /app + +WORKDIR /app + +RUN git submodule update --init +RUN make setup-golpe +RUN make clean +RUN make -j4 + +RUN apt update && apt install -y --no-install-recommends \ + liblmdb0 libflatbuffers1 libsecp256k1-0 libb2-1 libzstd1 torsocks cron\ + && rm -rf /var/lib/apt/lists/* RUN echo "TorAddress 127.0.0.1" >> /etc/tor/torsocks.conf RUN echo "TorPort 9050" >> /etc/tor/torsocks.conf # Setting up crontab -COPY crontab /tmp/crontab -RUN cat /tmp/crontab > /etc/crontabs/root +COPY crontab /etc/cron.d/crontab +RUN chmod 0644 /etc/cron.d/crontab +RUN crontab /etc/cron.d/crontab +# Setting up entrypoints COPY sync.sh /etc/strfry/sync.sh COPY entrypoint.sh /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/entrypoint.sh RUN chmod +x /etc/strfry/sync.sh +#Setting up logs +RUN touch /var/log/cron.log && chmod 0644 /var/log/cron.log + ENTRYPOINT ["/etc/strfry/entrypoint.sh"] diff --git a/docker/strfry/crontab b/docker/strfry/crontab index 71a96d9a..fb04c491 100644 --- a/docker/strfry/crontab +++ b/docker/strfry/crontab @@ -21,4 +21,4 @@ # For more information see the manual pages of crontab(5) and cron(8) # # m h dom mon dow command -*/1 * * * * torsocks /etc/strfry/sync.sh +*/1 * * * * torsocks /etc/strfry/sync.sh >> /var/log/cron.log 2>&1 diff --git a/docker/strfry/entrypoint.sh b/docker/strfry/entrypoint.sh index 2ebe6b38..efb3062f 100755 --- a/docker/strfry/entrypoint.sh +++ b/docker/strfry/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -crond -f -l 8 & /app/strfry.sh +cron -f -l 8 & tail -f /var/log/cron.log & /app/strfry relay From 58d21d93e9bec4a5127bd6035ee566db82ecd1fc Mon Sep 17 00:00:00 2001 From: koalasat Date: Fri, 9 Aug 2024 17:06:41 +0200 Subject: [PATCH 33/33] Better sync urls --- docker/strfry/onion_urls.txt | 8 ++++---- docker/strfry/sync.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/strfry/onion_urls.txt b/docker/strfry/onion_urls.txt index 2c81ddc1..92abf187 100644 --- a/docker/strfry/onion_urls.txt +++ b/docker/strfry/onion_urls.txt @@ -1,4 +1,4 @@ -testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion -jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion -ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion -wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion \ No newline at end of file +ws://testraliar7xkhos2gipv2k65obykofb4jqzl5l4danfryacifi4t7qd.onion/nostr +ws://jpp3w5tpxtyg6lifonisdszpriiapszzem4wod2zsdweyfenlsxeoxid.onion/nostr +ws://ghbtv7lhoyhomyir4xvxaeyqgx4ylxksia343jaat3njqqlkqpdjqcyd.onion/nostr +ws://wsjyhbashc4zrrex6vijpryujggbka5plry2o62dxqoz3pxinblnj4ad.onion/nostr \ No newline at end of file diff --git a/docker/strfry/sync.sh b/docker/strfry/sync.sh index 338fb53d..8c476172 100755 --- a/docker/strfry/sync.sh +++ b/docker/strfry/sync.sh @@ -3,5 +3,5 @@ filters='{"kinds":[38383]}' while IFS= read -r line; do - /app/strfry --config /etc/strfry.conf sync ws://${line}/nostr --filter "$filters" --dir both + /app/strfry --config /etc/strfry.conf sync ${line} --filter "$filters" --dir both done < /app/onion_urls.txt