Traditional environment for backend development

.env-sample

@@ -47,6 +47,9 @@ REDIS_HOST="localhost"
 REDIS_PORT="6379"
 REDIS_DB_NUMBER="1"
 
+LOG_TO_CONSOLE=False
+LOGGER_LEVEL="WARNING"
+
 # List of market price public APIs. If the currency is available in more than 1 API, will use median price.
 MARKET_PRICE_APIS = https://blockchain.info/ticker, https://api.yadio.io/exrates/BTC, https://bitpay.com/rates/BTC, https://criptoya.com/api/btc
 

.gitignore

@@ -653,3 +653,6 @@ mobile/html/Web.bundle/assets*
 
 # Protocol Buffers
 api/lightning/*.proto
+
+# Traditional environment
+regtest/

api/utils.py

@@ -387,7 +387,7 @@ def compute_avg_premium(queryset):
 
 def validate_pgp_keys(pub_key, enc_priv_key):
     """Validates PGP valid keys. Formats them in a way understandable by the frontend"""
-    gpg = gnupg.GPG()
+    gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None))
 
     # Standardize format with linux linebreaks '\n'. Windows users submitting their own keys have '\r\n' breaking communication.
     enc_priv_key = enc_priv_key.replace("\r\n", "\n").replace("\\", "\n")
@@ -441,7 +441,7 @@ def verify_signed_message(pub_key, signed_message):
     Verifies a signed cleartext PGP message. Returns whether the signature
     is valid (was made by the given pub_key) and the content of the message.
     """
-    gpg = gnupg.GPG()
+    gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None))
 
     # import the public key
     import_result = gpg.import_keys(pub_key)

api/views.py

@@ -16,6 +16,7 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from api.logics import Logics
+from api.tasks import cache_market
 from api.models import (
     Currency,
     LNPayment,
@@ -146,6 +147,9 @@ class MakerView(CreateAPIView):
                 status.HTTP_400_BAD_REQUEST,
             )
 
+        if len(Currency.objects.all()) == 0:
+            cache_market()
+
         # Creates a new order
         order = Order(
             type=type,

requirements_dev.txt

@@ -1,4 +1,5 @@
 coverage==7.5.0
 ruff==0.5.1
 git+https://github.com/Reckless-Satoshi/drf-openapi-tester.git@soften-django-requirements
-pre-commit==3.7.0
+pre-commit==3.7.0
+python-dotenv[cli]==1.0.1

robosats/settings.py

@@ -59,7 +59,7 @@ CORS_ALLOW_ALL_ORIGINS = True
 SESSION_COOKIE_HTTPONLY = False
 
 # Logging settings
-if os.environ.get("LOG_TO_CONSOLE"):
+if config("LOG_TO_CONSOLE", cast=bool, default=False):
     LOGGING = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -70,12 +70,12 @@ if os.environ.get("LOG_TO_CONSOLE"):
         },
         "root": {
             "handlers": ["console"],
-            "level": "WARNING",
+            "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")),
         },
         "loggers": {
             "api.utils": {
                 "handlers": ["console"],
-                "level": "WARNING",
+                "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")),
             },
         },
     }

scripts/traditional/README.md

@@ -0,0 +1,222 @@
+# Robosats traditional environment
+
+Robosats backend development and testing without docker and containers.
+
+Binaries needed:
+* postgresql (`postgres`, `initdb`, `psql`)
+* redis (`redis-server`)
+* bitcoin (`bitcoind`, `bitcoin-cli`)
+* cln (`lightningd`, `lightning-cli`, `holdinvoice`)
+* lnd (`lnd`, `lncli`)
+
+## Preparation
+
+Postgresql and redis can be found in all linux distros and bsds.
+
+Some distros do not put postgresql binaries in `PATH`, if this is the
+case simply link them somewhere in your `PATH`.
+
+Example on debian:
+```
+ln -sf /usr/lib/postgresql/16/bin/postgres ~/.local/bin/
+ln -sf /usr/lib/postgresql/16/bin/initdb ~/.local/bin/
+ln -sf /usr/lib/postgresql/16/bin/psql ~/.local/bin/
+```
+
+Bitcoin nodes if not already installed need to be manually downloaded.
+* bitcoin core binaries can be found here: https://bitcoincore.org/en/download
+* cln binaries can be found here: https://github.com/ElementsProject/lightning/releases
+* holdinvoice binary can be found here: https://github.com/daywalker90/holdinvoice/releases
+* lnd binaries can be found here: https://github.com/lightningnetwork/lnd/releases
+
+Example preparation:
+```
+$ python3 -m venv venv
+$ . venv/bin/activate
+$ pip install -r requirements_dev.txt
+$ pip install -r requirements.txt
+
+$ mkdir regtest
+$ mkdir regtest/programs
+$ cd regtest/programs
+$ mkdir bitcoin cln lnd
+# download bitcoin, cln (and holdinvoice) and lnd binaries
+
+# if you want to use roboauto
+# still in regtest/programs
+$ git clone https://github.com/jerryfletcher21/roboauto
+$ cd roboauto
+$ pip install -r requirements.txt
+$ pip install .
+```
+
+## env file
+
+```
+$ cp .env-sample .env
+```
+Edit `.env`, both robosats and regtest scripts will read from there.
+
+Variables to change:
+```
+LNVENDOR = "CLN"
+# LNVENDOR = "LND"
+
+LND_DIR = "regtest/nodes/lnd-coord/"
+MACAROON_PATH = "data/chain/bitcoin/regtest/admin.macaroon"
+CLN_DIR = "regtest/nodes/cln-coord/regtest/"
+BITCOIND_RPCPORT = "18443"
+POSTGRES_DB = "robosats"
+POSTGRES_USER = "robosats"
+POSTGRES_PASSWORD = "robosats"
+USE_TOR = False
+LOG_TO_CONSOLE = True
+LOGGER_LEVEL = "INFO"
+```
+
+Variables to add:
+```
+DEVELOPMENT = True
+TESTING = True
+
+LNVENDOR_ROBOT = "LND"
+# LNVENDOR_ROBOT = "CLN"
+
+BITCOIND_BIN = "regtest/programs/bitcoin/bitcoin-27.1/bin/bitcoind"
+BITCOIN_CLI_BIN = "regtest/programs/bitcoin/bitcoin-27.1/bin/bitcoin-cli"
+LIGHTNINGD_BIN = "regtest/programs/cln/usr/bin/lightningd"
+LIGHTNING_CLI_BIN = "regtest/programs/cln/usr/bin/lightning-cli"
+HOLDINVOICE_PLUGIN_BIN = "regtest/programs/cln/holdinvoice"
+LND_BIN = "regtest/programs/lnd/lnd-linux-amd64-v0.18.1-beta/lnd"
+LNCLI_BIN = "regtest/programs/lnd/lnd-linux-amd64-v0.18.1-beta/lncli"
+ROBOAUTO_GIT_DIR = "regtest/programs/roboauto"
+
+REGTEST_NODES_DIR = "regtest/nodes"
+REGTEST_SERVICES_DIR = "regtest/services"
+REGTEST_LOGS_DIR = "regtest/logs"
+GNUPG_DIR = "regtest/services/gnupg"
+
+BITCOIND_TEST_RPCUSER = "robodev"
+BITCOIND_TEST_RPCPASSWORD = "robodev"
+CLN_TEST_COORD_LISTEN_PORT = 9785
+LND_TEST_COORD_LISTEN_PORT = 9885
+LND_TEST_COORD_MACAROON_PATH = "regtest/nodes/lnd-coord/data/chain/bitcoin/regtest/admin.macaroon"
+LND_TEST_ROBOT_MACAROON_PATH = "regtest/nodes/lnd-user/data/chain/bitcoin/regtest/admin.macaroon"
+LND_TEST_ROBOT_REST_PORT = 8180
+LND_TEST_COORD_REST_PORT = 8181
+```
+
+Paths can be relative or absolute. Binaries should be paths, they are
+not resolved with `PATH`.
+
+Roboauto can be disabled by not setting `ROBOAUTO_GIT_DIR` or setting it
+to `false`.
+
+If some ports are already in use, change their value.
+
+To check which port are already in use, `netstat -tulnp` with root
+privileges can be used.
+
+For example if there is alread an instance of postgresql running on the
+default port, change `POSTGRES_PORT = "5433"`.
+
+## Usage
+
+For development and testing two scripts are provided:
+* `regtest-services` for non bitcoin related services
+* `regtest-nodes` for bitcoin and lightning nodes
+
+`regtest-services` sets up the database and manages the services.
+
+Everything is done locally, so no root privileges/service managers are
+needed.
+
+`regtest-nodes` is a script that should be sourced, it sets up a regtest
+environment, with bitcoin core, cln, lnd and roboauto, connecting them
+and creating channels. It then exposes the functions `btc_reg`,
+`cln_coord`, `cln_user`, `lnd_coord`, `lnd_user`, `ra_reg` to interact
+with the nodes and roboauto.
+
+If the script is sourced in a `bash` shell, it will also source
+completions for all the functions.
+
+`regtest-nodes` can also be run without arguments to simply expose the
+functions to start and remove the nodes and to create the channels
+between them, without setting up a specific environment.
+
+Setup:
+```
+# after having changed .env file
+
+$ . venv/bin/activate
+
+# generate cln and lnd grpc
+$ scripts/generate_grpc.sh
+
+$ scripts/traditional/regtest-services postgres-setup
+
+$ scripts/traditional/regtest-services postgres-database
+
+# postgres-database will create the database specified by
+# POSTGRES_DB in .env, it can be run multiple times with
+# different values of POSTGRES_DB for different databases
+```
+
+Testing:
+```
+# edit .env setting LNVENDOR to either "CLN" or "LND"
+# LNVENDOR_ROBOT while running tests should be set to "LND"
+
+# in the main window
+$ . venv/bin/activate
+$ . scripts/traditional/regtest-nodes test
+
+# in a secondary window
+$ . venv/bin/activate
+# can be stopped with Control-C
+$ scripts/traditional/regtest-services test
+
+# back in the main window
+$ python3 manage.py test
+
+# after having run the tests run
+$ robosats_regtest_stop_and_remove_all
+# to remove the nodes, they will be recreated when
+# running the tests again
+
+# the tests should be run with a clean database so if you have already run
+# the server and want to keep the database, either use a different value
+# of POSTGRES_DB or use a different directory by moving
+# regtest/services/postgres somewhere and the moving it back when you want
+# to run the server again
+```
+
+Development:
+```
+# edit .env setting LNVENDOR to either "CLN" or "LND"
+# and LNVENDOR_ROBOT to either "CLN" or "LND"
+
+# in the main window
+$ . venv/bin/activate
+$ . scripts/traditional/regtest-nodes server
+
+# in a secondary window
+$ . venv/bin/activate
+# can be stopped with Control-C
+$ scripts/traditional/regtest-services server
+
+# to see the output of the django runserver command
+# in a third window
+$ tail -f regtest/logs/runserver
+
+# if roboauto is active, use it to test the backend
+# back in the main window
+$ ra_reg --help
+...
+$ ra_reg create-order "$(ra_reg generate-robot --loc)" type=buy currency=btc min_amount=0.01 max_amount=0.02 payment_method="On-Chain BTC" premium=-1.5
+...
+$ ra_reg take-order $(ra_reg generate-robot --loc) order-id
+...
+$ ra_reg escrow-pay RobotName
+...
+```

scripts/traditional/regtest-services

@@ -0,0 +1,372 @@
+#!/bin/sh
+
+_command_exist() {
+    if command -v "$1" >/dev/null 2>&1; then
+        return 0
+    else
+        echo "error: $1 command not found" >&2
+        return 1
+    fi
+}
+
+_create_dir() {
+    if [ ! -e "$1" ]; then
+        mkdir -p "$1" || return "$?"
+        if [ "$#" -ge 2 ]; then
+            if ! chmod "$2" "$1"; then
+                echo "error: setting chmod $2 of $1" >&2
+                return 1
+            fi
+        fi
+    elif [ ! -d "$1" ]; then
+        echo "error: $1 is not a directory" >&2
+        return 1
+    fi
+}
+
+_get_env_var() {
+    if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then
+        echo "error: getting $1 from .env" >&2
+        return 1
+    fi
+    printf "%s\n" "$env_var"
+
+    return 0
+}
+
+# transform relative path into absolute and remove trailing slashes
+_get_env_var_path() {
+    env_var="$(_get_env_var "$1")" || return "$?"
+    real_path="$(realpath -m "$env_var")" || return "$?"
+    printf "%s\n" "$real_path"
+
+    return 0
+}
+
+_services_environment_set() {
+    # docker-compose.yml
+    CELERY_WORKER_COMMAND="celery -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000"
+    CELERY_BEAT_COMMAND="celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler"
+
+    REGTEST_SERVICES_DIR="$(_get_env_var_path "REGTEST_SERVICES_DIR")" || return "$?"
+    REGTEST_LOGS_DIR="$(_get_env_var_path "REGTEST_LOGS_DIR")" || return "$?"
+    _create_dir "$REGTEST_SERVICES_DIR" || return "$?"
+    _create_dir "$REGTEST_LOGS_DIR" || return "$?"
+
+    POSTGRES_DIR="$REGTEST_SERVICES_DIR/postgres"
+    REDIS_DIR="$REGTEST_SERVICES_DIR/redis"
+    GNUPG_DIR="$(_get_env_var_path "GNUPG_DIR")" || return "$?"
+
+    POSTGRES_DB="$(_get_env_var "POSTGRES_DB")" || return "$?"
+    POSTGRES_USER="$(_get_env_var "POSTGRES_USER")" || return "$?"
+    POSTGRES_PASS="$(_get_env_var "POSTGRES_PASSWORD")" || return "$?"
+    POSTGRES_PORT="$(_get_env_var "POSTGRES_PORT")" || return "$?"
+
+    REDIS_PORT="$(_get_env_var "REDIS_PORT")" || return "$?"
+
+    # defaults
+
+    POSTGRES_DB="${POSTGRES_DB:-postgres}"
+    POSTGRES_USER="${POSTGRES_USER:-postgres}"
+    POSTGRES_PASS="${POSTGRES_PASS:-example}"
+    POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+    REDIS_PORT="${REDIS_REGTEST_PORT:-6379}"
+}
+
+postgres_action() {
+    if [ "$#" -lt 1 ]; then
+        echo "error: insert postgres action" >&2
+        return 1
+    fi
+    action="$1"
+    shift 1
+    case "$action" in
+        setup|database) ;;
+        *)
+            echo "error: wrong action" >&2
+            return 1
+        ;;
+    esac
+
+    if ! _command_exist postgres; then
+        return 1
+    fi
+    if ! _command_exist initdb; then
+        return 1
+    fi
+    if ! _command_exist psql; then
+        return 1
+    fi
+
+    case "$action" in
+        setup)
+            if [ -e "$POSTGRES_DIR" ]; then
+                echo "postgres directory $POSTGRES_DIR already exists"
+                return 0
+            fi
+            _create_dir "$POSTGRES_DIR" "0700" || return "$?"
+
+            if ! initdb -D "$POSTGRES_DIR"; then
+                echo "error: running initdb" >&2
+                return 1
+            fi
+
+            cat << EOF > "$POSTGRES_DIR/postgresql.conf"
+port = $POSTGRES_PORT
+unix_socket_directories = '$POSTGRES_DIR'
+EOF
+        ;;
+        database)
+            if [ ! -d "$POSTGRES_DIR" ]; then
+                printf "%s%s\n" \
+                    "error: $POSTGRES_DIR is not a directory, " \
+                    "should run postgres-setup" \
+                    >&2
+                return 1
+            fi
+        ;;
+    esac
+
+    postgres_setup_log_file="$REGTEST_LOGS_DIR/postgres-setup-log"
+    echo "starting postgres, setup log file is $postgres_setup_log_file"
+    postgres -D "$POSTGRES_DIR" >>"$postgres_setup_log_file" 2>&1 &
+    postgres_pid="$!"
+
+    _postgres_shut_down() {
+        echo "shutting down postgres"
+        if ! kill "$postgres_pid"; then
+            echo "error: shutting down postgres" >&2
+            return 1
+        fi
+    }
+
+    sleep 5
+
+    case "$action" in
+        setup)
+            echo "setting up postgres user $POSTGRES_USER"
+            psql_stdin=$(cat << EOF
+CREATE ROLE $POSTGRES_USER WITH LOGIN PASSWORD '$POSTGRES_PASS';
+ALTER ROLE $POSTGRES_USER CREATEDB;
+EOF
+            )
+        ;;
+        database)
+            psql_stdin=$(cat << EOF
+CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
+EOF
+            )
+        ;;
+    esac
+    printf "%s\n" "$psql_stdin" |
+    psql -h localhost -p "$POSTGRES_PORT" -U "$USER" -d postgres
+
+    sleep 5
+
+    case "$action" in
+        database)
+            if ! DJANGO_SUPERUSER_USERNAME="$(_get_env_var "ESCROW_USERNAME")"; then
+                _postgres_shut_down || return "$?"
+                return 1
+            fi
+            # shellcheck disable=SC2034
+            DJANGO_SUPERUSER_PASSWORD="password"
+            DJANGO_SUPERUSER_EMAIL="superuser@email.com"
+
+            if ! python3 manage.py migrate; then
+                _postgres_shut_down || return "$?"
+                return 1
+            fi
+
+            if ! python3 manage.py createsuperuser \
+                --noinput \
+                --username "$DJANGO_SUPERUSER_USERNAME" \
+                --email "$DJANGO_SUPERUSER_EMAIL"
+            then
+                _postgres_shut_down || return "$?"
+                return 1
+            fi
+        ;;
+    esac
+
+    _postgres_shut_down || return "$?"
+
+    return 0
+}
+
+cleanup_signal() {
+    printf "\n"
+    printf "%s\n" "Caught $1 signal, sending it to services..."
+
+    pkill -"$2" -f "$CELERY_BEAT_COMMAND"
+    pkill -"$2" -f "$CELERY_WORKER_COMMAND"
+    pkill -"$2" -f "python3 manage.py follow_invoices"
+    pkill -"$2" -f "python3 manage.py clean_orders"
+    pkill -"$2" -f "python3 manage.py runserver 0.0.0.0:8000"
+    pkill -"$2" -f "redis-server \*:${REDIS_PORT}"
+    pkill -"$2" -f "postgres -D $POSTGRES_DIR"
+
+    exit 0
+}
+
+cleanup_int() {
+    printf "\n"
+    printf "%s\n" "Caught INT signal, shutting down services..."
+
+    pkill -INT -f "$CELERY_BEAT_COMMAND"
+    pkill -INT -f "$CELERY_WORKER_COMMAND"
+    pkill -TERM -f "python3 manage.py follow_invoices"
+    pkill -TERM -f "python3 manage.py clean_orders"
+    pkill -TERM -f "python3 manage.py runserver 0.0.0.0:8000"
+    # pkill -INT -f "python3 manage.py follow_invoices"
+    # pkill -INT -f "python3 manage.py clean_orders"
+    # pkill -INT -f "python3 manage.py runserver 0.0.0.0:8000"
+    pkill -INT -f "redis-server \*:${REDIS_PORT}"
+    pkill -INT -f "postgres -D $POSTGRES_DIR"
+
+    exit 0
+}
+
+main_loop() {
+    if [ "$#" -lt 1 ]; then
+        echo "error: insert main loop action" >&2
+        return 1
+    fi
+    case "$1" in
+        test)
+            is_test=true
+        ;;
+        server)
+            is_test=false
+        ;;
+        *)
+            echo "error: $1 is invalid" >&2
+            return 1
+        ;;
+    esac
+    shift 1
+
+    if ! _command_exist postgres; then
+        return 1
+    fi
+    if ! _command_exist redis-server; then
+        return 1
+    fi
+    if ! _command_exist celery; then
+        return 1
+    fi
+
+    if [ ! -d "$POSTGRES_DIR" ]; then
+        printf "%s%s\n" \
+            "error: $POSTGRES_DIR is not a directory, " \
+            "should run postgres-setup and postgres-database" \
+            >&2
+        return 1
+    fi
+
+    if ! pgrep -a bitcoind >/dev/null 2>&1 || {
+        ! pgrep -a lightningd >/dev/null 2>&1 &&
+        ! pgrep -a lnd >/dev/null 2>&1
+    }; then
+        printf "%s%s\n" \
+            "error: bitcoin or lightning not running, " \
+            "make sure to run this script after running regtest-nodes" \
+            >&2
+        return 1
+    fi
+
+    _create_dir "$REDIS_DIR" || return "$?"
+    _create_dir "$GNUPG_DIR" "0700" || return "$?"
+
+    trap "cleanup_signal HUP" HUP
+    trap "cleanup_signal QUIT" QUIT
+    trap "cleanup_signal TERM" TERM
+    trap "cleanup_int" INT
+
+    while true; do
+        if ! pgrep -f "postgres -D $POSTGRES_DIR" >/dev/null; then
+            echo "starting postgres"
+            postgres -D "$POSTGRES_DIR" >> "$REGTEST_LOGS_DIR/postgres" 2>&1 &
+        fi
+
+        if ! pgrep -f "redis-server \*:${REDIS_PORT}" >/dev/null; then
+            echo "starting redis"
+            printf "%s\n%s\n%s\n" \
+                "dir $REDIS_DIR" \
+                "port $REDIS_PORT" \
+                "maxclients 1024" |
+            redis-server - >> "$REGTEST_LOGS_DIR/redis" 2>&1 &
+        fi
+
+        if [ "$is_test" = false ]; then
+            if ! pgrep -f "python3 manage.py runserver 0.0.0.0:8000" >/dev/null; then
+                echo "starting runserver"
+                python3 manage.py runserver 0.0.0.0:8000 \
+                    >> "$REGTEST_LOGS_DIR/runserver" 2>&1 &
+            fi
+
+            if ! pgrep -f "python3 manage.py clean_orders" >/dev/null; then
+                echo "starting clean-orders"
+                python3 manage.py clean_orders \
+                    >> "$REGTEST_LOGS_DIR/clean-orders" 2>&1 &
+            fi
+
+            if ! pgrep -f "python3 manage.py follow_invoices" >/dev/null; then
+                echo "starting follow-invoices"
+                python3 manage.py follow_invoices \
+                    >> "$REGTEST_LOGS_DIR/follow-invoices" 2>&1 &
+            fi
+
+            if ! pgrep -f "$CELERY_WORKER_COMMAND" >/dev/null; then
+                echo "starting celery worker"
+                $CELERY_WORKER_COMMAND >> "$REGTEST_LOGS_DIR/celery-worker" 2>&1 &
+            fi
+
+            if ! pgrep -f "$CELERY_BEAT_COMMAND" >/dev/null; then
+                echo "starting celery beat"
+                $CELERY_BEAT_COMMAND >> "$REGTEST_LOGS_DIR/celery-beat" 2>&1 &
+            fi
+        fi
+
+        sleep 2
+    done
+}
+
+_services_main() {
+    if [ "$#" -lt 1 ]; then
+        echo "error: insert action" >&2
+        return 1
+    fi
+    action="$1"
+    shift 1
+
+    case "$action" in
+        -h|--help)
+            cat << EOF
+regtest-services postgres-setup|postgres-database|server|test
+EOF
+            return 0
+        ;;
+    esac
+
+    _services_environment_set || return "$?"
+
+    case "$action" in
+        postgres-setup)
+            postgres_action "setup"
+        ;;
+        postgres-database)
+            postgres_action "database"
+        ;;
+        server|test)
+            main_loop "$action"
+        ;;
+        *)
+            echo "error: action $action not recognized" >&2
+            return 1
+        ;;
+    esac
+}
+
+_services_main "$@"

scripts/traditional/robosats.bash-completion

@@ -0,0 +1,281 @@
+#!/usr/bin/env bash
+
+# adapted from
+# https://github.com/bitcoin/bitcoin/blob/master/contrib/completions/bash/bitcoin-cli.bash
+_bitcoin_cli() {
+    local cur prev words=() cword
+    local bitcoin_cli
+
+    # save and use original argument to invoke bitcoin-cli for -help, help and RPC
+    # as bitcoin-cli might not be in $PATH
+    bitcoin_cli="$1"
+
+    if ! command -v "$bitcoin_cli" >/dev/null 2>&1; then
+        return 0
+    fi
+
+    COMPREPLY=()
+    _get_comp_words_by_ref -n = cur prev words cword
+
+    if ((cword > 5)); then
+        case ${words[cword-5]} in
+            sendtoaddress)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+                return 0
+            ;;
+        esac
+    fi
+
+    if ((cword > 4)); then
+        case ${words[cword-4]} in
+            importaddress|listtransactions|setban)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+                return 0
+            ;;
+            signrawtransactionwithkey|signrawtransactionwithwallet)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "ALL NONE SINGLE ALL|ANYONECANPAY NONE|ANYONECANPAY SINGLE|ANYONECANPAY" -- "$cur" ) )
+                return 0
+            ;;
+        esac
+    fi
+
+    if ((cword > 3)); then
+        case ${words[cword-3]} in
+            addmultisigaddress)
+                return 0
+            ;;
+            getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+                return 0
+            ;;
+        esac
+    fi
+
+    if ((cword > 2)); then
+        case ${words[cword-2]} in
+            addnode)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "add remove onetry" -- "$cur" ) )
+                return 0
+            ;;
+            setban)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) )
+                return 0
+            ;;
+            fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction)
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+                return 0
+            ;;
+        esac
+    fi
+
+    case "$prev" in
+        backupwallet|dumpwallet|importwallet)
+            _filedir
+            return 0
+        ;;
+        getaddednodeinfo|getrawmempool|lockunspent)
+            # shellcheck disable=SC2207
+            COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+            return 0
+        ;;
+        getbalance|getnewaddress|listtransactions|sendmany)
+            return 0
+        ;;
+    esac
+
+    # determine already specified args necessary for RPC
+    local rpcargs=()
+    local i
+    for i in ${COMP_LINE}; do
+        case "$i" in
+            -conf=*|-datadir=*|-rpc*|-chain=*|-testnet|-signet|-regtest)
+                rpcargs=( "${rpcargs[@]}" "$i" )
+            ;;
+        esac
+    done
+
+    case "$cur" in
+        -conf=*)
+            cur="${cur#*=}"
+            _filedir
+            return 0
+        ;;
+        -datadir=*)
+            cur="${cur#*=}"
+            _filedir -d
+            return 0
+        ;;
+        -rpcwallet=*)
+            cur="${cur#*=}"
+            wallets="$($bitcoin_cli "${rpcargs[@]}" listwallets | jq -r '.[]')"
+            # shellcheck disable=SC2207
+            COMPREPLY=( $( compgen -W "$wallets" -- "$cur" ) )
+            return 0
+        ;;
+        -*=*) # prevent nonsense completions
+            return 0
+        ;;
+        *)
+            local helpopts commands completions
+
+            # only parse -help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^- ]]; then
+                helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' )
+            fi
+
+            # only parse help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then
+                commands=$($bitcoin_cli "${rpcargs[@]}" help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }')
+            fi
+
+            completions="$helpopts $commands generatetoaddress"
+
+            # shellcheck disable=SC2207
+            COMPREPLY=( $( compgen -W "$completions" -- "$cur" ) )
+
+            # Prevent space if an argument is desired
+            local word
+            for word in "${COMPREPLY[@]}"; do
+                case "$word" in
+                *=)
+                    compopt -o nospace
+                    break
+                ;;
+                esac
+            done
+            return 0
+        ;;
+    esac
+} &&
+complete -F _bitcoin_cli btc_reg
+
+# adapted from
+# https://github.com/ElementsProject/lightning/blob/master/contrib/lightning-cli.bash-completion
+_lightning_cli() {
+    local command_name="$1"
+    # local current_word="$2"
+    local previous_word="$3"
+
+    local lightning_cli
+
+    # lightning_cli might not be in $PATH
+    lightning_cli="$command_name"
+
+    if ! command -v "$lightning_cli" >/dev/null 2>&1; then
+        return 0
+    fi
+
+    if [ "${COMP_CWORD}" -eq 1 ]; then
+        complete_opt=true
+    else
+        case "$previous_word" in
+            --help|-h) complete_opt=false ;;
+            help|-*) complete_opt=true ;;
+            *) complete_opt=false ;;
+        esac
+    fi
+
+    if [ "$complete_opt" = true ]; then
+        # shellcheck disable=SC2034
+        local cur prev words=() cword
+
+        COMPREPLY=()
+        _get_comp_words_by_ref -n = cur prev words cword
+
+        case "$cur" in
+            -*=*) # prevent nonsense completions
+                return 0
+            ;;
+            *)
+                local helpopts globalcmds
+
+                # get the global options, starting with --
+                if [[ -z "$cur" || "$cur" =~ ^- ]]; then
+                    globalcmds="$(
+                        $lightning_cli --help 2>&1 |
+                        tr '|' '\n' |
+                        sed -n -e 's/ .*//' -e 's/\(-[-a-z0-9A-Z]*\).*/\1/p'
+                    )"
+                fi
+
+                # get the regular commands
+                if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then
+                    helpopts="$(
+                        $lightning_cli help 2>/dev/null |
+                        sed -n 's/^\([a-z][a-z_-]*\).*/\1/p' |
+                        sed '$ d'
+                    )"
+                fi
+
+                # shellcheck disable=SC2207
+                COMPREPLY=( $( compgen -W "$helpopts $globalcmds" -X "*," -- "$cur" ) )
+            ;;
+        esac
+    else
+        _minimal
+    fi
+} &&
+complete -F _lightning_cli cln_coord &&
+complete -F _lightning_cli cln_user
+
+# adapted from
+# https://github.com/lightningnetwork/lnd/blob/master/contrib/lncli.bash-completion
+_lncli() {
+    local cur prev words=() cword
+    local lncli
+
+    # lncli might not be in $PATH
+    lncli="$1"
+
+    if ! command -v "$lncli" >/dev/null 2>&1; then
+        return 0
+    fi
+
+    COMPREPLY=()
+    _get_comp_words_by_ref -n = cur prev words cword
+
+    case "$prev" in
+        # example of further completion
+        newaddress)
+            # shellcheck disable=SC2207
+            COMPREPLY=( $( compgen -W "p2wkh np2wkh" -- "$cur" ) )
+            return 0
+        ;;
+    esac
+
+    case "$cur" in
+        -*=*) # prevent nonsense completions
+            return 0
+        ;;
+    esac
+
+    if [ "$cword" -eq 1 ] || {
+        [ "$cword" -eq 2 ] && [ "$prev" = "help" ]
+    }; then
+        local helpopts globalcmds completions
+
+        # get the global options, starting with --
+        if [[ -z "$cur" || "$cur" =~ ^- ]]; then
+            globalcmds=$($lncli help 2>&1 | awk '$1 ~ /^-/ { sub(/,/, ""); print $1}')
+        fi
+
+        # get the regular commands
+        if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then
+            helpopts=$($lncli help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }' )
+        fi
+
+        completions="$helpopts $globalcmds help"
+
+        # shellcheck disable=SC2207
+        COMPREPLY=( $( compgen -W "$completions" -X "*," -- "$cur" ) )
+    fi
+} &&
+complete -F _lncli lnd_coord &&
+complete -F _lncli lnd_user

setup.md

@@ -111,3 +111,9 @@ You will need these commands also often or eventually:
 `docker exec -it lnd-dev lncli -network=testnet payinvoice <BOLT_11_INVOICE> --allow_self_payment`
 
 **RoboSats development site should be accessible on 127.0.0.1:8000**
+
+# Backend Development
+
+## Traditional environment without docker
+
+See [scripts/traditional/README.md](scripts/traditional/README.md)

tests/utils/node.py

@@ -16,17 +16,24 @@ def get_node(name="robot"):
     We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user)
     """
     if name == "robot":
-        macaroon = codecs.encode(
+        admin_macaroon_file = config(
-            open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(),
+            "LND_TEST_ROBOT_MACAROON_PATH", cast=str,
-            "hex",
+            default="/lndrobot/data/chain/bitcoin/regtest/admin.macaroon"
         )
-        port = 8080
+        macaroon = codecs.encode(
+            open(admin_macaroon_file, "rb").read(), "hex",
+        )
+        port = config("LND_TEST_ROBOT_REST_PORT", cast=int, default=8080)
 
     elif name == "coordinator":
-        macaroon = codecs.encode(
+        admin_macaroon_file = config(
-            open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), "hex"
+            "LND_TEST_COORD_MACAROON_PATH", cast=str,
+            default="/lnd/data/chain/bitcoin/regtest/admin.macaroon"
         )
-        port = 8081
+        macaroon = codecs.encode(
+            open(admin_macaroon_file, "rb").read(), "hex"
+        )
+        port = config("LND_TEST_COORD_REST_PORT", cast=int, default=8081)
 
     return {"port": port, "headers": {"Grpc-Metadata-macaroon": macaroon}}
 
@@ -158,10 +165,10 @@ def set_up_regtest_network():
     # Coordinator is either LND or CLN. Robot user is always LND.
     if LNVENDOR == "LND":
         coordinator_node_id = get_lnd_node_id("coordinator")
-        coordinator_port = 9735
+        coordinator_port = config("LND_TEST_COORD_LISTEN_PORT", cast=int, default=9735)
     elif LNVENDOR == "CLN":
         coordinator_node_id = get_cln_node_id()
-        coordinator_port = 9737
+        coordinator_port = config("CLN_TEST_COORD_LISTEN_PORT", cast=int, default=9737)
 
     print("Coordinator Node ID: ", coordinator_node_id)
 
@@ -258,8 +265,10 @@ def generate_blocks(address, num_blocks):
         "method": "generatetoaddress",
         "params": [num_blocks, address],
     }
+    rpc_user = config("BITCOIND_TEST_RPCUSER", cast=str, default="test")
+    rpc_pass = config("BITCOIND_TEST_RPCPASSWORD", cast=str, default="test")
     response = requests.post(
-        "http://localhost:18443", json=data, auth=HTTPBasicAuth("test", "test")
+        "http://localhost:18443", json=data, auth=HTTPBasicAuth(rpc_user, rpc_pass)
     )
     return response.json()
 

tests/utils/pgp.py

@@ -1,8 +1,9 @@
+from decouple import config
 import gnupg
 
 
 def sign_message(message, private_key_path, passphrase_path):
-    gpg = gnupg.GPG()
+    gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None))
 
     with open(private_key_path, "r") as f:
         private_key = f.read()