diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 747ace11..21e11fb3 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -22,8 +22,8 @@ jobs: matrix: python-tag: ['3.12.3-slim-bookworm', '3.13-rc-slim-bookworm'] lnd-version: ['v0.17.4-beta'] - cln-version: ['v23.11.2'] #,'v24.02'] - ln-vendor: ['LND'] #, 'CLN'] + cln-version: ['v24.05'] + ln-vendor: ['LND', 'CLN'] steps: - name: 'Checkout' @@ -56,7 +56,6 @@ jobs: env: LND_VERSION: ${{ matrix.lnd-version }} CLN_VERSION: ${{ matrix.cln-version }} - BITCOIND_VERSION: ${{ matrix.bitcoind-version }} ROBOSATS_ENVS_FILE: ".env-sample" - name: Wait for coordinator (django server) diff --git a/api/lightning/cln.py b/api/lightning/cln.py index 590ac721..c446a83a 100755 --- a/api/lightning/cln.py +++ b/api/lightning/cln.py @@ -90,9 +90,9 @@ class CLNNode: @classmethod def decode_payreq(cls, invoice): """Decodes a lightning payment request (invoice)""" - request = hold_pb2.DecodeBolt11Request(bolt11=invoice) - holdstub = hold_pb2_grpc.HoldStub(cls.hold_channel) - response = holdstub.DecodeBolt11(request) + nodestub = node_pb2_grpc.NodeStub(cls.node_channel) + request = node_pb2.DecodeRequest(string=invoice) + response = nodestub.Decode(request) return response @classmethod @@ -236,7 +236,7 @@ class CLNNode: holdstub = hold_pb2_grpc.HoldStub(cls.hold_channel) response = holdstub.HoldInvoiceCancel(request) - return response.state == hold_pb2.HoldInvoiceCancelResponse.Holdstate.CANCELED + return response.state == hold_pb2.Holdstate.CANCELED @classmethod def settle_hold_invoice(cls, preimage): @@ -247,7 +247,7 @@ class CLNNode: holdstub = hold_pb2_grpc.HoldStub(cls.hold_channel) response = holdstub.HoldInvoiceSettle(request) - return response.state == hold_pb2.HoldInvoiceSettleResponse.Holdstate.SETTLED + return response.state == hold_pb2.Holdstate.SETTLED @classmethod def gen_hold_invoice( @@ -272,7 +272,7 @@ class CLNNode: request = hold_pb2.HoldInvoiceRequest( description=description, - amount_msat=primitives__pb2.Amount(msat=num_satoshis * 1_000), + amount_msat=hold_pb2.Amount(msat=num_satoshis * 1_000), label=f"Order:{order_id}-{lnpayment_concept}-{time}", expiry=invoice_expiry, cltv=cltv_expiry_blocks, @@ -286,7 +286,7 @@ class CLNNode: hold_payment["preimage"] = preimage.hex() hold_payment["payment_hash"] = response.payment_hash.hex() hold_payment["created_at"] = timezone.make_aware( - datetime.fromtimestamp(payreq_decoded.timestamp) + datetime.fromtimestamp(payreq_decoded.created_at) ) hold_payment["expires_at"] = timezone.make_aware( datetime.fromtimestamp(response.expires_at) @@ -309,13 +309,13 @@ class CLNNode: # Will fail if 'unable to locate invoice'. Happens if invoice expiry # time has passed (but these are 15% padded at the moment). Should catch it # and report back that the invoice has expired (better robustness) - if response.state == hold_pb2.HoldInvoiceLookupResponse.Holdstate.OPEN: + if response.state == hold_pb2.Holdstate.OPEN: pass - if response.state == hold_pb2.HoldInvoiceLookupResponse.Holdstate.SETTLED: + if response.state == hold_pb2.Holdstate.SETTLED: pass - if response.state == hold_pb2.HoldInvoiceLookupResponse.Holdstate.CANCELED: + if response.state == hold_pb2.Holdstate.CANCELED: pass - if response.state == hold_pb2.HoldInvoiceLookupResponse.Holdstate.ACCEPTED: + if response.state == hold_pb2.Holdstate.ACCEPTED: lnpayment.expiry_height = response.htlc_expiry lnpayment.status = LNPayment.Status.LOCKED lnpayment.save(update_fields=["expiry_height", "status"]) @@ -359,7 +359,7 @@ class CLNNode: except Exception as e: # If it fails at finding the invoice: it has been expired for more than an hour (and could be paid or just expired). # In RoboSats DB we make a distinction between cancelled and returned - # (cln-grpc-hodl has separate state for hodl-invoices, which it forgets after an invoice expired more than an hour ago) + # (holdinvoice plugin has separate state for hodl-invoices, which it forgets after an invoice expired more than an hour ago) if "empty result for listdatastore_state" in str(e): print(str(e)) request2 = node_pb2.ListinvoicesRequest( @@ -418,7 +418,7 @@ class CLNNode: # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm # These payments will fail. So it is best to let the user know in advance this invoice is not valid. - route_hints = payreq_decoded.route_hints.hints + route_hints = payreq_decoded.routes.hints # Max amount RoboSats will pay for routing if routing_budget_ppm == 0: @@ -438,8 +438,10 @@ class CLNNode: route_cost = 0 # ...add up the cost of every hinted hop... for hop_hint in hinted_route.hops: - route_cost += hop_hint.feebase.msat / 1_000 - route_cost += hop_hint.feeprop * num_satoshis / 1_000_000 + route_cost += hop_hint.fee_base_msat.msat / 1_000 + route_cost += ( + hop_hint.fee_proportional_millionths * num_satoshis / 1_000_000 + ) # ...and store the cost of the route to the array routes_cost.append(route_cost) @@ -466,7 +468,7 @@ class CLNNode: return payout payout["created_at"] = timezone.make_aware( - datetime.fromtimestamp(payreq_decoded.timestamp) + datetime.fromtimestamp(payreq_decoded.created_at) ) payout["expires_at"] = payout["created_at"] + timedelta( seconds=payreq_decoded.expiry @@ -869,4 +871,4 @@ class CLNNode: else: raise e - return response.state == hold_pb2.HoldInvoiceLookupResponse.Holdstate.SETTLED + return response.state == hold_pb2.Holdstate.SETTLED diff --git a/docker-tests.yml b/docker-tests.yml index 94fff59d..0a4af45e 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -3,6 +3,8 @@ # Some useful handy commands that hopefully are never needed +# docker-compose -f docker-tests.yml --env-file tests/compose.env down --volumes + # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101 @@ -27,6 +29,8 @@ services: - "6379:6379" volumes: - bitcoin:/bitcoin/.bitcoin/ + - ./tests/bitcoind/entrypoint.sh:/entrypoint.sh + entrypoint: ["/entrypoint.sh"] command: --txindex=1 --printtoconsole @@ -77,16 +81,16 @@ services: network_mode: service:bitcoind coordinator-CLN: - image: elementsproject/lightningd:${CLN_VERSION:-v23.08.1} + image: elementsproject/lightningd:${CLN_VERSION:-v24.05} restart: always container_name: coordinator-CLN environment: LIGHTNINGD_NETWORK: 'regtest' volumes: - cln:/root/.lightning - - ./docker/cln/plugins/cln-grpc-hold:/root/.lightning/plugins/cln-grpc-hold + - ./docker/cln/plugins/holdinvoice:/root/.lightning/plugins/holdinvoice - bitcoin:/root/.bitcoin - command: --regtest --wumbo --bitcoin-rpcuser=test --bitcoin-rpcpassword=test --log-level=debug --rest-host=0.0.0.0 --rest-port=3010 --bind-addr=127.0.0.1:9737 --max-concurrent-htlcs=483 --grpc-port=9999 --grpc-hold-port=9998 --important-plugin=/root/.lightning/plugins/cln-grpc-hold --database-upgrade=true + command: --regtest --bitcoin-rpcuser=test --bitcoin-rpcpassword=test --developer --dev-bitcoind-poll=1 --dev-fast-gossip --log-level=debug --bind-addr=127.0.0.1:9737 --max-concurrent-htlcs=483 --grpc-port=9999 --grpc-hold-port=9998 --important-plugin=/root/.lightning/plugins/holdinvoice --database-upgrade=true depends_on: - bitcoind network_mode: service:bitcoind diff --git a/docker/cln/Dockerfile b/docker/cln/Dockerfile index 4d484d8e..653e160a 100644 --- a/docker/cln/Dockerfile +++ b/docker/cln/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bullseye-slim as builder ARG DEBIAN_FRONTEND=noninteractive -ARG LIGHTNINGD_VERSION=v23.08 +ARG LIGHTNINGD_VERSION=v24.05 RUN apt-get update -qq && \ apt-get install -qq -y --no-install-recommends \ autoconf \ @@ -18,13 +18,13 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y RUN rustup toolchain install stable --component rustfmt --allow-downgrade WORKDIR /opt/lightningd -RUN git clone --recursive --branch cln-grpc-hold https://github.com/daywalker90/lightning.git /tmp/cln-grpc-hold -RUN cd /tmp/cln-grpc-hold \ +RUN git clone https://github.com/daywalker90/holdinvoice.git /tmp/holdinvoice +RUN cd /tmp/holdinvoice \ && cargo build --release -FROM elementsproject/lightningd:v23.08 as final +FROM elementsproject/lightningd:v24.05 as final -COPY --from=builder /tmp/cln-grpc-hold/target/release/cln-grpc-hold /tmp/cln-grpc-hold +COPY --from=builder /tmp/holdinvoice/target/release/holdinvoice /tmp/holdinvoice COPY config /tmp/config COPY entrypoint.sh entrypoint.sh RUN chmod +x entrypoint.sh diff --git a/docker/cln/config b/docker/cln/config index 0730d1e7..1210a976 100644 --- a/docker/cln/config +++ b/docker/cln/config @@ -5,6 +5,6 @@ addr=statictor:127.0.0.1:9051 grpc-port=9999 grpc-hold-port=9998 always-use-proxy=true -important-plugin=/root/.lightning/plugins/cln-grpc-hold +important-plugin=/root/.lightning/plugins/holdinvoice # wallet=postgres://user:pass@localhost:5433/cln # bookkeeper-db=postgres://user:pass@localhost:5433/cln \ No newline at end of file diff --git a/docker/cln/entrypoint.sh b/docker/cln/entrypoint.sh index 637c0e3d..d9061091 100644 --- a/docker/cln/entrypoint.sh +++ b/docker/cln/entrypoint.sh @@ -17,9 +17,9 @@ if [ "$EXPOSE_TCP" == "true" ]; then socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" & fg %- else - # Always copy the cln-grpc-hodl plugin into the plugins directory on start up + # Always copy the holdinvoice plugin into the plugins directory on start up mkdir -p /root/.lightning/plugins - cp /tmp/cln-grpc-hold /root/.lightning/plugins/cln-grpc-hold + cp /tmp/holdinvoice /root/.lightning/plugins/holdinvoice if [ ! -f /root/.lightning/config ]; then cp /tmp/config /root/.lightning/config fi diff --git a/docker/cln/plugins/cln-grpc-hold b/docker/cln/plugins/cln-grpc-hold deleted file mode 100755 index 7a0d4653..00000000 Binary files a/docker/cln/plugins/cln-grpc-hold and /dev/null differ diff --git a/docker/cln/plugins/holdinvoice b/docker/cln/plugins/holdinvoice new file mode 100755 index 00000000..4818f580 Binary files /dev/null and b/docker/cln/plugins/holdinvoice differ diff --git a/scripts/generate_grpc.sh b/scripts/generate_grpc.sh index 0857eba1..fd56ea05 100755 --- a/scripts/generate_grpc.sh +++ b/scripts/generate_grpc.sh @@ -10,9 +10,9 @@ curl --parallel -o lightning.proto https://raw.githubusercontent.com/lightningne -o router.proto https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/routerrpc/router.proto \ -o signer.proto https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/signrpc/signer.proto \ -o verrpc.proto https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/verrpc/verrpc.proto \ - -o hold.proto https://raw.githubusercontent.com/daywalker90/lightning/cln-grpc-hold/proto/hold.proto \ - -o primitives.proto https://raw.githubusercontent.com/ElementsProject/lightning/v23.08/cln-grpc/proto/primitives.proto \ - -o node.proto https://raw.githubusercontent.com/ElementsProject/lightning/v23.08/cln-grpc/proto/node.proto + -o hold.proto https://raw.githubusercontent.com/daywalker90/holdinvoice/master/proto/hold.proto \ + -o primitives.proto https://raw.githubusercontent.com/ElementsProject/lightning/v24.05/cln-grpc/proto/primitives.proto \ + -o node.proto https://raw.githubusercontent.com/ElementsProject/lightning/v24.05/cln-grpc/proto/node.proto echo -n "Done\nBuilding api from GRPC specs..." python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. lightning.proto invoices.proto router.proto signer.proto verrpc.proto diff --git a/tests/bitcoind/entrypoint.sh b/tests/bitcoind/entrypoint.sh new file mode 100755 index 00000000..481b8be7 --- /dev/null +++ b/tests/bitcoind/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Start bitcoind in the background +bitcoind "$@" & + +# Wait for bitcoind to be ready +while ! bitcoin-cli --regtest --rpcuser=test --rpcpassword=test getblockchaininfo 2>/dev/null | grep '"verificationprogress":'; do + echo "Waiting for bitcoind to be ready..." + sleep 1 +done + +# Run initialization commands +bitcoin-cli --regtest --rpcuser=test --rpcpassword=test createwallet default +bitcoin-cli --regtest --rpcuser=test --rpcpassword=test generatetoaddress 1 $(bitcoin-cli --regtest --rpcuser=test --rpcpassword=test getnewaddress) + +# Bring bitcoind to the foreground +wait diff --git a/tests/compose.env b/tests/compose.env index 8fffd756..4cb71680 100644 --- a/tests/compose.env +++ b/tests/compose.env @@ -2,6 +2,6 @@ ROBOSATS_ENVS_FILE=".env-sample" BITCOIND_VERSION='24.0.1' LND_VERSION='v0.17.0-beta' -CLN_VERSION='v23.08.1' +CLN_VERSION='v24.05' REDIS_VERSION='7.2.1' POSTGRES_VERSION='14.2' \ No newline at end of file