Mirror/robosats
1
0
Fork 0
mirror of https://github.com/RoboSats/robosats.git synced 2025-01-18 12:11:35 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor LNNode, use versioner for LND get_version, refactor macaroon (#432)

Browse Source 
* Add Versioner rpc, use versioner for LND get_version, refactor macaroon

* Move LND specific rpc calls from the follow-invoices thread to LNNode

* Move LND specific rpc calls from tasks to LNNode
This commit is contained in:
Reckless_Satoshi 2023-04-22 18:54:03 +00:00 committed by GitHub
parent b227df7c7c
commit ac24c310c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 246 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -652,6 +652,7 @@ frontend/static/assets/avatars*
api/lightning/lightning*
api/lightning/invoices*
api/lightning/router*
api/lightning/verrpc*
api/lightning/googleapis*
frontend/static/locales/collected_phrases.json
frontend/static/admin*

256
api/lightning/node.py
View File

@ -16,6 +16,8 @@ from . import lightning_pb2 as lnrpc
from . import lightning_pb2_grpc as lightningstub
from . import router_pb2 as routerrpc
from . import router_pb2_grpc as routerstub
from . import verrpc_pb2 as verrpc
from . import verrpc_pb2_grpc as verrpcstub
#######
# Works with LND (c-lightning in the future for multi-vendor resiliance)
@ -44,16 +46,23 @@ class LNNode:
os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
creds = grpc.ssl_channel_credentials(CERT)
channel = grpc.secure_channel(LND_GRPC_HOST, creds)
def metadata_callback(context, callback):
callback([("macaroon", MACAROON.hex())], None)
ssl_creds = grpc.ssl_channel_credentials(CERT)
auth_creds = grpc.metadata_call_credentials(metadata_callback)
combined_creds = grpc.composite_channel_credentials(ssl_creds, auth_creds)
channel = grpc.secure_channel(LND_GRPC_HOST, combined_creds)
lightningstub = lightningstub.LightningStub(channel)
invoicesstub = invoicesstub.InvoicesStub(channel)
routerstub = routerstub.RouterStub(channel)
verrpcstub = verrpcstub.VersionerStub(channel)
lnrpc = lnrpc
invoicesrpc = invoicesrpc
routerrpc = routerrpc
verrpc = verrpc
payment_failure_context = {
0: "Payment isn't failed (yet)",
@ -64,13 +73,21 @@ class LNNode:
5: "Insufficient local balance.",
}
@classmethod
def get_version(cls):
try:
request = verrpc.VersionRequest()
response = cls.verrpcstub.GetVersion(request)
return response.version
except Exception as e:
print(e)
return None
@classmethod
def decode_payreq(cls, invoice):
"""Decodes a lightning payment request (invoice)"""
request = lnrpc.PayReqString(pay_req=invoice)
response = cls.lightningstub.DecodePayReq(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.lightningstub.DecodePayReq(request)
return response
@classmethod
@ -85,9 +102,7 @@ class LNNode:
spend_unconfirmed=False,
)
response = cls.lightningstub.EstimateFee(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.lightningstub.EstimateFee(request)
return {
"mining_fee_sats": response.fee_sat,
@ -101,9 +116,7 @@ class LNNode:
def wallet_balance(cls):
"""Returns onchain balance"""
request = lnrpc.WalletBalanceRequest()
response = cls.lightningstub.WalletBalance(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.lightningstub.WalletBalance(request)
return {
"total_balance": response.total_balance,
@ -118,9 +131,7 @@ class LNNode:
def channel_balance(cls):
"""Returns channels balance"""
request = lnrpc.ChannelBalanceRequest()
response = cls.lightningstub.ChannelBalance(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.lightningstub.ChannelBalance(request)
return {
"local_balance": response.local_balance.sat,
@ -154,9 +165,7 @@ class LNNode:
# Changing the state to "MEMPO" should be atomic with SendCoins.
onchainpayment.status = on_mempool_code
onchainpayment.save()
response = cls.lightningstub.SendCoins(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.lightningstub.SendCoins(request)
if response.txid:
onchainpayment.txid = response.txid
@ -172,9 +181,7 @@ class LNNode:
def cancel_return_hold_invoice(cls, payment_hash):
"""Cancels or returns a hold invoice"""
request = invoicesrpc.CancelInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
response = cls.invoicesstub.CancelInvoice(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.invoicesstub.CancelInvoice(request)
# Fix this: tricky because canceling sucessfully an invoice has no response. TODO
return str(response) == "" # True if no response, false otherwise.
@ -182,9 +189,7 @@ class LNNode:
def settle_hold_invoice(cls, preimage):
"""settles a hold invoice"""
request = invoicesrpc.SettleInvoiceMsg(preimage=bytes.fromhex(preimage))
response = cls.invoicesstub.SettleInvoice(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.invoicesstub.SettleInvoice(request)
# Fix this: tricky because settling sucessfully an invoice has None response. TODO
return str(response) == "" # True if no response, false otherwise.
@ -210,9 +215,7 @@ class LNNode:
), # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
cltv_expiry=cltv_expiry_blocks,
)
response = cls.invoicesstub.AddHoldInvoice(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.invoicesstub.AddHoldInvoice(request)
hold_payment["invoice"] = response.payment_request
payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
@ -236,9 +239,7 @@ class LNNode:
request = invoicesrpc.LookupInvoiceMsg(
payment_hash=bytes.fromhex(lnpayment.payment_hash)
)
response = cls.invoicesstub.LookupInvoiceV2(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.invoicesstub.LookupInvoiceV2(request)
# Will fail if 'unable to locate invoice'. Happens if invoice expiry
# time has passed (but these are 15% padded at the moment). Should catch it
@ -255,12 +256,64 @@ class LNNode:
lnpayment.save()
return True
@classmethod
def lookup_invoice_status(cls, lnpayment):
"""
Returns the status (as LNpayment.Status) of the given payment_hash
If unchanged, returns the previous status
"""
from api.models import LNPayment
status = lnpayment.status
lnd_response_state_to_lnpayment_status = {
0: LNPayment.Status.INVGEN, # OPEN
1: LNPayment.Status.SETLED, # SETTLED
2: LNPayment.Status.CANCEL, # CANCELLED
3: LNPayment.Status.LOCKED, # ACCEPTED
}
try:
# this is similar to LNNnode.validate_hold_invoice_locked
request = invoicesrpc.LookupInvoiceMsg(
payment_hash=bytes.fromhex(lnpayment.payment_hash)
)
response = cls.invoicesstub.LookupInvoiceV2(request)
# try saving expiry height
if hasattr(response, "htlcs"):
try:
lnpayment.expiry_height = response.htlcs[0].expiry_height
except Exception:
pass
status = lnd_response_state_to_lnpayment_status[response.state]
lnpayment.status = status
lnpayment.save()
except Exception as e:
# If it fails at finding the invoice: it has been canceled.
# In RoboSats DB we make a distinction between cancelled and returned (LND does not)
if "unable to locate invoice" in str(e):
print(str(e))
status = LNPayment.Status.CANCEL
lnpayment.status = status
lnpayment.save()
# LND restarted.
if "wallet locked, unlock it" in str(e):
print(str(timezone.now()) + " :: Wallet Locked")
# Other write to logs
else:
print(str(e))
return status
@classmethod
def resetmc(cls):
request = routerrpc.ResetMissionControlRequest()
_ = cls.routerstub.ResetMissionControl(
request, metadata=[("macaroon", MACAROON.hex())]
)
_ = cls.routerstub.ResetMissionControl(request)
return True
@classmethod
@ -373,9 +426,7 @@ class LNNode:
timeout_seconds=timeout_seconds,
)
for response in cls.routerstub.SendPaymentV2(
request, metadata=[("macaroon", MACAROON.hex())]
):
for response in cls.routerstub.SendPaymentV2(request):
if response.status == 0: # Status 0 'UNKNOWN'
# Not sure when this status happens
@ -407,13 +458,142 @@ class LNNode:
return False
@classmethod
def follow_send_payment(cls, lnpayment, fee_limit_sat, timeout_seconds):
"""
Sends sats to buyer, continuous update.
Has a lot of boilerplate to correctly handle every possible condition and failure case.
"""
from api.models import LNPayment, Order
hash = lnpayment.payment_hash
request = cls.routerrpc.SendPaymentRequest(
payment_request=lnpayment.invoice,
fee_limit_sat=fee_limit_sat,
timeout_seconds=timeout_seconds,
allow_self_payment=True,
)
order = lnpayment.order_paid_LN
if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
print(f"Order: {order.id} Payout is larger than collateral !?")
return
def handle_response(response, was_in_transit=False):
lnpayment.status = LNPayment.Status.FLIGHT
lnpayment.in_flight = True
lnpayment.save()
order.status = Order.Status.PAY
order.save()
if response.status == 0: # Status 0 'UNKNOWN'
# Not sure when this status happens
print(f"Order: {order.id} UNKNOWN. Hash {hash}")
lnpayment.in_flight = False
lnpayment.save()
if response.status == 1: # Status 1 'IN_FLIGHT'
print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
# If payment was already "payment is in transition" we do not
# want to spawn a new thread every 3 minutes to check on it.
# in case this thread dies, let's move the last_routing_time
# 20 minutes in the future so another thread spawns.
if was_in_transit:
lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
lnpayment.save()
if response.status == 3: # Status 3 'FAILED'
lnpayment.status = LNPayment.Status.FAILRO
lnpayment.last_routing_time = timezone.now()
lnpayment.routing_attempts += 1
lnpayment.failure_reason = response.failure_reason
lnpayment.in_flight = False
if lnpayment.routing_attempts > 2:
lnpayment.status = LNPayment.Status.EXPIRE
lnpayment.routing_attempts = 0
lnpayment.save()
order.status = Order.Status.FAI
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.FAI)
)
order.save()
print(
f"Order: {order.id} FAILED. Hash: {hash} Reason: {LNNode.payment_failure_context[response.failure_reason]}"
)
return {
"succeded": False,
"context": f"payment failure reason: {LNNode.payment_failure_context[response.failure_reason]}",
}
if response.status == 2: # Status 2 'SUCCEEDED'
print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
lnpayment.status = LNPayment.Status.SUCCED
lnpayment.fee = float(response.fee_msat) / 1000
lnpayment.preimage = response.payment_preimage
lnpayment.save()
order.status = Order.Status.SUC
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.SUC)
)
order.save()
results = {"succeded": True}
return results
try:
for response in cls.routerstub.SendPaymentV2(request):
handle_response(response)
except Exception as e:
if "invoice expired" in str(e):
print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
lnpayment.status = LNPayment.Status.EXPIRE
lnpayment.last_routing_time = timezone.now()
lnpayment.in_flight = False
lnpayment.save()
order.status = Order.Status.FAI
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.FAI)
)
order.save()
results = {
"succeded": False,
"context": "The payout invoice has expired",
}
return results
elif "payment is in transition" in str(e):
print(f"Order: {order.id} ALREADY IN TRANSITION. Hash: {hash}.")
request = routerrpc.TrackPaymentRequest(
payment_hash=bytes.fromhex(hash)
)
for response in cls.routerstub.TrackPaymentV2(request):
handle_response(response, was_in_transit=True)
elif "invoice is already paid" in str(e):
print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
request = routerrpc.TrackPaymentRequest(
payment_hash=bytes.fromhex(hash)
)
for response in cls.routerstub.TrackPaymentV2(request):
handle_response(response)
else:
print(str(e))
@classmethod
def double_check_htlc_is_settled(cls, payment_hash):
"""Just as it sounds. Better safe than sorry!"""
request = invoicesrpc.LookupInvoiceMsg(payment_hash=bytes.fromhex(payment_hash))
response = cls.invoicesstub.LookupInvoiceV2(
request, metadata=[("macaroon", MACAROON.hex())]
)
response = cls.invoicesstub.LookupInvoiceV2(request)
return (
response.state == 1

44
api/management/commands/follow_invoices.py
View File

@ -39,22 +39,13 @@ class Command(BaseCommand):
"""Follows and updates LNpayment objects
until settled or canceled
Background: SubscribeInvoices stub iterator would be great to use here.
LND Background: SubscribeInvoices stub iterator would be great to use here.
However, it only sends updates when the invoice is OPEN (new) or SETTLED.
We are very interested on the other two states (CANCELLED and ACCEPTED).
Therefore, this thread (follow_invoices) will iterate over all LNpayment
objects and do InvoiceLookupV2 every X seconds to update their state 'live'
"""
lnd_state_to_lnpayment_status = {
0: LNPayment.Status.INVGEN, # OPEN
1: LNPayment.Status.SETLED, # SETTLED
2: LNPayment.Status.CANCEL, # CANCELLED
3: LNPayment.Status.LOCKED, # ACCEPTED
}
stub = LNNode.invoicesstub
# time it for debugging
t0 = time.time()
queryset = LNPayment.objects.filter(
@ -69,38 +60,9 @@ class Command(BaseCommand):
for idx, hold_lnpayment in enumerate(queryset):
old_status = LNPayment.Status(hold_lnpayment.status).label
try:
# this is similar to LNNnode.validate_hold_invoice_locked
request = LNNode.invoicesrpc.LookupInvoiceMsg(
payment_hash=bytes.fromhex(hold_lnpayment.payment_hash)
)
response = stub.LookupInvoiceV2(
request, metadata=[("macaroon", MACAROON.hex())]
)
hold_lnpayment.status = lnd_state_to_lnpayment_status[response.state]
# try saving expiry height
if hasattr(response, "htlcs"):
try:
hold_lnpayment.expiry_height = response.htlcs[0].expiry_height
except Exception:
pass
except Exception as e:
# If it fails at finding the invoice: it has been canceled.
# In RoboSats DB we make a distinction between cancelled and returned (LND does not)
if "unable to locate invoice" in str(e):
self.stderr.write(str(e))
hold_lnpayment.status = LNPayment.Status.CANCEL
# LND restarted.
if "wallet locked, unlock it" in str(e):
self.stderr.write(str(timezone.now()) + " :: Wallet Locked")
# Other write to logs
else:
self.stderr.write(str(e))
new_status = LNPayment.Status(hold_lnpayment.status).label
status = LNNode.lookup_invoice_status(hold_lnpayment)
new_status = LNPayment.Status(status).label
# Only save the hold_payments that change (otherwise this function does not scale)
changed = not old_status == new_status

133
api/tasks.py
View File

@ -78,13 +78,11 @@ def give_rewards():
def follow_send_payment(hash):
"""Sends sats to buyer, continuous update"""
from datetime import timedelta
from decouple import config
from django.utils import timezone
from api.lightning.node import MACAROON, LNNode
from api.models import LNPayment, Order
from api.lightning.node import LNNode
from api.models import LNPayment
lnpayment = LNPayment.objects.get(payment_hash=hash)
lnpayment.last_routing_time = timezone.now()
@ -94,131 +92,10 @@ def follow_send_payment(hash):
fee_limit_sat = int(
float(lnpayment.num_satoshis) * float(lnpayment.routing_budget_ppm) / 1000000
)
timeout_seconds = int(config("PAYOUT_TIMEOUT_SECONDS"))
timeout_seconds = config("PAYOUT_TIMEOUT_SECONDS", cast=int, default=90)
request = LNNode.routerrpc.SendPaymentRequest(
payment_request=lnpayment.invoice,
fee_limit_sat=fee_limit_sat,
timeout_seconds=timeout_seconds,
allow_self_payment=True,
)
order = lnpayment.order_paid_LN
if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
print(f"Order: {order.id} Payout is larger than collateral !?")
return
def handle_response(response, was_in_transit=False):
lnpayment.status = LNPayment.Status.FLIGHT
lnpayment.in_flight = True
lnpayment.save()
order.status = Order.Status.PAY
order.save()
if response.status == 0: # Status 0 'UNKNOWN'
# Not sure when this status happens
print(f"Order: {order.id} UNKNOWN. Hash {hash}")
lnpayment.in_flight = False
lnpayment.save()
if response.status == 1: # Status 1 'IN_FLIGHT'
print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
# If payment was already "payment is in transition" we do not
# want to spawn a new thread every 3 minutes to check on it.
# in case this thread dies, let's move the last_routing_time
# 20 minutes in the future so another thread spawns.
if was_in_transit:
lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
lnpayment.save()
if response.status == 3: # Status 3 'FAILED'
lnpayment.status = LNPayment.Status.FAILRO
lnpayment.last_routing_time = timezone.now()
lnpayment.routing_attempts += 1
lnpayment.failure_reason = response.failure_reason
lnpayment.in_flight = False
if lnpayment.routing_attempts > 2:
lnpayment.status = LNPayment.Status.EXPIRE
lnpayment.routing_attempts = 0
lnpayment.save()
order.status = Order.Status.FAI
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.FAI)
)
order.save()
print(
f"Order: {order.id} FAILED. Hash: {hash} Reason: {LNNode.payment_failure_context[response.failure_reason]}"
)
return {
"succeded": False,
"context": f"payment failure reason: {LNNode.payment_failure_context[response.failure_reason]}",
}
if response.status == 2: # Status 2 'SUCCEEDED'
print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
lnpayment.status = LNPayment.Status.SUCCED
lnpayment.fee = float(response.fee_msat) / 1000
lnpayment.preimage = response.payment_preimage
lnpayment.save()
order.status = Order.Status.SUC
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.SUC)
)
order.save()
results = {"succeded": True}
return results
try:
for response in LNNode.routerstub.SendPaymentV2(
request, metadata=[("macaroon", MACAROON.hex())]
):
handle_response(response)
except Exception as e:
if "invoice expired" in str(e):
print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
lnpayment.status = LNPayment.Status.EXPIRE
lnpayment.last_routing_time = timezone.now()
lnpayment.in_flight = False
lnpayment.save()
order.status = Order.Status.FAI
order.expires_at = timezone.now() + timedelta(
seconds=order.t_to_expire(Order.Status.FAI)
)
order.save()
results = {"succeded": False, "context": "The payout invoice has expired"}
return results
elif "payment is in transition" in str(e):
print(f"Order: {order.id} ALREADY IN TRANSITION. Hash: {hash}.")
request = LNNode.routerrpc.TrackPaymentRequest(
payment_hash=bytes.fromhex(hash)
)
for response in LNNode.routerstub.TrackPaymentV2(
request, metadata=[("macaroon", MACAROON.hex())]
):
handle_response(response, was_in_transit=True)
elif "invoice is already paid" in str(e):
print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
request = LNNode.routerrpc.TrackPaymentRequest(
payment_hash=bytes.fromhex(hash)
)
for response in LNNode.routerstub.TrackPaymentV2(
request, metadata=[("macaroon", MACAROON.hex())]
):
handle_response(response)
else:
print(str(e))
results = LNNode.follow_send_payment(lnpayment, fee_limit_sat, timeout_seconds)
return results
@shared_task(name="payments_cleansing", time_limit=600)

23
api/utils.py
View File

@ -118,23 +118,16 @@ def get_exchange_rates(currencies):
return median_rates.tolist()
lnd_version_cache = {}
@ring.dict(lnd_version_cache, expire=3600)
def get_lnd_version():
# If dockerized, return LND_VERSION envvar used for docker image.
# Otherwise it would require LND's version.grpc libraries...
try:
lnd_version = config("LND_VERSION")
return lnd_version
except Exception:
pass
from api.lightning.node import LNNode
# If not dockerized and LND is local, read from CLI
try:
stream = os.popen("lnd --version")
lnd_version = stream.read()[:-1]
return lnd_version
except Exception:
return ""
print(LNNode.get_version())
return LNNode.get_version()
robosats_commit_cache = {}
@ -163,7 +156,6 @@ def get_robosats_version():
with open("version.json") as f:
version_dict = json.load(f)
print(version_dict)
return version_dict
@ -177,7 +169,6 @@ def compute_premium_percentile(order):
currency=order.currency, status=Order.Status.PUB, type=order.type
).exclude(id=order.id)
print(len(queryset))
if len(queryset) <= 1:
return 0.5

12
generate_grpc.sh
View File

@ -3,16 +3,28 @@
# generate grpc definitions
cd api/lightning
[ -d googleapis ] || git clone https://github.com/googleapis/googleapis.git googleapis
# LND Lightning proto
curl -o lightning.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/lightning.proto
python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. lightning.proto
# LND Invoices proto
curl -o invoices.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/invoicesrpc/invoices.proto
python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. invoices.proto
# LND Router proto
curl -o router.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/routerrpc/router.proto
python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. router.proto
# LND Versioner proto
curl -o verrpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/verrpc/verrpc.proto
python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. verrpc.proto
# patch generated files relative imports
sed -i 's/^import .*_pb2 as/from . \0/' router_pb2.py
sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2.py
sed -i 's/^import .*_pb2 as/from . \0/' verrpc_pb2.py
sed -i 's/^import .*_pb2 as/from . \0/' router_pb2_grpc.py
sed -i 's/^import .*_pb2 as/from . \0/' lightning_pb2_grpc.py
sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2_grpc.py
sed -i 's/^import .*_pb2 as/from . \0/' verrpc_pb2_grpc.py