Fix lnproxy support and add lnproxy relays workflow (#586)

* Use new lnproxy API

- Use POST instead of GET, so create and send a body parameter
- Path is /spec/ instead of /api/, and list of relays from lnproxy will contain /spec already, so path parameter for ApiClient.post() is an empty string

* add lnproxy sync workflow

* Use new lnproxy JSON structure

* Remove virtualenv

doing this so that the “scripts” subfolder in .github/workflows can be added

* Move workflow script to subfolder

* Add translation support

Locale strings not added yet

* Simplify coordinator updates, automatic migrations and collect statics (#583)

* Delete commited proto files

* Run sync workflow weekly instead of hourly

* Tweak display name for relays

* Update sync script to be append-only

* Use new naming convention for relays

* Fix bitcoinNetwork hydration

* PR Feedback

- Change hook deps from settings.network to settings
- routing_msat param updates for lnproxy API

* Actually set host in settings

* Updated parsing of LnProxy response

---------

Co-authored-by: +shyfire131 <shyfire131@shyfire131.net>
Co-authored-by: Reckless_Satoshi <90936742+Reckless-Satoshi@users.noreply.github.com>
Co-authored-by: Reckless_Satoshi <reckless.satoshi@protonmail.com>

.github/workflows/lnproxy-sync.yml

@@ -0,0 +1,30 @@
+name: Syncs relay list from lnproxy.org and stores in format used by RoboSats
+
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 12 * * 0'  # Run every Sunday at noon
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Fetch and process JSON
+        run: |
+          curl https://github.com/shyfire131/lnproxy-webui2/blob/17d6131a72a92978e5c0dc57ab2b2fbe467c7722/assets/relays.json -o lnproxy_tmplist.json
+          node .github/workflows/scripts/lnproxy-sync.js
+
+      - name: Commit and push if it's not up to date
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git add ./frontend/static/lnproxies.json
+          git diff --quiet && git diff --staged --quiet || git commit -m "Updated LNProxy relay list"
+          git push
+
+      - name: Remove tmp lnproxy json file
+        run: rm lnproxy_tmplist.json

.github/workflows/scripts/lnproxy-sync.js

@@ -0,0 +1,46 @@
+const fs = require('fs');
+
+let incomingRelays = JSON.parse(fs.readFileSync('./lnproxy_tmplist.json'));
+let existingRelays = JSON.parse(fs.readFileSync('./frontend/static/lnproxies.json'))
+
+let newRelays = [];
+
+let torCount = 0;
+let i2pCount = 0;
+let clearnetCount = 0;
+
+//Merge relay lists. URL is the unique ID used to merge records and only inserts supported. No updates or deletes
+let existingRelayURLs = existingRelays.map((relay) => relay.url);
+let newIncomingRelays = incomingRelays.filter((relay)=> existingRelayURLs.indexOf(relay) === -1)
+
+for (let url of newIncomingRelays) {
+  let relayType;
+  const LNPROXY_API_PATH = '/spec'
+  const fqdn = url.replace(LNPROXY_API_PATH, '');
+  if (fqdn.endsWith('.onion')) {
+    relayType = "TOR";
+    torCount++;
+  }
+  else if (fqdn.endsWith('i2p')) {
+    relayType = "I2P";
+    i2pCount++;
+  }
+  else {
+    relayType = "Clearnet";
+    clearnetCount++;
+  }
+
+  let relayName = `${relayType}${relayType === "TOR" ? torCount : ''}${relayType === "I2P" ? i2pCount : ''}${relayType === "Clearnet" ? clearnetCount : ''} ${url.split('/')[2].substring(0,6)}`
+
+  newRelays.push({
+    name: relayName,
+    url: url,
+    relayType: relayType,
+    network: "mainnet" //TODO: testnet
+  });
+}
+
+if (newRelays.length > 0) {
+  existingRelays.push(...newRelays);
+  fs.writeFileSync('./frontend/static/lnproxies.json', JSON.stringify(existingRelays, null, 2));
+}

frontend/src/components/TradeBox/Forms/LightningPayout.tsx

@@ -30,9 +30,10 @@ import { pn } from '../../../utils';
 import { ContentCopy, Help, SelfImprovement } from '@mui/icons-material';
 import { apiClient } from '../../../services/api';
 
-import lnproxies from '../../../../static/lnproxies.json';
 import { systemClient } from '../../../services/System';
 
+import lnproxies from '../../../../static/lnproxies.json';
+let filteredProxies: { [key: string]: any }[] = [];
 export interface LightningForm {
   invoice: string;
   amount: number;
@@ -92,7 +93,7 @@ export const LightningPayoutForm = ({
   const theme = useTheme();
 
   const [loadingLnproxy, setLoadingLnproxy] = useState<boolean>(false);
-  const [badLnproxyServer, setBadLnproxyServer] = useState<string>('');
+  const [noMatchingLnProxies, setNoMatchingLnProxies] = useState<string>('');
 
   const computeInvoiceAmount = function () {
     const tradeAmount = order.trade_satoshis;
@@ -145,49 +146,51 @@ export const LightningPayoutForm = ({
     }
   }, [lightning.lnproxyInvoice, lightning.lnproxyAmount]);
 
-  const lnproxyUrl = function () {
+  //filter lnproxies when the network settings are updated
-    const bitcoinNetwork = settings?.network ?? 'mainnet';
+  let bitcoinNetwork: string = 'mainnet';
   let internetNetwork: 'Clearnet' | 'I2P' | 'TOR' = 'Clearnet';
+  useEffect(() => {
+    bitcoinNetwork = settings?.network ?? 'mainnet';
     if (settings.host?.includes('.i2p')) {
       internetNetwork = 'I2P';
     } else if (settings.host?.includes('.onion') || window.NativeRobosats != undefined) {
       internetNetwork = 'TOR';
     }
 
-    const url = lnproxies[lightning.lnproxyServer][`${bitcoinNetwork}${internetNetwork}`];
+    filteredProxies = lnproxies
-    if (url != 'undefined') {
+      .filter((node) => node.relayType == internetNetwork)
-      return url;
+      .filter((node) => node.network == bitcoinNetwork);
-    } else {
+  }, [settings]);
-      setBadLnproxyServer(
+
-        t(`Server not available for {{bitcoinNetwork}} bitcoin over {{internetNetwork}}`, {
+  //if "use lnproxy" checkbox is enabled, but there are no matching proxies, enter error state
-          bitcoinNetwork,
+  useEffect(() => {
+    setNoMatchingLnProxies('');
+    if (filteredProxies.length === 0) {
+      setNoMatchingLnProxies(
+        t(`No proxies available for {{bitcoinNetwork}} bitcoin over {{internetNetwork}}`, {
+          bitcoinNetwork: settings?.network ?? 'mainnet',
           internetNetwork: t(internetNetwork),
         }),
       );
     }
-  };
+  }, [lightning.useLnproxy]);
-
-  useEffect(() => {
-    setBadLnproxyServer('');
-    lnproxyUrl();
-  }, [lightning.lnproxyServer]);
 
   const fetchLnproxy = function () {
     setLoadingLnproxy(true);
+    let body: { invoice: string; description: string; routing_msat?: string } = {
+      invoice: lightning.lnproxyInvoice,
+      description: '',
+    };
+    if (lightning.lnproxyBudgetSats > 0) {
+      body['routing_msat'] = String(lightning.lnproxyBudgetSats * 1000);
+    }
     apiClient
-      .get(
+      .post(filteredProxies[lightning.lnproxyServer]['url'], '', body)
-        lnproxyUrl(),
-        `/api/${lightning.lnproxyInvoice}${
-          lightning.lnproxyBudgetSats > 0
-            ? `?routing_msat=${lightning.lnproxyBudgetSats * 1000}`
-            : ''
-        }&format=json`,
-      )
       .then((data) => {
         if (data.reason) {
           setLightning({ ...lightning, badLnproxy: data.reason });
-        } else if (data.wpr) {
+        } else if (data.proxy_invoice) {
-          setLightning({ ...lightning, invoice: data.wpr, badLnproxy: '' });
+          setLightning({ ...lightning, invoice: data.proxy_invoice, badLnproxy: '' });
         } else {
           setLightning({ ...lightning, badLnproxy: 'Unknown lnproxy response' });
         }
@@ -416,7 +419,7 @@ export const LightningPayoutForm = ({
                       spacing={1}
                     >
                       <Grid item>
-                        <FormControl error={badLnproxyServer != ''}>
+                        <FormControl error={noMatchingLnProxies != ''}>
                           <InputLabel id='select-label'>{t('Server')}</InputLabel>
                           <Select
                             sx={{ width: '14em' }}
@@ -427,14 +430,14 @@ export const LightningPayoutForm = ({
                               setLightning({ ...lightning, lnproxyServer: Number(e.target.value) });
                             }}
                           >
-                            {lnproxies.map((lnproxyServer, index) => (
+                            {filteredProxies.map((lnproxyServer, index) => (
                               <MenuItem key={index} value={index}>
                                 <Typography>{lnproxyServer.name}</Typography>
                               </MenuItem>
                             ))}
                           </Select>
-                          {badLnproxyServer != '' ? (
+                          {noMatchingLnProxies != '' ? (
-                            <FormHelperText>{t(badLnproxyServer)}</FormHelperText>
+                            <FormHelperText>{t(noMatchingLnProxies)}</FormHelperText>
                           ) : (
                             <></>
                           )}
@@ -563,7 +566,7 @@ export const LightningPayoutForm = ({
                   loading={loadingLnproxy}
                   disabled={
                     lightning.lnproxyInvoice.length < 20 ||
-                    badLnproxyServer != '' ||
+                    noMatchingLnProxies != '' ||
                     lightning.badLnproxy != ''
                   }
                   onClick={fetchLnproxy}

frontend/src/models/Settings.model.ts

@@ -1,5 +1,6 @@
 import i18n from '../i18n/Web';
 import { systemClient } from '../services/System';
+import { getHost } from '../utils';
 import type Coordinator from './Coordinator.model';
 
 export type Language =
@@ -42,6 +43,7 @@ class BaseSettings {
 
     const networkCookie = systemClient.getItem('settings_network');
     this.network = networkCookie !== '' ? networkCookie : 'mainnet';
+    this.host = getHost();
   }
 
   public frontend: 'basic' | 'pro' = 'basic';

frontend/static/lnproxies.json

@@ -1,11 +1,20 @@
 [
   {
-    "name": "↬ Lnproxy Dev",
+    "name": "TOR1 w3sqmn",
-    "mainnetClearnet": "https://lnproxy.org",
+    "url": "http://w3sqmns2ct7ai2wiwzq5uplp2pqglpm6qpeey4blvn6agj3jr5abthqd.onion/spec",
-    "mainnetTOR": "http://rdq6tvulanl7aqtupmoboyk2z3suzkdwurejwyjyjf4itr3zhxrm2lad.onion",
+    "relayType": "TOR",
-    "mainnetI2P": "undefined",
+    "network": "mainnet"
-    "testnetClearnet": "undefined",
+  },
-    "testnetTOR": "undefined",
+  {
-    "testnetI2P": "undefined"
+    "name": "TOR2 rdq6tv",
+    "url": "http://rdq6tvulanl7aqtupmoboyk2z3suzkdwurejwyjyjf4itr3zhxrm2lad.onion/spec",
+    "relayType": "TOR",
+    "network": "mainnet"
+  },
+  {
+    "name": "Clearnet1 lnprox",
+    "url": "https://lnproxy.org/spec",
+    "relayType": "Clearnet",
+    "network": "mainnet"
   }
 ]