Improve market price fetching and caching

api/admin.py

@@ -2,7 +2,7 @@ from django.contrib import admin
 from django_admin_relation_links import AdminChangeLinksMixin
 from django.contrib.auth.models import Group, User
 from django.contrib.auth.admin import UserAdmin
-from .models import Order, LNPayment, Profile, MarketTick
+from .models import Order, LNPayment, Profile, MarketTick, CachedExchangeRate
 
 admin.site.unregister(Group)
 admin.site.unregister(User)
@@ -31,7 +31,7 @@ class OrderAdmin(AdminChangeLinksMixin, admin.ModelAdmin):
 
 @admin.register(LNPayment)
 class LNPaymentAdmin(AdminChangeLinksMixin, admin.ModelAdmin):
-    list_display = ('id','concept','status','num_satoshis','type','invoice','expires_at','sender_link','receiver_link')
+    list_display = ('id','concept','status','num_satoshis','type','expires_at','sender_link','receiver_link')
     list_display_links = ('id','concept')
     change_links = ('sender','receiver')
     list_filter = ('type','concept','status')
@@ -43,6 +43,11 @@ class UserProfileAdmin(AdminChangeLinksMixin, admin.ModelAdmin):
     change_links =['user']
     readonly_fields = ['avatar_tag']
 
+@admin.register(CachedExchangeRate)
+class CachedExchangeRateAdmin(admin.ModelAdmin):
+    list_display = ('currency','exchange_rate','timestamp')
+    readonly_fields = ('currency','exchange_rate','timestamp')
+
 @admin.register(MarketTick)
 class MarketTickAdmin(admin.ModelAdmin):
     list_display = ('timestamp','price','volume','premium','currency','fee')

api/logics.py

@@ -2,9 +2,8 @@ from datetime import time, timedelta
 from django.utils import timezone
 from .lightning.node import LNNode
 
-from .models import Order, LNPayment, MarketTick, User
+from .models import Order, LNPayment, MarketTick, User, CachedExchangeRate
 from decouple import config
-from .utils import get_exchange_rate
 
 import math
 
@@ -73,7 +72,7 @@ class Logics():
         if order.is_explicit:
             satoshis_now = order.satoshis
         else:
-            exchange_rate = get_exchange_rate(Order.currency_dict[str(order.currency)])
+            exchange_rate = float(CachedExchangeRate.objects.get(currency=order.currency).exchange_rate)
             premium_rate = exchange_rate * (1+float(order.premium)/100)
             satoshis_now = (float(order.amount) / premium_rate) * 100*1000*1000
 
@@ -81,12 +80,11 @@ class Logics():
 
     def price_and_premium_now(order):
         ''' computes order premium live '''
-        exchange_rate = get_exchange_rate(Order.currency_dict[str(order.currency)])
+        exchange_rate = float(CachedExchangeRate.objects.get(currency=order.currency).exchange_rate)
         if not order.is_explicit:
             premium = order.premium
             price = exchange_rate * (1+float(premium)/100)
         else:
-            exchange_rate = get_exchange_rate(Order.currency_dict[str(order.currency)])
             order_rate = float(order.amount) / (float(order.satoshis) / 100000000)
             premium = order_rate / exchange_rate - 1
             premium = int(premium*10000)/100  # 2 decimals left
@@ -339,7 +337,7 @@ class Logics():
             order.taker_bond.status = LNPayment.Status.LOCKED
             order.taker_bond.save()
 
-            # Both users profile have one more contract done
+            # Both users profiles are added one more contract
             order.maker.profile.total_contracts = order.maker.profile.total_contracts + 1
             order.taker.profile.total_contracts = order.taker.profile.total_contracts + 1
             order.maker.profile.save()

api/models.py

@@ -8,7 +8,6 @@ import uuid
 
 from decouple import config
 from pathlib import Path
-from .utils import get_exchange_rate
 import json
 
 MIN_TRADE = int(config('MIN_TRADE'))
@@ -220,6 +219,13 @@ class Profile(models.Model):
     def avatar_tag(self):
         return mark_safe('<img src="%s" width="50" height="50" />' % self.get_avatar())
 
+class CachedExchangeRate(models.Model):
+
+    currency = models.PositiveSmallIntegerField(choices=Order.currency_choices, null=False, unique=True)
+    exchange_rate = models.DecimalField(max_digits=10, decimal_places=2, default=None, null=True, validators=[MinValueValidator(0)])
+    timestamp = models.DateTimeField(auto_now_add=True)
+
+
 class MarketTick(models.Model):
     ''' 
     Records tick by tick Non-KYC Bitcoin price. 
@@ -253,7 +259,8 @@ class MarketTick(models.Model):
         elif order.taker_bond.status == LNPayment.Status.LOCKED:
             volume = order.last_satoshis / 100000000
             price = float(order.amount) / volume  # Amount Fiat / Amount BTC
-            premium = 100 * (price / get_exchange_rate(Order.currency_dict[str(order.currency)]) - 1)
+            market_exchange_rate = float(CachedExchangeRate.objects.get(currency=order.currency).exchange_rate)
+            premium = 100 * (price / market_exchange_rate - 1)
 
             tick = MarketTick.objects.create(
                 price=price, 

api/tasks.py

@@ -2,10 +2,11 @@ from celery import shared_task
 
 from .lightning.node import LNNode
 from django.contrib.auth.models import User
-from .models import LNPayment, Order
+from .models import LNPayment, Order, CachedExchangeRate
 from .logics import Logics
-from django.db.models import Q
+from .utils import get_exchange_rates
 
+from django.db.models import Q
 from datetime import timedelta
 from django.utils import timezone
 
@@ -40,7 +41,7 @@ def users_cleansing():
     return results
 
 
-@shared_task
+@shared_task(name="orders_expire")
 def orders_expire():
     pass
 
@@ -52,6 +53,21 @@ def follow_lnd_payment():
 def query_all_lnd_invoices():
     pass
 
-@shared_task
+@shared_task(name="cache_market", ignore_result=True)
 def cache_market():
-    pass
+    exchange_rates = get_exchange_rates(list(Order.currency_dict.values()))
+    results = {}
+    for val in Order.currency_dict:
+        rate = exchange_rates[int(val)-1] # currecies are indexed starting at 1 (USD)
+        results[val] = {Order.currency_dict[val], rate}
+
+        # Create / Update database cached prices
+        CachedExchangeRate.objects.update_or_create(
+            currency = int(val),
+            # if there is a Cached Exchange rate matching that value, it updates it with defaults below
+            defaults = {
+                'exchange_rate': rate,
+                'timestamp': timezone.now(),
+            })
+
+    return results

api/utils.py

@@ -1,32 +1,52 @@
 
 import requests, ring, os
 from decouple import config
-from statistics import median
+import numpy as np
 
 market_cache = {}
 
-@ring.dict(market_cache, expire=30) #keeps in cache for 30 seconds
-def get_exchange_rate(currency):
+# @ring.dict(market_cache, expire=30) #keeps in cache for 30 seconds
+def get_exchange_rates(currencies):
     '''
+    Params: list of currency codes.
     Checks for exchange rates in several public APIs.
-    Returns the median price.
+    Returns the median price list.
     '''
 
     APIS = config('MARKET_PRICE_APIS', cast=lambda v: [s.strip() for s in v.split(',')])
-    exchange_rates = []
 
+    api_rates = []
     for api_url in APIS:
-        try:
+        try: # If one API is unavailable pass
             if 'blockchain.info' in api_url:
                 blockchain_prices = requests.get(api_url).json()
-                exchange_rates.append(float(blockchain_prices[currency]['last']))
+                blockchain_rates = []
+                for currency in currencies:
+                    try: # If a currency is missing place a None
+                        blockchain_rates.append(float(blockchain_prices[currency]['last']))
+                    except:
+                        blockchain_rates.append(np.nan)
+                api_rates.append(blockchain_rates)
+
             elif 'yadio.io' in api_url:
                 yadio_prices = requests.get(api_url).json()
-                exchange_rates.append(float(yadio_prices['BTC'][currency]))
+                yadio_rates = []
+                for currency in currencies:
+                    try:
+                        yadio_rates.append(float(yadio_prices['BTC'][currency]))
+                    except:
+                        yadio_rates.append(np.nan)
+                api_rates.append(yadio_rates)
         except:
             pass
 
-    return median(exchange_rates)
+    if len(api_rates) == 0:
+        return None # Wops there is not API available!
+
+    exchange_rates = np.array(api_rates)
+    median_rates = np.nanmedian(exchange_rates, axis=0)
+
+    return median_rates.tolist()
 
 lnd_v_cache = {}
 

frontend/src/components/getFlags.js

@@ -7,7 +7,7 @@ export default function getFlags(code){
     if(code == 'CLP') return '🇨🇱';
     if(code == 'CNY') return '🇨🇳';
     if(code == 'EUR') return '🇪🇺';
-    if(code == 'HKR') return '🇨🇷';
+    if(code == 'HRK') return '🇨🇷';
     if(code == 'CZK') return '🇨🇿';
     if(code == 'DKK') return '🇩🇰';
     if(code == 'GBP') return '🇬🇧';

frontend/static/assets/currencies.json

@@ -22,7 +22,7 @@
 	"21":  "CLP",
 	"22":  "CZK",
 	"23":  "DKK",
-	"24":  "HKR",
+	"24":  "HRK",
 	"25":  "HUF",
 	"26":  "INR",
 	"27":  "ISK",

robosats/celery/__init__.py

@@ -4,6 +4,8 @@ import os
 from celery import Celery
 from celery.schedules import crontab
 
+from datetime import timedelta
+
 # You can use rabbitmq instead here.
 BASE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379')
 
@@ -27,11 +29,18 @@ app.conf.broker_url = BASE_REDIS_URL
 app.conf.beat_scheduler = 'django_celery_beat.schedulers:DatabaseScheduler'
 
 
-## Configure the periodic tasks
+# Configure the periodic tasks
 app.conf.beat_schedule = {
-    'users-cleasing-every-hour': {
+    # User cleansing every 6 hours
+    'users-cleansing': {
         'task': 'users_cleansing',
-        'schedule': 60*60,
+        'schedule': timedelta(hours=6),
+    },
+
+    'cache-market-rates': {
+        'task': 'cache_market',
+        'schedule': timedelta(seconds=60), # Cache market prices every minutes for now.
     },
 }
+
 app.conf.timezone = 'UTC'