Add celery background and scheduled tasks. Add user cleansing task

.gitignore

@@ -639,6 +639,9 @@ FodyWeavers.xsd
 *migrations*
 frontend/static/frontend/main*
 
+# Celery
+django
+
 # robosats
 frontend/static/assets/avatars*
 api/lightning/lightning*

api/admin.py

@@ -38,7 +38,7 @@ class LNPaymentAdmin(AdminChangeLinksMixin, admin.ModelAdmin):
 
 @admin.register(Profile)
 class UserProfileAdmin(AdminChangeLinksMixin, admin.ModelAdmin):
-    list_display = ('avatar_tag','id','user_link','total_ratings','avg_rating','num_disputes','lost_disputes')
+    list_display = ('avatar_tag','id','user_link','total_contracts','total_ratings','avg_rating','num_disputes','lost_disputes')
     list_display_links = ('avatar_tag','id')
     change_links =['user']
     readonly_fields = ['avatar_tag']

api/logics.py

@@ -339,6 +339,12 @@ class Logics():
             order.taker_bond.status = LNPayment.Status.LOCKED
             order.taker_bond.save()
 
+            # Both users profile have one more contract done
+            order.maker.profile.total_contracts = order.maker.profile.total_contracts + 1
+            order.taker.profile.total_contracts = order.taker.profile.total_contracts + 1
+            order.maker.profile.save()
+            order.taker.profile.save()
+
             # Log a market tick
             MarketTick.log_a_tick(order) 
 

api/models.py

@@ -159,12 +159,12 @@ class Order(models.Model):
         return (f'Order {self.id}: {self.Types(self.type).label} BTC for {float(self.amount)} {self.currency_dict[str(self.currency)]}')
 
 @receiver(pre_delete, sender=Order)
-def delete_HTLCs_at_order_deletion(sender, instance, **kwargs):
+def delete_lnpayment_at_order_deletion(sender, instance, **kwargs):
     to_delete = (instance.maker_bond, instance.buyer_invoice, instance.taker_bond, instance.trade_escrow)
 
-    for htlc in to_delete:
+    for lnpayment in to_delete:
         try:
-            htlc.delete()
+            lnpayment.delete()
         except:
             pass
 
@@ -172,6 +172,9 @@ class Profile(models.Model):
 
     user = models.OneToOneField(User,on_delete=models.CASCADE)
 
+    # Total trades
+    total_contracts = models.PositiveIntegerField(null=False, default=0) 
+
     # Ratings stored as a comma separated integer list
     total_ratings = models.PositiveIntegerField(null=False, default=0) 
     latest_ratings = models.CharField(max_length=999, null=True, default=None, validators=[validate_comma_separated_integer_list], blank=True) # Will only store latest ratings
@@ -198,8 +201,11 @@ class Profile(models.Model):
 
     @receiver(pre_delete, sender=User)
     def del_avatar_from_disk(sender, instance, **kwargs):
-        avatar_file=Path('frontend/' + instance.profile.avatar.url)
+        try:
-        avatar_file.unlink() # FIX deleting user fails if avatar is not found
+            avatar_file=Path('frontend/' + instance.profile.avatar.url)
+            avatar_file.unlink()
+        except:
+            pass
 
     def __str__(self):
         return self.user.username

api/nick_generator/dicts/en/nouns.py

@@ -3633,7 +3633,7 @@ nouns = [
     "Fever",
     "Few",
     "Fiance",
-    "Fiancé",
+    "Fiance",
     "Fiasco",
     "Fiat",
     "Fiber",

api/tasks.py

@@ -0,0 +1,57 @@
+from celery import shared_task
+
+from .lightning.node import LNNode
+from django.contrib.auth.models import User
+from .models import LNPayment, Order
+from .logics import Logics
+from django.db.models import Q
+
+from datetime import timedelta
+from django.utils import timezone
+
+from decouple import config
+
+@shared_task(name="users_cleansing")
+def users_cleansing():
+    '''
+    Deletes users never used 12 hours after creation
+    '''
+    # Users who's last login has not been in the last 12 hours
+    active_time_range = (timezone.now() - timedelta(hours=12), timezone.now())
+    queryset = User.objects.filter(~Q(last_login__range=active_time_range))
+    
+    # And do not have an active trade or any pass finished trade.
+    deleted_users = []
+    for user in queryset:
+        if user.username == str(config('ESCROW_USERNAME')): # Do not delete admin user by mistake
+            continue
+        if not user.profile.total_contracts == 0:
+            continue
+        valid, _ = Logics.validate_already_maker_or_taker(user)
+        if valid:
+            deleted_users.append(str(user))
+            user.delete()
+
+    results = {
+        'num_deleted': len(deleted_users),
+        'deleted_users': deleted_users,
+    }
+
+    return results
+
+
+@shared_task
+def orders_expire():
+    pass
+
+@shared_task
+def follow_lnd_payment():
+    pass
+
+@shared_task
+def query_all_lnd_invoices():
+    pass
+
+@shared_task
+def cache_market():
+    pass

api/views.py

@@ -422,7 +422,7 @@ class InfoView(ListAPIView):
         context['num_public_sell_orders'] = len(Order.objects.filter(type=Order.Types.SELL, status=Order.Status.PUB))
         
         # Number of active users (logged in in last 30 minutes)
-        active_user_time_range = (timezone.now() - timedelta(minutes=30), timezone.now())
+        active_user_time_range = (timezone.now() - timedelta(minutes=120), timezone.now())
         context['num_active_robotsats'] = len(User.objects.filter(last_login__range=active_user_time_range))
 
         # Compute average premium and volume of today

chat/templates/chatroom.html

@@ -1,82 +0,0 @@
-<html lang="en">
-
-<head>
-    <!-- Required meta tags -->
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-
-    <!-- Bootstrap CSS -->
-    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css"
-        integrity="sha384-JcKb8q3iqJ61gNV9KGb8thSsNjpSL0n8PARn9HuZOnIxN0hoP+VmmDGMN5t9UJ0Z" crossorigin="anonymous">
-
-    <title>Hello, world!</title>
-</head>
-
-<body>
-
-    <div class="container">
-        <div class="row d-flex justify-content-center">
-            <div class="col-6">
-                <form>
-                    <div class="form-group">
-                        <label for="exampleFormControlTextarea1" class="h4 pt-5">Chatroom</label>
-                        <textarea class="form-control" id="chat-text" rows="10"></textarea><br>
-                    </div>
-                    <div class="form-group">
-                        <input class="form-control" id="input" type="text"></br>
-                    </div>
-                    <input class="btn btn-secondary btn-lg btn-block" id="submit" type="button" value="Send">
-                </form>
-            </div>
-        </div>
-    </div>
-
-    {{ request.user.username|json_script:"user_username" }}
-    {{ order_id|json_script:"order-id" }}
-    
-    <script>
-        const user_username = JSON.parse(document.getElementById('user_username').textContent);
-        document.querySelector('#submit').onclick = function (e) {
-            const messageInputDom = document.querySelector('#input');
-            const message = messageInputDom.value;
-            chatSocket.send(JSON.stringify({
-                'message': message,
-                'username': user_username,
-            }));
-            messageInputDom.value = '';
-        };
-
-
-
-
-        const orderId = JSON.parse(document.getElementById('order-id').textContent);
-
-        const chatSocket = new WebSocket(
-            'ws://' +
-            window.location.host +
-            '/ws/chat/' +
-            orderId +
-            '/'
-        );
-
-        chatSocket.onmessage = function (e) {
-            const data = JSON.parse(e.data);
-            console.log(data)
-            document.querySelector('#chat-text').value += (data.username + ': ' + data.message + '\n')
-        }
-    </script>
-
-    <!-- Optional JavaScript -->
-    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
-    <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"
-        integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj" crossorigin="anonymous">
-    </script>
-    <script src="https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js"
-        integrity="sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN" crossorigin="anonymous">
-    </script>
-    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"
-        integrity="sha384-B4gt1jrGC7Jh4AgTPSdUtOBvfO8shuf57BaghqFfPlYxofvL8/KUEfYiJOMMV+rV" crossorigin="anonymous">
-    </script>
-</body>
-
-</html>

requirements.txt

@@ -0,0 +1,140 @@
+aioredis==1.3.1
+aiorpcX==0.18.7
+amqp==5.0.9
+apturl==0.5.2
+asgiref==3.4.1
+async-timeout==4.0.2
+attrs==21.4.0
+autobahn==21.11.1
+Automat==20.2.0
+backports.zoneinfo==0.2.1
+bcrypt==3.1.7
+billiard==3.6.4.0
+blinker==1.4
+Brlapi==0.7.0
+celery==5.2.3
+certifi==2019.11.28
+cffi==1.15.0
+channels==3.0.4
+channels-redis==3.3.1
+chardet==3.0.4
+charge-lnd==0.2.4
+click==8.0.3
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.2.0
+colorama==0.4.4
+command-not-found==0.3
+constantly==15.1.0
+cryptography==36.0.1
+cupshelpers==1.0
+daphne==3.0.2
+dbus-python==1.2.16
+defer==1.0.6
+Deprecated==1.2.13
+distlib==0.3.4
+distro==1.4.0
+distro-info===0.23ubuntu1
+Django==3.2.11
+django-admin-relation-links==0.2.5
+django-celery-beat==2.2.1
+django-celery-results==2.2.0
+django-model-utils==4.2.0
+django-private-chat2==1.0.2
+django-redis==5.2.0
+django-timezone-field==4.2.3
+djangorestframework==3.13.1
+duplicity==0.8.12.0
+entrypoints==0.3
+fasteners==0.14.1
+filelock==3.4.2
+future==0.18.2
+googleapis-common-protos==1.53.0
+grpcio==1.39.0
+grpcio-tools==1.43.0
+hiredis==2.0.0
+httplib2==0.14.0
+hyperlink==21.0.0
+idna==2.8
+incremental==21.3.0
+keyring==18.0.1
+kombu==5.2.3
+language-selector==0.1
+launchpadlib==1.10.13
+lazr.restfulclient==0.14.2
+lazr.uri==1.0.3
+lockfile==0.12.2
+louis==3.12.0
+macaroonbakery==1.3.1
+Mako==1.1.0
+MarkupSafe==1.1.0
+monotonic==1.5
+msgpack==1.0.3
+natsort==8.0.2
+netifaces==0.10.4
+numpy==1.22.0
+oauthlib==3.1.0
+olefile==0.46
+packaging==21.3
+paramiko==2.6.0
+pbr==5.8.0
+pexpect==4.6.0
+Pillow==7.0.0
+platformdirs==2.4.1
+prompt-toolkit==3.0.24
+protobuf==3.17.3
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycairo==1.16.2
+pycparser==2.21
+pycups==1.9.73
+PyGObject==3.36.0
+PyJWT==1.7.1
+pymacaroons==0.13.0
+PyNaCl==1.3.0
+pyOpenSSL==21.0.0
+pyparsing==3.0.6
+pyRFC3339==1.1
+PySocks==1.7.1
+python-apt==2.0.0+ubuntu0.20.4.5
+python-crontab==2.6.0
+python-dateutil==2.7.3
+python-debian===0.1.36ubuntu1
+python-decouple==3.5
+pytz==2021.3
+pyxdg==0.26
+PyYAML==5.3.1
+redis==4.1.0
+reportlab==3.5.34
+requests==2.22.0
+requests-unixsocket==0.2.0
+ring==0.9.1
+robohash==1.1
+scipy==1.7.3
+SecretStorage==2.3.1
+service-identity==21.1.0
+simplejson==3.16.0
+six==1.16.0
+sqlparse==0.4.2
+stevedore==3.5.0
+systemd-python==234
+termcolor==1.1.0
+Twisted==21.7.0
+txaio==21.2.1
+typing-extensions==4.0.1
+ubuntu-advantage-tools==27.2
+ubuntu-drivers-common==0.0.0
+ufw==0.36
+unattended-upgrades==0.1
+urllib3==1.25.8
+usb-creator==0.3.7
+vine==5.0.0
+virtualenv==20.12.1
+virtualenv-clone==0.5.7
+virtualenvwrapper==4.8.4
+wadllib==1.3.3
+wcwidth==0.2.5
+wirerope==0.4.5
+wrapt==1.13.3
+xkit==0.0.0
+zope.interface==5.4.0

robosats/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

robosats/celery/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import absolute_import, unicode_literals
+import os
+
+from celery import Celery
+from celery.schedules import crontab
+
+# You can use rabbitmq instead here.
+BASE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379')
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'robosats.settings')
+
+app = Celery('robosats')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+app.conf.broker_url = BASE_REDIS_URL
+
+# this allows schedule items in the Django admin.
+app.conf.beat_scheduler = 'django_celery_beat.schedulers:DatabaseScheduler'
+
+
+## Configure the periodic tasks
+app.conf.beat_schedule = {
+    'users-cleasing-every-hour': {
+        'task': 'users_cleansing',
+        'schedule': 60*60,
+    },
+}
+app.conf.timezone = 'UTC'

robosats/celery/conf.py

@@ -0,0 +1,2 @@
+# This sets the django-celery-results backend
+CELERY_RESULT_BACKEND = 'django-db'

robosats/settings.py

@@ -40,10 +40,13 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'rest_framework',
     'channels',
+    'django_celery_beat',
+    'django_celery_results',
     'api',
     'chat',
     'frontend.apps.FrontendConfig',
 ]
+from .celery.conf import *
 
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',

setup.md

@@ -45,8 +45,18 @@ pip install channels
 pip install django-redis
 pip install channels-redis
 ```
+## Install Celery for Django tasks
+```
+pip install celery
+pip install django-celery-beat
+pip install django-celery-results
+```
 
-*Django 4.0 at the time of writting*
+Start up celery worker
+`celery -A robosats worker --beat -l info -S django`
+
+*Django 3.2.11 at the time of writting*
+*Celery 5.2.3*
 
 ### Launch the local development node