diff --git a/.gitignore b/.gitignore index 07c2094f..f3a0f6bc 100755 --- a/.gitignore +++ b/.gitignore @@ -639,6 +639,9 @@ FodyWeavers.xsd *migrations* frontend/static/frontend/main* +# Celery +django + # robosats frontend/static/assets/avatars* api/lightning/lightning* diff --git a/api/admin.py b/api/admin.py index 9d8ab64f..6cba65a7 100644 --- a/api/admin.py +++ b/api/admin.py @@ -38,7 +38,7 @@ class LNPaymentAdmin(AdminChangeLinksMixin, admin.ModelAdmin): @admin.register(Profile) class UserProfileAdmin(AdminChangeLinksMixin, admin.ModelAdmin): - list_display = ('avatar_tag','id','user_link','total_ratings','avg_rating','num_disputes','lost_disputes') + list_display = ('avatar_tag','id','user_link','total_contracts','total_ratings','avg_rating','num_disputes','lost_disputes') list_display_links = ('avatar_tag','id') change_links =['user'] readonly_fields = ['avatar_tag'] diff --git a/api/logics.py b/api/logics.py index cee59d8a..1a4994e9 100644 --- a/api/logics.py +++ b/api/logics.py @@ -339,6 +339,12 @@ class Logics(): order.taker_bond.status = LNPayment.Status.LOCKED order.taker_bond.save() + # Both users profile have one more contract done + order.maker.profile.total_contracts = order.maker.profile.total_contracts + 1 + order.taker.profile.total_contracts = order.taker.profile.total_contracts + 1 + order.maker.profile.save() + order.taker.profile.save() + # Log a market tick MarketTick.log_a_tick(order) diff --git a/api/models.py b/api/models.py index 9ad4be95..1505b303 100644 --- a/api/models.py +++ b/api/models.py @@ -159,12 +159,12 @@ class Order(models.Model): return (f'Order {self.id}: {self.Types(self.type).label} BTC for {float(self.amount)} {self.currency_dict[str(self.currency)]}') @receiver(pre_delete, sender=Order) -def delete_HTLCs_at_order_deletion(sender, instance, **kwargs): +def delete_lnpayment_at_order_deletion(sender, instance, **kwargs): to_delete = (instance.maker_bond, instance.buyer_invoice, instance.taker_bond, instance.trade_escrow) - for htlc in to_delete: + for lnpayment in to_delete: try: - htlc.delete() + lnpayment.delete() except: pass @@ -172,6 +172,9 @@ class Profile(models.Model): user = models.OneToOneField(User,on_delete=models.CASCADE) + # Total trades + total_contracts = models.PositiveIntegerField(null=False, default=0) + # Ratings stored as a comma separated integer list total_ratings = models.PositiveIntegerField(null=False, default=0) latest_ratings = models.CharField(max_length=999, null=True, default=None, validators=[validate_comma_separated_integer_list], blank=True) # Will only store latest ratings @@ -198,8 +201,11 @@ class Profile(models.Model): @receiver(pre_delete, sender=User) def del_avatar_from_disk(sender, instance, **kwargs): - avatar_file=Path('frontend/' + instance.profile.avatar.url) - avatar_file.unlink() # FIX deleting user fails if avatar is not found + try: + avatar_file=Path('frontend/' + instance.profile.avatar.url) + avatar_file.unlink() + except: + pass def __str__(self): return self.user.username diff --git a/api/nick_generator/dicts/en/nouns.py b/api/nick_generator/dicts/en/nouns.py index e48a739d..ae3243e8 100755 --- a/api/nick_generator/dicts/en/nouns.py +++ b/api/nick_generator/dicts/en/nouns.py @@ -3633,7 +3633,7 @@ nouns = [ "Fever", "Few", "Fiance", - "Fiancé", + "Fiance", "Fiasco", "Fiat", "Fiber", diff --git a/api/tasks.py b/api/tasks.py new file mode 100644 index 00000000..8876b84d --- /dev/null +++ b/api/tasks.py @@ -0,0 +1,57 @@ +from celery import shared_task + +from .lightning.node import LNNode +from django.contrib.auth.models import User +from .models import LNPayment, Order +from .logics import Logics +from django.db.models import Q + +from datetime import timedelta +from django.utils import timezone + +from decouple import config + +@shared_task(name="users_cleansing") +def users_cleansing(): + ''' + Deletes users never used 12 hours after creation + ''' + # Users who's last login has not been in the last 12 hours + active_time_range = (timezone.now() - timedelta(hours=12), timezone.now()) + queryset = User.objects.filter(~Q(last_login__range=active_time_range)) + + # And do not have an active trade or any pass finished trade. + deleted_users = [] + for user in queryset: + if user.username == str(config('ESCROW_USERNAME')): # Do not delete admin user by mistake + continue + if not user.profile.total_contracts == 0: + continue + valid, _ = Logics.validate_already_maker_or_taker(user) + if valid: + deleted_users.append(str(user)) + user.delete() + + results = { + 'num_deleted': len(deleted_users), + 'deleted_users': deleted_users, + } + + return results + + +@shared_task +def orders_expire(): + pass + +@shared_task +def follow_lnd_payment(): + pass + +@shared_task +def query_all_lnd_invoices(): + pass + +@shared_task +def cache_market(): + pass \ No newline at end of file diff --git a/api/views.py b/api/views.py index e779498f..d6a2ee0d 100644 --- a/api/views.py +++ b/api/views.py @@ -422,7 +422,7 @@ class InfoView(ListAPIView): context['num_public_sell_orders'] = len(Order.objects.filter(type=Order.Types.SELL, status=Order.Status.PUB)) # Number of active users (logged in in last 30 minutes) - active_user_time_range = (timezone.now() - timedelta(minutes=30), timezone.now()) + active_user_time_range = (timezone.now() - timedelta(minutes=120), timezone.now()) context['num_active_robotsats'] = len(User.objects.filter(last_login__range=active_user_time_range)) # Compute average premium and volume of today diff --git a/chat/templates/chatroom.html b/chat/templates/chatroom.html deleted file mode 100644 index f638de64..00000000 --- a/chat/templates/chatroom.html +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - - - - Hello, world! - - - - -
-
-
-
-
- -
-
-
-
-
- -
-
-
-
- - {{ request.user.username|json_script:"user_username" }} - {{ order_id|json_script:"order-id" }} - - - - - - - - - - - \ No newline at end of file diff --git a/frontend/static/assets/misc/unknown_avatar.png b/frontend/static/assets/misc/unknown_avatar.png deleted file mode 100644 index 9e19b6a9..00000000 Binary files a/frontend/static/assets/misc/unknown_avatar.png and /dev/null differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..dbb03b5b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,140 @@ +aioredis==1.3.1 +aiorpcX==0.18.7 +amqp==5.0.9 +apturl==0.5.2 +asgiref==3.4.1 +async-timeout==4.0.2 +attrs==21.4.0 +autobahn==21.11.1 +Automat==20.2.0 +backports.zoneinfo==0.2.1 +bcrypt==3.1.7 +billiard==3.6.4.0 +blinker==1.4 +Brlapi==0.7.0 +celery==5.2.3 +certifi==2019.11.28 +cffi==1.15.0 +channels==3.0.4 +channels-redis==3.3.1 +chardet==3.0.4 +charge-lnd==0.2.4 +click==8.0.3 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.2.0 +colorama==0.4.4 +command-not-found==0.3 +constantly==15.1.0 +cryptography==36.0.1 +cupshelpers==1.0 +daphne==3.0.2 +dbus-python==1.2.16 +defer==1.0.6 +Deprecated==1.2.13 +distlib==0.3.4 +distro==1.4.0 +distro-info===0.23ubuntu1 +Django==3.2.11 +django-admin-relation-links==0.2.5 +django-celery-beat==2.2.1 +django-celery-results==2.2.0 +django-model-utils==4.2.0 +django-private-chat2==1.0.2 +django-redis==5.2.0 +django-timezone-field==4.2.3 +djangorestframework==3.13.1 +duplicity==0.8.12.0 +entrypoints==0.3 +fasteners==0.14.1 +filelock==3.4.2 +future==0.18.2 +googleapis-common-protos==1.53.0 +grpcio==1.39.0 +grpcio-tools==1.43.0 +hiredis==2.0.0 +httplib2==0.14.0 +hyperlink==21.0.0 +idna==2.8 +incremental==21.3.0 +keyring==18.0.1 +kombu==5.2.3 +language-selector==0.1 +launchpadlib==1.10.13 +lazr.restfulclient==0.14.2 +lazr.uri==1.0.3 +lockfile==0.12.2 +louis==3.12.0 +macaroonbakery==1.3.1 +Mako==1.1.0 +MarkupSafe==1.1.0 +monotonic==1.5 +msgpack==1.0.3 +natsort==8.0.2 +netifaces==0.10.4 +numpy==1.22.0 +oauthlib==3.1.0 +olefile==0.46 +packaging==21.3 +paramiko==2.6.0 +pbr==5.8.0 +pexpect==4.6.0 +Pillow==7.0.0 +platformdirs==2.4.1 +prompt-toolkit==3.0.24 +protobuf==3.17.3 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycairo==1.16.2 +pycparser==2.21 +pycups==1.9.73 +PyGObject==3.36.0 +PyJWT==1.7.1 +pymacaroons==0.13.0 +PyNaCl==1.3.0 +pyOpenSSL==21.0.0 +pyparsing==3.0.6 +pyRFC3339==1.1 +PySocks==1.7.1 +python-apt==2.0.0+ubuntu0.20.4.5 +python-crontab==2.6.0 +python-dateutil==2.7.3 +python-debian===0.1.36ubuntu1 +python-decouple==3.5 +pytz==2021.3 +pyxdg==0.26 +PyYAML==5.3.1 +redis==4.1.0 +reportlab==3.5.34 +requests==2.22.0 +requests-unixsocket==0.2.0 +ring==0.9.1 +robohash==1.1 +scipy==1.7.3 +SecretStorage==2.3.1 +service-identity==21.1.0 +simplejson==3.16.0 +six==1.16.0 +sqlparse==0.4.2 +stevedore==3.5.0 +systemd-python==234 +termcolor==1.1.0 +Twisted==21.7.0 +txaio==21.2.1 +typing-extensions==4.0.1 +ubuntu-advantage-tools==27.2 +ubuntu-drivers-common==0.0.0 +ufw==0.36 +unattended-upgrades==0.1 +urllib3==1.25.8 +usb-creator==0.3.7 +vine==5.0.0 +virtualenv==20.12.1 +virtualenv-clone==0.5.7 +virtualenvwrapper==4.8.4 +wadllib==1.3.3 +wcwidth==0.2.5 +wirerope==0.4.5 +wrapt==1.13.3 +xkit==0.0.0 +zope.interface==5.4.0 diff --git a/robosats/__init__.py b/robosats/__init__.py index e69de29b..d128d39c 100644 --- a/robosats/__init__.py +++ b/robosats/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) \ No newline at end of file diff --git a/robosats/celery/__init__.py b/robosats/celery/__init__.py new file mode 100644 index 00000000..797aa2c9 --- /dev/null +++ b/robosats/celery/__init__.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import, unicode_literals +import os + +from celery import Celery +from celery.schedules import crontab + +# You can use rabbitmq instead here. +BASE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379') + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'robosats.settings') + +app = Celery('robosats') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +app.conf.broker_url = BASE_REDIS_URL + +# this allows schedule items in the Django admin. +app.conf.beat_scheduler = 'django_celery_beat.schedulers:DatabaseScheduler' + + +## Configure the periodic tasks +app.conf.beat_schedule = { + 'users-cleasing-every-hour': { + 'task': 'users_cleansing', + 'schedule': 60*60, + }, +} +app.conf.timezone = 'UTC' \ No newline at end of file diff --git a/robosats/celery/conf.py b/robosats/celery/conf.py new file mode 100644 index 00000000..6b1aa603 --- /dev/null +++ b/robosats/celery/conf.py @@ -0,0 +1,2 @@ +# This sets the django-celery-results backend +CELERY_RESULT_BACKEND = 'django-db' \ No newline at end of file diff --git a/robosats/settings.py b/robosats/settings.py index d41f249b..2bf7a7d3 100644 --- a/robosats/settings.py +++ b/robosats/settings.py @@ -40,10 +40,13 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'rest_framework', 'channels', + 'django_celery_beat', + 'django_celery_results', 'api', 'chat', 'frontend.apps.FrontendConfig', ] +from .celery.conf import * MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', diff --git a/setup.md b/setup.md index 032a374f..ccc946aa 100644 --- a/setup.md +++ b/setup.md @@ -45,8 +45,18 @@ pip install channels pip install django-redis pip install channels-redis ``` +## Install Celery for Django tasks +``` +pip install celery +pip install django-celery-beat +pip install django-celery-results +``` -*Django 4.0 at the time of writting* +Start up celery worker +`celery -A robosats worker --beat -l info -S django` + +*Django 3.2.11 at the time of writting* +*Celery 5.2.3* ### Launch the local development node