Browse Source

Make celery task queue

devel
Dmitry Novikov 7 years ago
parent
commit
143c6b70ea
  1. 66
      abonapp/templates/abonapp/charts.html
  2. 5
      abonapp/templates/abonapp/ext.htm
  3. 3
      abonapp/urls.py
  4. 119
      abonapp/views.py
  5. 27
      chatbot/email_bot.py
  6. 5
      chatbot/send_func.py
  7. 28
      devapp/views.py
  8. 6
      djing/__init__.py
  9. 9
      djing/celery.py
  10. 10
      djing/settings.py
  11. 56
      djing/tasks.py
  12. 2
      djing/urls.py
  13. 7
      ip_pool/models.py
  14. 115
      msg_app/models.py
  15. 4
      requirements.txt
  16. 0
      statistics/__init__.py
  17. 0
      statistics/admin.py
  18. 5
      statistics/apps.py
  19. 53
      statistics/fields.py
  20. 29
      statistics/migrations/0001_initial.py
  21. 19
      statistics/migrations/0002_auto_20180808_1236.py
  22. 79
      statistics/migrations/0003_auto_20180814_1921.py
  23. 0
      statistics/migrations/__init__.py
  24. 132
      statistics/models.py
  25. 37
      statistics/templates/statistics/index.html
  26. 9
      statistics/urls.py
  27. 9
      statistics/views.py
  28. 9
      taskapp/handle.py

66
abonapp/templates/abonapp/charts.html

@ -1,66 +0,0 @@
{% extends request.is_ajax|yesno:'nullcont.htm,abonapp/ext.htm' %}
{% load i18n %}
{% block content %}
<div class="row">
<div class="col-sm-12">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{% blocktrans with wantdate_d=wantdate|date:'j E Y' %}Graph of use by {{ wantdate_d }}{% endblocktrans %}</h3>
</div>
<div class="panel-body">
{% if charts_data %}
<div id="chrt"></div>
<script type="text/javascript">
$(document).ready(function ($) {
new Chartist.Line('#chrt', {
series: [
{
data: [
{{ charts_data }}
]
}
]
}, {
height: '600px',
showArea: true,
showLine: false,
showPoint: false,
high: {{ high }},
axisX: {
type: Chartist.FixedScaleAxis,
divisor: 12,
labelInterpolationFnc: function (value) {
return moment(value).format('HH:mm:ss');
}
},
lineSmooth: Chartist.Interpolation.cardinal({
tension: 0
})
});
});
</script>
{% else %}
<h2>{% trans 'Static info was Not found' %}</h2>
{% endif %}
<form action="{% url 'abonapp:charts' group.pk abon.username %}" method="get" class="input-group">
<span class="input-group-btn">
<button class="btn btn-default" type="submit">
<span class="glyphicon glyphicon-calendar"></span> {% trans 'Show graph by date' %}
</button>
</span>
<input type="text" class="form-control" placeholder="{% trans 'Choose a date' %}" id="date_choose" name="wantdate" value="{{ wantdate|date:'dmY' }}">
</form>
<script type="text/javascript">
$(document).ready(function ($) {
$('#date_choose').datetimepicker({
format: 'DDMMYYYY'
});
});
</script>
</div>
</div>
</div>
</div>
{% endblock %}

5
abonapp/templates/abonapp/ext.htm

@ -39,11 +39,6 @@
<a href="{{ abtasklog }}">{% trans 'History of tasks' %}</a> <a href="{{ abtasklog }}">{% trans 'History of tasks' %}</a>
</li> </li>
{% url 'abonapp:charts' group.pk abon.username as abtasklog %}
<li{% if abtasklog == request.path %} class="active"{% endif %}>
<a href="{{ abtasklog }}">{% trans 'Charts' %}</a>
</li>
{% url 'abonapp:dials' group.pk abon.username as abdials %} {% url 'abonapp:dials' group.pk abon.username as abdials %}
<li{% if abdials == request.path %} class="active"{% endif %}> <li{% if abdials == request.path %} class="active"{% endif %}>
<a href="{{ abdials }}">{% trans 'Dialing' %}</a> <a href="{{ abdials }}">{% trans 'Dialing' %}</a>

3
abonapp/urls.py

@ -1,6 +1,6 @@
from django.urls import path, include, re_path from django.urls import path, include, re_path
from . import views
from abonapp import views
app_name = 'abonapp' app_name = 'abonapp'
@ -13,7 +13,6 @@ subscriber_patterns = [
path('addinvoice/', views.add_invoice, name='add_invoice'), path('addinvoice/', views.add_invoice, name='add_invoice'),
path('pick/', views.pick_tariff, name='pick_tariff'), path('pick/', views.pick_tariff, name='pick_tariff'),
path('passport_view/', views.PassportUpdateView.as_view(), name='passport_view'), path('passport_view/', views.PassportUpdateView.as_view(), name='passport_view'),
path('chart/', views.charts, name='charts'),
path('dials/', views.DialsListView.as_view(), name='dials'), path('dials/', views.DialsListView.as_view(), name='dials'),
# path('reset_ip/', views.reset_ip, name='reset_ip'), # path('reset_ip/', views.reset_ip, name='reset_ip'),
path('unsubscribe_service/<int:abon_tariff_id>/', views.unsubscribe_service, name='unsubscribe_service'), path('unsubscribe_service/<int:abon_tariff_id>/', views.unsubscribe_service, name='unsubscribe_service'),

119
abonapp/views.py

@ -1,4 +1,4 @@
from datetime import datetime, date
from datetime import datetime
from typing import Dict, Optional from typing import Dict, Optional
from agent.commands.dhcp import dhcp_commit, dhcp_expiry, dhcp_release from agent.commands.dhcp import dhcp_commit, dhcp_expiry, dhcp_release
@ -11,7 +11,7 @@ from django.contrib.auth.mixins import LoginRequiredMixin, \
PermissionRequiredMixin as PermissionRequiredMixin_django, PermissionRequiredMixin PermissionRequiredMixin as PermissionRequiredMixin_django, PermissionRequiredMixin
from django.core.exceptions import PermissionDenied, ValidationError from django.core.exceptions import PermissionDenied, ValidationError
from django.db import IntegrityError, ProgrammingError, transaction, \ from django.db import IntegrityError, ProgrammingError, transaction, \
OperationalError, DatabaseError
DatabaseError
from django.db.models import Count, Q from django.db.models import Count, Q
from django.http import HttpResponse, HttpResponseBadRequest, \ from django.http import HttpResponse, HttpResponseBadRequest, \
HttpResponseRedirect HttpResponseRedirect
@ -33,7 +33,6 @@ from guardian.shortcuts import get_objects_for_user, assign_perm
from gw_app.models import NASModel from gw_app.models import NASModel
from gw_app.nas_managers import NasFailedResult, NasNetworkError from gw_app.nas_managers import NasFailedResult, NasNetworkError
from ip_pool.models import NetworkModel from ip_pool.models import NetworkModel
from statistics.models import getModel
from tariff_app.models import Tariff from tariff_app.models import Tariff
from taskapp.models import Task from taskapp.models import Task
from xmlview.decorators import xml_view from xmlview.decorators import xml_view
@ -440,15 +439,17 @@ def pick_tariff(request, gid: int, uname):
trf = Tariff.objects.get(pk=request.POST.get('tariff')) trf = Tariff.objects.get(pk=request.POST.get('tariff'))
deadline = request.POST.get('deadline') deadline = request.POST.get('deadline')
log_comment = _( log_comment = _(
"Service '%(service_name)s' has connected via admin") % {
'service_name': trf.title
}
if deadline == '' or deadline is None:
abon.pick_tariff(trf, request.user, comment=log_comment)
else:
"Service '%(service_name)s' "
"has connected via admin until %(deadline)s") % {
'service_name': trf.title,
'deadline': deadline
}
if deadline:
deadline = datetime.strptime(deadline, '%Y-%m-%d %H:%M:%S') deadline = datetime.strptime(deadline, '%Y-%m-%d %H:%M:%S')
abon.pick_tariff(trf, request.user, deadline=deadline, abon.pick_tariff(trf, request.user, deadline=deadline,
comment=log_comment) comment=log_comment)
else:
abon.pick_tariff(trf, request.user, comment=log_comment)
r = abon.nas_sync_self() r = abon.nas_sync_self()
if r is None: if r is None:
messages.success(request, _('Tariff has been picked')) messages.success(request, _('Tariff has been picked'))
@ -689,56 +690,56 @@ def clear_dev(request, gid: int, uname):
return redirect('abonapp:abon_home', gid=gid, uname=uname) return redirect('abonapp:abon_home', gid=gid, uname=uname)
@login_required
@only_admins
@permission_required('group_app.view_group', (Group, 'pk', 'gid'))
def charts(request, gid: int, uname):
high = 100
wandate = request.GET.get('wantdate')
if wandate:
wandate = datetime.strptime(wandate, '%d%m%Y').date()
else:
wandate = date.today()
try:
StatElem = getModel(wandate)
abon = models.Abon.objects.get(username=uname)
if abon.group is None:
abon.group = Group.objects.get(pk=gid)
abon.save(update_fields=('group',))
charts_data = StatElem.objects.chart(
abon,
count_of_parts=30,
want_date=wandate
)
abontariff = abon.active_tariff()
if abontariff is not None:
trf = abontariff.tariff
high = trf.speedIn + trf.speedOut
if high > 100:
high = 100
except models.Abon.DoesNotExist:
messages.error(request, _('Abon does not exist'))
return redirect('abonapp:people_list', gid)
except Group.DoesNotExist:
messages.error(request, _("Group what you want doesn't exist"))
return redirect('abonapp:group_list')
except (ProgrammingError, OperationalError) as e:
messages.error(request, e)
return redirect('abonapp:charts', gid=gid, uname=uname)
return render(request, 'abonapp/charts.html', {
'group': abon.group,
'abon': abon,
'charts_data': ',\n'.join(
charts_data) if charts_data is not None else None,
'high': high,
'wantdate': wandate
})
# @login_required
# @only_admins
# @permission_required('group_app.view_group', (Group, 'pk', 'gid'))
# def charts(request, gid: int, uname):
# high = 100
#
# wandate = request.GET.get('wantdate')
# if wandate:
# wandate = datetime.strptime(wandate, '%d%m%Y').date()
# else:
# wandate = date.today()
#
# try:
# StatElem = getModel(wandate)
# abon = models.Abon.objects.get(username=uname)
# if abon.group is None:
# abon.group = Group.objects.get(pk=gid)
# abon.save(update_fields=('group',))
#
# charts_data = StatElem.objects.chart(
# abon,
# count_of_parts=30,
# want_date=wandate
# )
#
# abontariff = abon.active_tariff()
# if abontariff is not None:
# trf = abontariff.tariff
# high = trf.speedIn + trf.speedOut
# if high > 100:
# high = 100
#
# except models.Abon.DoesNotExist:
# messages.error(request, _('Abon does not exist'))
# return redirect('abonapp:people_list', gid)
# except Group.DoesNotExist:
# messages.error(request, _("Group what you want doesn't exist"))
# return redirect('abonapp:group_list')
# except (ProgrammingError, OperationalError) as e:
# messages.error(request, e)
# return redirect('abonapp:charts', gid=gid, uname=uname)
#
# return render(request, 'abonapp/charts.html', {
# 'group': abon.group,
# 'abon': abon,
# 'charts_data': ',\n'.join(
# charts_data) if charts_data is not None else None,
# 'high': high,
# 'wantdate': wandate
# })
@login_required @login_required

27
chatbot/email_bot.py

@ -1,27 +0,0 @@
from _socket import gaierror
from smtplib import SMTPException
from django.core.mail import EmailMultiAlternatives
from django.utils.html import strip_tags
from django.conf import settings
from chatbot.models import ChatException
def send_notify(msg_text, account, tag='none'):
try:
# MessageQueue.objects.push(msg=msg_text, user=account, tag=tag)
target_email = account.email
text_content = strip_tags(msg_text)
msg = EmailMultiAlternatives(
subject=getattr(settings, 'COMPANY_NAME', 'Djing notify'),
body=text_content,
from_email=getattr(settings, 'DEFAULT_FROM_EMAIL'),
to=(target_email,)
)
msg.attach_alternative(msg_text, 'text/html')
msg.send()
except SMTPException as e:
raise ChatException('SMTPException: %s' % e)
except gaierror as e:
raise ChatException('Socket error: %s' % e)

5
chatbot/send_func.py

@ -1,5 +0,0 @@
# send via email
from .email_bot import send_notify
# for Telegram
# from chatbot.telebot import send_notify

28
devapp/views.py

@ -4,7 +4,6 @@ from ipaddress import ip_address
from abonapp.models import Abon from abonapp.models import Abon
from accounts_app.models import UserProfile from accounts_app.models import UserProfile
from chatbot.models import ChatException from chatbot.models import ChatException
from chatbot.send_func import send_notify
from devapp.base_intr import DeviceImplementationError from devapp.base_intr import DeviceImplementationError
from django.conf import settings from django.conf import settings
from django.contrib import messages from django.contrib import messages
@ -25,6 +24,7 @@ from djing.lib.decorators import only_admins, hash_auth_view
from djing.lib.mixins import LoginAdminPermissionMixin, LoginAdminMixin from djing.lib.mixins import LoginAdminPermissionMixin, LoginAdminMixin
from djing.lib.tln import ZteOltConsoleError, OnuZteRegisterError, \ from djing.lib.tln import ZteOltConsoleError, OnuZteRegisterError, \
ZteOltLoginFailed ZteOltLoginFailed
from djing.tasks import multicast_email_notify
from easysnmp import EasySNMPTimeoutError, EasySNMPError from easysnmp import EasySNMPTimeoutError, EasySNMPError
from group_app.models import Group from group_app.models import Group
from guardian.decorators import \ from guardian.decorators import \
@ -700,24 +700,18 @@ class OnDeviceMonitoringEvent(global_base_views.SecureApiView):
recipients = UserProfile.objects.get_profiles_by_group( recipients = UserProfile.objects.get_profiles_by_group(
device_down.group.pk) device_down.group.pk)
names = list()
for recipient in recipients.iterator():
send_notify(
msg_text=gettext(notify_text) % {
'device_name': "%s(%s) %s" % (
device_down.ip_address,
device_down.mac_addr,
device_down.comment
)
},
account=recipient,
tag='devmon'
multicast_email_notify.delay(msg_text=gettext(notify_text) % {
'device_name': "%s(%s) %s" % (
device_down.ip_address,
device_down.mac_addr,
device_down.comment
) )
names.append(recipient.username)
}, account_ids=(
recipient.pk for recipient in recipients.only('pk').iterator()
))
return { return {
'text': 'notification successfully sent',
'recipients': names
'text': 'notification successfully sent'
} }
except ChatException as e: except ChatException as e:
return { return {

6
djing/__init__.py

@ -1,14 +1,14 @@
import importlib
import os import os
import re import re
import importlib
import typing as t import typing as t
from urllib.parse import unquote from urllib.parse import unquote
from django.http import HttpResponseRedirect, HttpResponse from django.http import HttpResponseRedirect, HttpResponse
from netaddr import mac_unix, mac_eui48
from django.shortcuts import _get_queryset from django.shortcuts import _get_queryset
from django.utils.http import is_safe_url from django.utils.http import is_safe_url
from netaddr import mac_unix, mac_eui48
from djing.celery import app
MAC_ADDR_REGEX = '^([0-9A-Fa-f]{1,2}[:-]){5}([0-9A-Fa-f]{1,2})$' MAC_ADDR_REGEX = '^([0-9A-Fa-f]{1,2}[:-]){5}([0-9A-Fa-f]{1,2})$'

9
djing/celery.py

@ -0,0 +1,9 @@
import os
from celery import Celery
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djing.settings")
app = Celery('djing', broker='redis://localhost:6379/0')
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()

10
djing/settings.py

@ -228,7 +228,9 @@ EMAIL_USE_TLS = getattr(local_settings, 'EMAIL_USE_TLS', True)
SERVER_EMAIL = getattr(local_settings, 'SERVER_EMAIL', EMAIL_HOST_USER) SERVER_EMAIL = getattr(local_settings, 'SERVER_EMAIL', EMAIL_HOST_USER)
# Inactive ip lease time in seconds.
# If lease time more than time of create, and lease is inactive
# then delete it. Used in ip_pool app.
LEASE_LIVE_TIME = 86400
# REDIS related settings
REDIS_HOST = 'localhost'
REDIS_PORT = '6379'
BROKER_URL = 'redis://' + REDIS_HOST + ':' + REDIS_PORT + '/0'
BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}
CELERY_RESULT_BACKEND = 'redis://' + REDIS_HOST + ':' + REDIS_PORT + '/0'

56
djing/tasks.py

@ -0,0 +1,56 @@
import logging
from _socket import gaierror
from smtplib import SMTPException
from typing import Iterable
from accounts_app.models import UserProfile
from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.utils.html import strip_tags
from celery import shared_task
@shared_task
def send_email_notify(msg_text: str, account_id: int):
try:
account = UserProfile.objects.get(pk=account_id)
target_email = account.email
text_content = strip_tags(msg_text)
msg = EmailMultiAlternatives(
subject=getattr(settings, 'COMPANY_NAME', 'Djing notify'),
body=text_content,
from_email=getattr(settings, 'DEFAULT_FROM_EMAIL'),
to=(target_email,)
)
msg.attach_alternative(msg_text, 'text/html')
msg.send()
except SMTPException as e:
logging.error('SMTPException: %s' % e)
except gaierror as e:
logging.error('Socket error: %s' % e)
except UserProfile.DoesNotExist:
logging.error('UserProfile with pk=%d not found' % account_id)
@shared_task
def multicast_email_notify(msg_text: str, account_ids: Iterable):
text_content = strip_tags(msg_text)
for acc_id in account_ids:
try:
account = UserProfile.objects.get(pk=acc_id)
target_email = account.email
msg = EmailMultiAlternatives(
subject=getattr(settings, 'COMPANY_NAME', 'Djing notify'),
body=text_content,
from_email=getattr(settings, 'DEFAULT_FROM_EMAIL'),
to=(target_email,)
)
msg.attach_alternative(msg_text, 'text/html')
msg.send()
except SMTPException as e:
logging.error('SMTPException: %s' % e)
except gaierror as e:
logging.error('Socket error: %s' % e)
except UserProfile.DoesNotExist:
logging.error('UserProfile with pk=%d not found' % acc_id)

2
djing/urls.py

@ -11,7 +11,7 @@ urlpatterns = [
path('search/', include('searchapp.urls', namespace='searchapp')), path('search/', include('searchapp.urls', namespace='searchapp')),
path('dev/', include('devapp.urls', namespace='devapp')), path('dev/', include('devapp.urls', namespace='devapp')),
path('map/', include('mapapp.urls', namespace='mapapp')), path('map/', include('mapapp.urls', namespace='mapapp')),
path('statistic/', include('statistics.urls', namespace='statistics')),
# path('statistic/', include('statistics.urls', namespace='statistics')),
path('tasks/', include('taskapp.urls', namespace='taskapp')), path('tasks/', include('taskapp.urls', namespace='taskapp')),
path('client/', include('clientsideapp.urls', namespace='client_side')), path('client/', include('clientsideapp.urls', namespace='client_side')),
path('msg/', include('msg_app.urls', namespace='msg_app')), path('msg/', include('msg_app.urls', namespace='msg_app')),

7
ip_pool/models.py

@ -176,13 +176,6 @@ class IpLeaseManager(models.Manager):
except IntegrityError as e: except IntegrityError as e:
raise DuplicateEntry(e) raise DuplicateEntry(e)
def expired(self):
lease_live_time = getattr(settings, 'LEASE_LIVE_TIME')
if lease_live_time is None:
raise ImproperlyConfigured('You must specify LEASE_LIVE_TIME in settings')
senility = now() - timedelta(seconds=lease_live_time)
return self.filter(lease_time__lt=senility)
# Deprecated. Remove after migrations squashed # Deprecated. Remove after migrations squashed
class IpLeaseModel(models.Model): class IpLeaseModel(models.Model):

115
msg_app/models.py

@ -1,7 +1,7 @@
from django.db import models from django.db import models
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from accounts_app.models import UserProfile from accounts_app.models import UserProfile
from chatbot.send_func import send_notify
from djing.tasks import send_email_notify
from chatbot.models import ChatException from chatbot.models import ChatException
@ -10,17 +10,29 @@ class MessageError(Exception):
class MessageStatus(models.Model): class MessageStatus(models.Model):
msg = models.ForeignKey('Message', on_delete=models.CASCADE, related_name='msg_statuses')
user = models.ForeignKey(UserProfile, on_delete=models.CASCADE, related_name='usr_msg_status')
msg = models.ForeignKey(
'Message', on_delete=models.CASCADE,
related_name='msg_statuses'
)
user = models.ForeignKey(
UserProfile, on_delete=models.CASCADE,
related_name='usr_msg_status'
)
MESSAGE_STATES = ( MESSAGE_STATES = (
('new', _('New')), ('new', _('New')),
('old', _('Seen')), ('old', _('Seen')),
('del', _('Deleted')) ('del', _('Deleted'))
) )
state = models.CharField(max_length=3, choices=MESSAGE_STATES, default='new')
state = models.CharField(
max_length=3, choices=MESSAGE_STATES,
default='new'
)
def __str__(self): def __str__(self):
return "%s for %s (%s)" % (self.get_state_display(), self.user, self.msg)
return "%s for %s (%s)" % (
self.get_state_display(),
self.user, self.msg
)
class Meta: class Meta:
db_table = 'message_status' db_table = 'message_status'
@ -34,10 +46,22 @@ class MessageStatus(models.Model):
class Message(models.Model): class Message(models.Model):
text = models.TextField(_("Body")) text = models.TextField(_("Body"))
sent_at = models.DateTimeField(_("sent at"), auto_now_add=True) sent_at = models.DateTimeField(_("sent at"), auto_now_add=True)
author = models.ForeignKey(UserProfile, on_delete=models.CASCADE, related_name='messages')
conversation = models.ForeignKey('Conversation', on_delete=models.CASCADE, verbose_name=_('Conversation'))
attachment = models.FileField(upload_to='messages_attachments/%Y_%m_%d', blank=True, null=True)
account_status = models.ManyToManyField(UserProfile, through=MessageStatus, through_fields=('msg', 'user'))
author = models.ForeignKey(
UserProfile, on_delete=models.CASCADE,
related_name='messages'
)
conversation = models.ForeignKey(
'Conversation', on_delete=models.CASCADE,
verbose_name=_('Conversation')
)
attachment = models.FileField(
upload_to='messages_attachments/%Y_%m_%d',
blank=True, null=True
)
account_status = models.ManyToManyField(
UserProfile, through=MessageStatus,
through_fields=('msg', 'user')
)
def __str__(self): def __str__(self):
return self.text[:9] return self.text[:9]
@ -70,7 +94,10 @@ class Message(models.Model):
class ConversationMembership(models.Model): class ConversationMembership(models.Model):
account = models.ForeignKey(UserProfile, on_delete=models.CASCADE, related_name='memberships')
account = models.ForeignKey(
UserProfile, on_delete=models.CASCADE,
related_name='memberships'
)
conversation = models.ForeignKey('Conversation', on_delete=models.CASCADE) conversation = models.ForeignKey('Conversation', on_delete=models.CASCADE)
PARTICIPANT_STATUS = ( PARTICIPANT_STATUS = (
('adm', _('Admin')), ('adm', _('Admin')),
@ -78,9 +105,14 @@ class ConversationMembership(models.Model):
('ban', _('Banned user')), ('ban', _('Banned user')),
('inv', _('Inviter')) ('inv', _('Inviter'))
) )
status = models.CharField(max_length=3, choices=PARTICIPANT_STATUS, default='gst')
who_invite_that_user = models.ForeignKey(UserProfile, on_delete=models.CASCADE, null=True, blank=True,
related_name='self_conversations')
status = models.CharField(
max_length=3, choices=PARTICIPANT_STATUS, default='gst'
)
who_invite_that_user = models.ForeignKey(
UserProfile, on_delete=models.CASCADE,
null=True, blank=True,
related_name='self_conversations'
)
def __str__(self): def __str__(self):
return "%s < %s" % (self.conversation, self.account) return "%s < %s" % (self.conversation, self.account)
@ -102,7 +134,9 @@ def id_to_userprofile(acc):
class ConversationManager(models.Manager): class ConversationManager(models.Manager):
def create_conversation(self, author, other_participants, title=None): def create_conversation(self, author, other_participants, title=None):
other_participants = tuple(id_to_userprofile(acc) for acc in other_participants)
other_participants = tuple(
id_to_userprofile(acc) for acc in other_participants
)
if not title: if not title:
usernames = tuple(acc.username for acc in other_participants) usernames = tuple(acc.username for acc in other_participants)
if not usernames: if not usernames:
@ -112,7 +146,8 @@ class ConversationManager(models.Manager):
conversation = self.create(title=title, author=author) conversation = self.create(title=title, author=author)
for acc in other_participants: for acc in other_participants:
ConversationMembership.objects.create( ConversationMembership.objects.create(
account=acc, conversation=conversation, status='adm', who_invite_that_user=author
account=acc, conversation=conversation,
status='adm', who_invite_that_user=author
) )
ConversationMembership.objects.create( ConversationMembership.objects.create(
@ -123,12 +158,16 @@ class ConversationManager(models.Manager):
@staticmethod @staticmethod
def get_new_messages_count(account): def get_new_messages_count(account):
if isinstance(account, UserProfile): if isinstance(account, UserProfile):
return MessageStatus.objects.filter(user=account, state='new').count()
return MessageStatus.objects.filter(
user=account, state='new'
).count()
else: else:
return 0 return 0
def fetch(self, account): def fetch(self, account):
conversations = self.filter(models.Q(author=account) | models.Q(participants__in=(account,))).annotate(
conversations = self.filter(
models.Q(author=account) | models.Q(participants__in=(account,))
).annotate(
msg_count=models.Count('message', distinct=True) msg_count=models.Count('message', distinct=True)
) )
return conversations return conversations
@ -136,9 +175,11 @@ class ConversationManager(models.Manager):
class Conversation(models.Model): class Conversation(models.Model):
title = models.CharField(max_length=32) title = models.CharField(max_length=32)
participants = models.ManyToManyField(UserProfile, related_name='conversations',
through='ConversationMembership',
through_fields=('conversation', 'account'))
participants = models.ManyToManyField(
UserProfile, related_name='conversations',
through='ConversationMembership',
through_fields=('conversation', 'account')
)
author = models.ForeignKey(UserProfile, on_delete=models.CASCADE) author = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
date_create = models.DateTimeField(auto_now_add=True) date_create = models.DateTimeField(auto_now_add=True)
@ -152,7 +193,9 @@ class Conversation(models.Model):
def get_messages_new_count(self, account): def get_messages_new_count(self, account):
msgs = Message.objects.filter(conversation=self) msgs = Message.objects.filter(conversation=self)
return MessageStatus.objects.filter(user=account, msg__in=msgs, state='new').count()
return MessageStatus.objects.filter(
user=account, msg__in=msgs, state='new'
).count()
def last_message(self): def last_message(self):
messages = Message.objects.filter(conversation=self) messages = Message.objects.filter(conversation=self)
@ -162,14 +205,18 @@ class Conversation(models.Model):
def new_message(self, text, attachment, author, with_status=True): def new_message(self, text, attachment, author, with_status=True):
try: try:
msg = Message.objects.create( msg = Message.objects.create(
text=text, conversation=self, attachment=attachment, author=author
text=text, conversation=self,
attachment=attachment, author=author
) )
if with_status: if with_status:
for participant in self.participants.all(): for participant in self.participants.all():
if participant == author: if participant == author:
continue continue
MessageStatus.objects.create(msg=msg, user=participant) MessageStatus.objects.create(msg=msg, user=participant)
send_notify(msg_text=text, account=participant, tag='msgapp')
send_email_notify.delay(
msg_text=text,
account_id=participant.pk
)
return msg return msg
except ChatException as e: except ChatException as e:
raise MessageError(e) raise MessageError(e)
@ -188,10 +235,13 @@ class Conversation(models.Model):
def _make_participant_status(self, user, status, cm=None): def _make_participant_status(self, user, status, cm=None):
if cm is None: if cm is None:
cm = ConversationMembership.objects.get(account=user, conversation=self)
cm = ConversationMembership.objects.get(
account=user, conversation=self
)
else: else:
if not isinstance(cm, ConversationMembership): if not isinstance(cm, ConversationMembership):
raise TypeError('cm must be instance of msg_app.ConversationMembership')
raise TypeError('cm must be instance of '
'msg_app.ConversationMembership')
cm.status = status cm.status = status
cm.save(update_fields=('status',)) cm.save(update_fields=('status',))
return cm return cm
@ -210,21 +260,28 @@ class Conversation(models.Model):
def remove_participant(self, user): def remove_participant(self, user):
try: try:
cm = ConversationMembership.objects.get(account=user, conversation=self)
cm = ConversationMembership.objects.get(
account=user, conversation=self
)
cm.delete() cm.delete()
except ConversationMembership.DoesNotExist: except ConversationMembership.DoesNotExist:
pass pass
def add_participant(self, author, user): def add_participant(self, author, user):
return ConversationMembership.objects.create( return ConversationMembership.objects.create(
account=user, conversation=self, status='gst', who_invite_that_user=author
account=user, conversation=self,
status='gst', who_invite_that_user=author
) )
def find_messages_by_text(self, text): def find_messages_by_text(self, text):
return Message.objects.filter(text__icontains=text, conversation=self)
return Message.objects.filter(
text__icontains=text, conversation=self
)
def _make_messages_status(self, account, status): def _make_messages_status(self, account, status):
qs = MessageStatus.objects.filter(msg__conversation=self, user=account).exclude(state='del')
qs = MessageStatus.objects.filter(
msg__conversation=self, user=account
).exclude(state='del')
if status != 'del': if status != 'del':
qs = qs.exclude(state=status) qs = qs.exclude(state=status)
return qs.update(state=status) return qs.update(state=status)

4
requirements.txt

@ -28,3 +28,7 @@ asterisk
# django-xmlview for pay system allpay # django-xmlview for pay system allpay
-e git://github.com/nerosketch/django-xmlview.git#egg=django-xmlview -e git://github.com/nerosketch/django-xmlview.git#egg=django-xmlview
Celery
redis==2.10.6
celery[redis]

0
statistics/__init__.py

0
statistics/admin.py

5
statistics/apps.py

@ -1,5 +0,0 @@
from django.apps import AppConfig
class StatisticsConfig(AppConfig):
name = 'statistics'

53
statistics/fields.py

@ -1,53 +0,0 @@
#
# Get from https://github.com/Niklas9/django-unixdatetimefield
#
import datetime
import time
import django.db.models as models
class UnixDateTimeField(models.DateTimeField):
# TODO(niklas9):
# * should we take care of transforming between time zones in any way here ?
# * get default datetime format from settings ?
DEFAULT_DATETIME_FMT = '%Y-%m-%d %H:%M:%S'
TZ_CONST = '+'
# TODO(niklas9):
# * metaclass below just for Django < 1.9, fix a if stmt for it?
# __metaclass__ = models.SubfieldBase
description = "Unix timestamp integer to datetime object"
def get_internal_type(self):
return 'PositiveIntegerField'
def to_python(self, val):
if val is None or isinstance(val, datetime.datetime):
return val
if isinstance(val, datetime.date):
return datetime.datetime(val.year, val.month, val.day)
elif self._is_string(val):
# TODO(niklas9):
# * not addressing time zone support as todo above for now
if self.TZ_CONST in val:
val = val.split(self.TZ_CONST)[0]
return datetime.datetime.strptime(val, self.DEFAULT_DATETIME_FMT)
else:
return datetime.datetime.fromtimestamp(float(val))
@staticmethod
def _is_string(val):
return isinstance(val, str)
def get_db_prep_value(self, val, *args, **kwargs):
if val is None:
if self.default == models.fields.NOT_PROVIDED: return None
return self.default
return int(time.mktime(val.timetuple()))
def value_to_string(self, obj):
val = self._get_val_from_obj(obj)
return self.to_python(val).strftime(self.DEFAULT_DATETIME_FMT)
def from_db_value(self, val, expression, connection):
return self.to_python(val)

29
statistics/migrations/0001_initial.py

@ -1,29 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-02-26 00:20
from __future__ import unicode_literals
from django.db import migrations, models
from djing.fields import MyGenericIPAddressField
import statistics.fields
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='StatCache',
fields=[
('last_time', statistics.fields.UnixDateTimeField()),
('ip', MyGenericIPAddressField(max_length=8, primary_key=True, protocol='ipv4', serialize=False)),
('octets', models.PositiveIntegerField(default=0)),
('packets', models.PositiveIntegerField(default=0)),
],
options={
'db_table': 'flowcache',
},
),
]

19
statistics/migrations/0002_auto_20180808_1236.py

@ -1,19 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-08-08 12:36
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('statistics', '0001_initial'),
]
operations = [
migrations.AlterModelOptions(
name='statcache',
options={'ordering': ('-last_time',)},
),
]

79
statistics/migrations/0003_auto_20180814_1921.py

@ -1,79 +0,0 @@
# Generated by Django 2.1 on 2018-09-22 14:30
from django.core.exceptions import ImproperlyConfigured
from django.db import migrations, connection, models
from statistics.fields import UnixDateTimeField
# def psql_migr(apps, _):
# pass
class Migration(migrations.Migration):
dependencies = [
('abonapp', '0005_current_tariff'),
('statistics', '0002_auto_20180808_1236'),
]
operations = [
migrations.AlterModelOptions(
name='statcache',
options={'ordering': ('-last_time',)},
),
]
db_e = connection.settings_dict.get('ENGINE')
if db_e is None:
raise ImproperlyConfigured('Database ENGINE is not set')
# if 'postgresql' in db_e:
# # Postgres
Migration.operations.insert(0, migrations.RunPython(psql_migr))
if 'mysql' in db_e:
Migration.operations.insert(0, migrations.RunSQL(
(
"DROP TABLE `flowcache`;",
"CREATE TABLE `flowcache` ( "
" `last_time` INT(10) UNSIGNED NOT NULL, "
" `abon_id` INT(11) DEFAULT NULL UNIQUE, "
" `octets` INT(10) UNSIGNED NOT NULL, "
" `packets` INT(10) UNSIGNED NOT NULL, "
" KEY `flowcache_abon_id_91e1085d` (`abon_id`) "
") ENGINE = MEMORY DEFAULT CHARSET = utf8;"
),
state_operations=[
migrations.DeleteModel(name='statcache'),
migrations.CreateModel(
name='statcache',
fields=[
('last_time', UnixDateTimeField()),
('abon', models.OneToOneField('abonapp.Abon', on_delete=models.CASCADE, primary_key=True)),
('octets', models.PositiveIntegerField(default=0)),
('packets', models.PositiveIntegerField(default=0))
],
options={
'db_table': 'flowcache',
},
)
]
))
else:
Migration.operations.extend(
(
migrations.DeleteModel(name='statcache'),
migrations.CreateModel(
name='statcache',
fields=[
('last_time', UnixDateTimeField()),
('abon', models.OneToOneField('abonapp.Abon', on_delete=models.CASCADE, primary_key=True)),
('octets', models.PositiveIntegerField(default=0)),
('packets', models.PositiveIntegerField(default=0))
],
options={
'db_table': 'flowcache',
'ordering': ('-last_time',),
#'db_tablespace': 'ram'
},
)
)
)

0
statistics/migrations/__init__.py

132
statistics/models.py

@ -1,132 +0,0 @@
import math
from datetime import datetime, timedelta, date, time
from django.db import models, connection, ProgrammingError
from django.utils.timezone import now
from djing.fields import MyGenericIPAddressField
from .fields import UnixDateTimeField
def get_dates():
tables = connection.introspection.table_names()
tables = (t.replace('flowstat_', '') for t in tables if t.startswith('flowstat_'))
return tuple(datetime.strptime(t, '%d%m%Y').date() for t in tables)
class StatManager(models.Manager):
def chart(self, user, count_of_parts=12, want_date=date.today()):
def byte_to_mbit(x):
return ((x / 60) * 8) / 2 ** 20
def split_list(lst, chunk_count):
chunk_size = len(lst) // chunk_count
if chunk_size == 0:
chunk_size = 1
return tuple(lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size))
def avarage(elements):
return sum(elements) / len(elements)
try:
charts_data = self.filter(abon=user)
charts_times = tuple(cd.cur_time.timestamp() * 1000 for cd in charts_data)
charts_octets = tuple(cd.octets for cd in charts_data)
if len(charts_octets) > 0 and len(charts_octets) == len(charts_times):
charts_octets = split_list(charts_octets, count_of_parts)
charts_octets = (byte_to_mbit(avarage(c)) for c in charts_octets)
charts_times = split_list(charts_times, count_of_parts)
charts_times = tuple(avarage(t) for t in charts_times)
charts_data = zip(charts_times, charts_octets)
charts_data = ["{x: new Date(%d), y: %.2f}" % (cd[0], cd[1]) for cd in charts_data]
midnight = datetime.combine(want_date, time.min)
charts_data.append("{x:new Date(%d),y:0}" % (int(charts_times[-1:][0]) + 1))
charts_data.append("{x:new Date(%d),y:0}" % (int((midnight + timedelta(days=1)).timestamp()) * 1000))
return charts_data
else:
return
except ProgrammingError as e:
if "Table 'djing_db_n.flowstat" in str(e):
return
class StatElem(models.Model):
cur_time = UnixDateTimeField(primary_key=True)
abon = models.ForeignKey('abonapp.Abon', on_delete=models.CASCADE, null=True, default=None, blank=True)
ip = MyGenericIPAddressField()
octets = models.PositiveIntegerField(default=0)
packets = models.PositiveIntegerField(default=0)
objects = StatManager()
# ReadOnly
def save(self, *args, **kwargs):
pass
# ReadOnly
def delete(self, *args, **kwargs):
pass
@property
def table_name(self):
return self._meta.db_table
def delete_month(self):
cursor = connection.cursor()
table_name = self._meta.db_table
sql = "DROP TABLE %s;" % table_name
cursor.execute(sql)
@staticmethod
def percentile(N, percent, key=lambda x: x):
"""
Find the percentile of a list of values.
@parameter N - is a list of values. Note N MUST BE already sorted.
@parameter percent - a float value from 0.0 to 1.0.
@parameter key - optional key function to compute value from each element of N.
@return - the percentile of the values
"""
if not N:
return None
k = (len(N) - 1) * percent
f = math.floor(k)
c = math.ceil(k)
if f == c:
return key(N[int(k)])
d0 = key(N[int(f)]) * (c - k)
d1 = key(N[int(c)]) * (k - f)
return d0 + d1
class Meta:
abstract = True
def getModel(want_date=now()):
class DynamicStatElem(StatElem):
class Meta:
db_table = 'flowstat_%s' % want_date.strftime("%d%m%Y")
abstract = False
return DynamicStatElem
class StatCache(models.Model):
last_time = UnixDateTimeField()
# ip = MyGenericIPAddressField(primary_key=True)
abon = models.OneToOneField('abonapp.Abon', on_delete=models.CASCADE, primary_key=True)
octets = models.PositiveIntegerField(default=0)
packets = models.PositiveIntegerField(default=0)
def is_online(self):
return self.last_time > now() - timedelta(minutes=55)
def is_today(self):
return date.today() == self.last_time.date()
class Meta:
db_table = 'flowcache'
ordering = ('-last_time',)
# db_tablespace = 'ram'

37
statistics/templates/statistics/index.html

@ -1,37 +0,0 @@
{% extends 'base.html' %}
{% block main %}
<link href="/static/css/chartist.min.css" rel="stylesheet" type="text/css"/>
<script src="/static/js/chartist.min.js"></script>
<script>
var chart = new Chartist.Line('#maincontent', {
labels: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'],
series: [
[1, 5, 2, 5, 4, 3],
[2, 3, 4, 8, 1, 2],
[5, 4, 3, 2, 1, 0.5]
]
}, {
low: 0,
showArea: true,
showPoint: false,
fullWidth: true,
height: 500
});
chart.on('draw', function (data) {
if (data.type === 'line' || data.type === 'area') {
data.element.animate({
d: {
begin: 2000 * data.index,
dur: 2000,
from: data.path.clone().scale(1, 0).translate(0, data.chartRect.height()).stringify(),
to: data.path.clone().stringify(),
easing: Chartist.Svg.Easing.easeOutQuint
}
});
}
});
</script>
{% endblock %}

9
statistics/urls.py

@ -1,9 +0,0 @@
from django.urls import path
from . import views
app_name = 'statistics'
urlpatterns = [
path('', views.home, name='home'),
]

9
statistics/views.py

@ -1,9 +0,0 @@
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from djing.lib.decorators import only_admins
@login_required
@only_admins
def home(request):
return render(request, 'statistics/index.html')

9
taskapp/handle.py

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from django.template.loader import render_to_string from django.template.loader import render_to_string
from django.utils.translation import gettext as _ from django.utils.translation import gettext as _
from chatbot.send_func import send_notify
from djing.tasks import send_email_notify
from chatbot.models import ChatException from chatbot.models import ChatException
from djing.lib import MultipleException from djing.lib import MultipleException
@ -30,12 +30,9 @@ def handle(task, author, recipients):
if task.state == 'F' or task.state == 'C': if task.state == 'F' or task.state == 'C':
# If task completed or failed than send one message to author # If task completed or failed than send one message to author
try:
send_notify(fulltext, author, tag='taskap')
except ChatException as e:
raise TaskException(e)
send_email_notify.delay(fulltext, author.pk)
else: else:
send_notify(fulltext, recipient, tag='taskap')
send_email_notify.delay(fulltext, recipient.pk)
except ChatException as e: except ChatException as e:
errors.append(e) errors.append(e)
if len(errors) > 0: if len(errors) > 0:

Loading…
Cancel
Save