recovery traffic statistics

abonapp/templates/abonapp/peoples.html

@@ -24,14 +24,14 @@
                 <table class="table table-striped table-bordered">
                     <thead>
                     <tr>
-{#						<th>#</th> #}
+						<th>#</th>
                         <th class="col-xs-1">
                             <a href="{% url 'abonapp:people_list' group.pk %}?{% url_order_by request order_by='username' %}">
                                 {% trans 'Sub' %}
                             </a>
                             {% if order_by == 'username' %}<span class="glyphicon glyphicon-filter"></span>{% endif %}
                         </th>
-{#                        <th class="hidden-xs">{% trans 'Last traffic' %}</th> #}
+                        <th class="hidden-xs">{% trans 'Last traffic' %}</th>
                         <th>{% trans 'Network' %}</th>
                         <th class="col-xs-3">
                             <a href="{% url 'abonapp:people_list' group.pk %}?{% url_order_by request order_by='fio' %}">
@@ -75,26 +75,25 @@
                         {% else %}
                             <tr class="danger">
                         {% endif %}
-{#                        <td> #}
-{#                            <span class="glyphicon glyphicon-question-sign text-muted"></span> #}
-{#                            {% if human.statcache.is_online %}#}
-{#                                <span class="glyphicon glyphicon-ok text-success"></span>#}
-{#                            {% else %}#}
-{#                                <span class="glyphicon glyphicon-remove-sign text-muted"></span>#}
-{#                            {% endif %}#}
-{#                        </td> #}
+                        <td>
+                            {% if human.statcache.is_online %}
+                                <span class="glyphicon glyphicon-ok text-success"></span>
+                            {% else %}
+                                <span class="glyphicon glyphicon-remove-sign text-muted"></span>
+                            {% endif %}
+                        </td>
                         <td class="col-xs-1">
                             <a href="{% url 'abonapp:abon_home' human.group_id human.username %}" title="{% trans 'Date joined' %}: {{ human.birth_day|date:'d E y' }}" data-toggle="tooltip">{{ human.username }}</a>
                         </td>
-{#                        <td class="hidden-xs"> #}
-{#                            {% if human.statcache %}#}
-{#                                {% if human.statcache.is_today %}#}
-{#                                    {{ human.statcache.last_time|date:"H:i" }}#}
-{#                                {% else %}#}
-{#                                    {{ human.statcache.last_time|date:"D H:i" }}#}
-{#                                {% endif %}#}
-{#                            {% endif %}#}
-{#                        </td> #}
+                        <td class="hidden-xs">
+                            {% if human.statcache %}
+                                {% if human.statcache.is_today %}
+                                    {{ human.statcache.last_time|date:"H:i" }}
+                                {% else %}
+                                    {{ human.statcache.last_time|date:"D H:i" }}
+                                {% endif %}
+                            {% endif %}
+                        </td>
                         <td class="col-xs-1">{{ human.ip_address|default_if_none:'&mdash;' }}</td>
                         <td class="col-xs-1">{{ human.fio|default:'&mdash;' }}</td>
                         <td class="col-xs-1">{{ human.street|default:_('Not assigned') }}</td>

abonapp/views.py

@@ -52,11 +52,11 @@ class PeoplesListView(LoginRequiredMixin, OnlyAdminsMixin,
         if street_id > 0:
             peoples_list = peoples_list.filter(street=street_id)
         peoples_list = peoples_list.select_related(
-            'group', 'street', 'current_tariff'
+            'group', 'street', 'current_tariff__tariff', 'statcache'
         ).only(
-            'group', 'street', 'fio',
+            'group', 'street', 'fio', 'birth_day',
             'street', 'house', 'telephone', 'ballance', 'markers',
-            'username', 'is_active', 'current_tariff'
+            'username', 'is_active', 'current_tariff', 'ip_address'
         )
         ordering = self.get_ordering()
         if ordering and isinstance(ordering, str):

djing/settings.py

@@ -50,7 +50,7 @@ INSTALLED_APPS = [
     'searchapp',
     'devapp',
     'mapapp',
-    'statistics',
+    'traf_stat',
     'taskapp',
     'clientsideapp',
     'messenger',

djing/urls.py

@@ -11,7 +11,7 @@ urlpatterns = [
     path('search/', include('searchapp.urls', namespace='searchapp')),
     path('dev/', include('devapp.urls', namespace='devapp')),
     path('map/', include('mapapp.urls', namespace='mapapp')),
-    # path('statistic/', include('statistics.urls', namespace='statistics')),
+    path('statistic/', include('traf_stat.urls', namespace='traf_stat')),
     path('tasks/', include('taskapp.urls', namespace='taskapp')),
     path('client/', include('clientsideapp.urls', namespace='client_side')),
     path('msg/', include('msg_app.urls', namespace='msg_app')),

locale/ru/LC_MESSAGES/django.po

@@ -125,3 +125,6 @@ msgstr "На сервере произошла ошибка. Пожалуйст
 
 msgid "Are you sure about them?"
 msgstr "Вы уверены в этом?"
+
+msgid "Traffic"
+msgstr "Траффик"

templates/base.html

@@ -97,6 +97,15 @@
                 </li>
             {% endif %}
 
+            {% if perms.traf_stat.statcache_view %}
+                {% url 'traf_stat:home' as stathome %}
+                <li{% if stathome in request.path %} class="active"{% endif %}>
+                    <a href="{{ stathome }}">
+                        <span class="glyphicon glyphicon-dashboard"></span> {% trans 'Traffic' %}
+                    </a>
+                </li>
+            {% endif %}
+
             {% url 'devapp:group_list' as devapp_groups %}
             <li{% if devapp_groups in request.path %} class="active"{% endif %}>
                 <a href="{{ devapp_groups }}">

traf_stat/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

traf_stat/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class TrafStatConfig(AppConfig):
+    name = 'traf_stat'

traf_stat/fields.py

@@ -0,0 +1,50 @@
+#
+# Get from https://github.com/Niklas9/django-unixdatetimefield
+#
+import datetime
+import time
+
+import django.db.models as models
+
+
+class UnixDateTimeField(models.DateTimeField):
+
+    # TODO(niklas9):
+    # * should we take care of transforming between time zones in any way here ?
+    # * get default datetime format from settings ?
+    DEFAULT_DATETIME_FMT = '%Y-%m-%d %H:%M:%S'
+    TZ_CONST = '+'
+    # TODO(niklas9):
+    # * metaclass below just for Django < 1.9, fix a if stmt for it?
+    #__metaclass__ = models.SubfieldBase
+    description = "Unix timestamp integer to datetime object"
+
+    def get_internal_type(self):
+        return 'PositiveIntegerField'
+
+    def to_python(self, val):
+        if val is None or isinstance(val, datetime.datetime):
+            return val
+        if isinstance(val, datetime.date):
+            return datetime.datetime(val.year, val.month, val.day)
+        elif isinstance(val, str):
+            # TODO(niklas9):
+            # * not addressing time zone support as todo above for now
+            if self.TZ_CONST in val:
+                val = val.split(self.TZ_CONST)[0]
+            return datetime.datetime.strptime(val, self.DEFAULT_DATETIME_FMT)
+        else:
+            return datetime.datetime.fromtimestamp(float(val))
+
+    def get_db_prep_value(self, val, *args, **kwargs):
+        if val is None:
+            if self.default == models.fields.NOT_PROVIDED:  return None
+            return self.default
+        return int(time.mktime(val.timetuple()))
+
+    def value_to_string(self, obj):
+        val = self._get_val_from_obj(obj)
+        return self.to_python(val).strftime(self.DEFAULT_DATETIME_FMT)
+
+    def from_db_value(self, val, *args, **kwargs):
+        return self.to_python(val)

traf_stat/migrations/0001_initial.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.1.1 on 2019-03-06 18:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+import traf_stat.fields
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('abonapp', '0001_squashed_0008_auto_20181115_1206'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StatCache',
+            fields=[
+                ('last_time', traf_stat.fields.UnixDateTimeField()),
+                ('abon', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='abonapp.Abon')),
+                ('octets', models.PositiveIntegerField(default=0)),
+                ('packets', models.PositiveIntegerField(default=0)),
+            ],
+            options={
+                'db_table': 'flowcache',
+                'ordering': ('-last_time',),
+            },
+        ),
+        migrations.RunSQL(sql=(r'ALTER TABLE flowcache ENGINE=MEMORY;',))
+    ]

traf_stat/models.py

@@ -0,0 +1,129 @@
+from datetime import datetime, timedelta, date, time
+import math
+
+from django.db import models, connection, ProgrammingError
+from django.utils.timezone import now
+from .fields import UnixDateTimeField
+
+
+def get_dates():
+    tables = connection.introspection.table_names()
+    tables = (t.replace('flowstat_', '') for t in tables if t.startswith('flowstat_'))
+    return tuple(datetime.strptime(t, '%d%m%Y').date() for t in tables)
+
+
+class StatManager(models.Manager):
+    def chart(self, user, count_of_parts=12, want_date=date.today()):
+        def byte_to_mbit(x):
+            return ((x / 60) * 8) / 2 ** 20
+
+        def split_list(lst, chunk_count):
+            chunk_size = len(lst) // chunk_count
+            if chunk_size == 0:
+                chunk_size = 1
+            return tuple(lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size))
+
+        def avarage(elements):
+            return sum(elements) / len(elements)
+
+        try:
+            charts_data = self.filter(abon=user)
+            charts_times = tuple(cd.cur_time.timestamp() * 1000 for cd in charts_data)
+            charts_octets = tuple(cd.octets for cd in charts_data)
+            if len(charts_octets) > 0 and len(charts_octets) == len(charts_times):
+                charts_octets = split_list(charts_octets, count_of_parts)
+                charts_octets = (byte_to_mbit(avarage(c)) for c in charts_octets)
+
+                charts_times = split_list(charts_times, count_of_parts)
+                charts_times = tuple(avarage(t) for t in charts_times)
+
+                charts_data = zip(charts_times, charts_octets)
+                charts_data = ["{x: new Date(%d), y: %.2f}" % (cd[0], cd[1]) for cd in charts_data]
+                midnight = datetime.combine(want_date, time.min)
+                charts_data.append("{x:new Date(%d),y:0}" % (int(charts_times[-1:][0]) + 1))
+                charts_data.append("{x:new Date(%d),y:0}" % (int((midnight + timedelta(days=1)).timestamp()) * 1000))
+                return charts_data
+            else:
+                return
+        except ProgrammingError as e:
+            if "flowstat" in str(e):
+                return
+
+
+class StatElem(models.Model):
+    cur_time = UnixDateTimeField(primary_key=True)
+    abon = models.ForeignKey('abonapp.Abon', on_delete=models.CASCADE, null=True, default=None, blank=True)
+    ip = models.PositiveIntegerField()
+    octets = models.PositiveIntegerField(default=0)
+    packets = models.PositiveIntegerField(default=0)
+
+    objects = StatManager()
+
+    # ReadOnly
+    def save(self, *args, **kwargs):
+        pass
+
+    # ReadOnly
+    def delete(self, *args, **kwargs):
+        pass
+
+    @property
+    def table_name(self):
+        return self._meta.db_table
+
+    def delete_month(self):
+        cursor = connection.cursor()
+        table_name = self._meta.db_table
+        sql = "DROP TABLE %s;" % table_name
+        cursor.execute(sql)
+
+    @staticmethod
+    def percentile(N, percent, key=lambda x: x):
+        """
+        Find the percentile of a list of values.
+
+        @parameter N - is a list of values. Note N MUST BE already sorted.
+        @parameter percent - a float value from 0.0 to 1.0.
+        @parameter key - optional key function to compute value from each element of N.
+
+        @return - the percentile of the values
+        """
+        if not N:
+            return None
+        k = (len(N) - 1) * percent
+        f = math.floor(k)
+        c = math.ceil(k)
+        if f == c:
+            return key(N[int(k)])
+        d0 = key(N[int(f)]) * (c - k)
+        d1 = key(N[int(c)]) * (k - f)
+        return d0 + d1
+
+    class Meta:
+        abstract = True
+
+
+def getModel(want_date=now()):
+    class DynamicStatElem(StatElem):
+        class Meta:
+            db_table = 'flowstat_%s' % want_date.strftime("%d%m%Y")
+            abstract = False
+
+    return DynamicStatElem
+
+
+class StatCache(models.Model):
+    last_time = UnixDateTimeField()
+    abon = models.OneToOneField('abonapp.Abon', on_delete=models.CASCADE, primary_key=True)
+    octets = models.PositiveIntegerField(default=0)
+    packets = models.PositiveIntegerField(default=0)
+
+    def is_online(self):
+        return self.last_time > now() - timedelta(minutes=55)
+
+    def is_today(self):
+        return date.today() == self.last_time.date()
+
+    class Meta:
+        db_table = 'flowcache'
+        ordering = ('-last_time',)

traf_stat/templates/statistics/index.html

@@ -0,0 +1,45 @@
+{% extends 'base.html' %}
+{% load i18n %}
+
+{% block additional_link %}
+    <link href="/static/css/chartist.min.css" rel="stylesheet" type="text/css"/>
+    <script src="/static/js/chartist.min.js"></script>
+{% endblock %}
+
+{% block page-header %}{% trans 'Traffic' %}{% endblock %}
+
+{% block main %}
+    <script>
+    $(document).ready(function () {
+        var chart = new Chartist.Line('#maincontent', {
+            labels: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'],
+            series: [
+                [1, 5, 2, 5, 4, 3],
+                [2, 3, 4, 8, 1, 2],
+                [5, 4, 3, 2, 1, 0.5]
+            ]
+        }, {
+            low: 0,
+            showArea: true,
+            showPoint: false,
+            fullWidth: true,
+            height: 500
+        });
+
+        chart.on('draw', function (data) {
+            if (data.type === 'line' || data.type === 'area') {
+                data.element.animate({
+                    d: {
+                        begin: 2000 * data.index,
+                        dur: 2000,
+                        from: data.path.clone().scale(1, 0).translate(0, data.chartRect.height()).stringify(),
+                        to: data.path.clone().stringify(),
+                        easing: Chartist.Svg.Easing.easeOutQuint
+                    }
+                });
+            }
+        });
+    });
+    </script>
+    <div id="maincontent"></div>
+{% endblock %}

traf_stat/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

traf_stat/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+
+from traf_stat.views import home
+
+app_name = 'traf_stat'
+
+urlpatterns = [
+    path('', home, name='home'),
+]

traf_stat/views.py

@@ -0,0 +1,9 @@
+from django.shortcuts import render
+from django.contrib.auth.decorators import login_required
+from djing.lib.decorators import only_admins
+
+
+@login_required
+@only_admins
+def home(request):
+    return render(request, 'statistics/index.html')