Merge branch 'optimize' into devel

abonapp/templates/abonapp/peoples.html

@@ -63,27 +63,27 @@
                     </thead>
                     <tbody>
                     {% with can_ch_trf=perms.tariff_app.change_tariff can_del_abon=perms.abonapp.delete_abon %}
-                    {% for human in peoples %}
+                    {% for human in object_list %}
                         {% if human.is_active %}
                             <tr>
                         {% else %}
                             <tr class="danger">
                         {% endif %}
-                        <td>{% if human.stat_cache and human.stat_cache.is_online %}
+                        <td>{% if human.statcache.is_online %}
                                 <span class="glyphicon glyphicon-ok text-success"></span>
                             {% else %}
                                 <span class="glyphicon glyphicon-remove-sign text-muted"></span>
                             {% endif %}
                         </td>
                         <td class="col-xs-1">
-                            <a href="{% url 'abonapp:abon_home' human.group.pk human.username %}">{{ human.username }}</a>
+                            <a href="{% url 'abonapp:abon_home' human.group_id human.username %}">{{ human.username }}</a>
                         </td>
                         <td class="hidden-xs">
-                            {% if human.stat_cache %}
-                                {% if human.stat_cache.is_today %}
-                                    {{ human.stat_cache.last_time|date:"H:i" }}
+                            {% if human.statcache %}
+                                {% if human.statcache.is_today %}
+                                    {{ human.statcache.last_time|date:"H:i" }}
                                 {% else %}
-                                    {{ human.stat_cache.last_time|date:"D H:i" }}
+                                    {{ human.statcache.last_time|date:"D H:i" }}
                                 {% endif %}
                             {% endif %}
                         </td>

abonapp/test.sql

@@ -0,0 +1,21 @@
+SELECT
+  `base_accounts`.`username`,
+  `base_accounts`.`telephone`,
+  `abonent`.`ballance`,
+  `abonent_tariff`.`tariff_id`,
+  `tariffs`.`title`,
+  `tariffs`.`speedIn`,
+  `tariffs`.`speedOut`,
+  `tariffs`.`amount`,
+  `groups`.`title`,
+  `abon_street`.`name`
+FROM `abonent`
+  INNER JOIN `base_accounts` ON (`abonent`.`baseaccount_ptr_id` = `base_accounts`.`id`)
+  INNER JOIN `groups` ON (`abonent`.`group_id` = `groups`.`id`)
+  LEFT OUTER JOIN `abonent_tariff` ON (`abonent`.`current_tariff_id` = `abonent_tariff`.`id`)
+  LEFT OUTER JOIN `tariffs` ON (`abonent_tariff`.`tariff_id` = `tariffs`.`id`)
+  LEFT OUTER JOIN `abon_street` ON (`abonent`.`street_id` = `abon_street`.`id`)
+  LEFT OUTER JOIN `flowcache` ON (`abonent`.`baseaccount_ptr_id` = `flowcache`.`abon_id`)
+WHERE (`base_accounts`.`is_admin` = 0 AND `abonent`.`group_id` = 46)
+ORDER BY `base_accounts`.`fio` ASC
+LIMIT 20;

abonapp/views.py

@@ -18,7 +18,6 @@ from djing.lib import DuplicateEntry
 from jsonview.decorators import json_view
 
 from agent.commands.dhcp import dhcp_commit, dhcp_expiry, dhcp_release
-from statistics.models import StatCache
 from tariff_app.models import Tariff
 from agent import NasFailedResult, Transmitter, NasNetworkError
 from . import forms
@@ -39,26 +38,21 @@ from djing.global_base_views import OrderedFilteredList, SecureApiView
 
 @method_decorator((login_required, lib.decorators.only_admins), name='dispatch')
 class PeoplesListView(OrderedFilteredList):
-    context_object_name = 'peoples'
     template_name = 'abonapp/peoples.html'
 
     def get_queryset(self):
-        # TODO: optimize that query
         street_id = lib.safe_int(self.request.GET.get('street'))
         gid = lib.safe_int(self.kwargs.get('gid'))
-        peoples_list = models.Abon.objects.all().select_related('group', 'street', 'current_tariff')
+        peoples_list = models.Abon.objects.filter(group__pk=gid)
         if street_id > 0:
-            peoples_list = peoples_list.filter(group__pk=gid, street=street_id)
-        else:
-            peoples_list = peoples_list.filter(group__pk=gid)
-
-        try:
-            for abon in peoples_list.iterator():
-                ips = tuple(p.ip for p in abon.ip_addresses.filter(is_active=True))
-                if len(ips) > 0:
-                    abon.stat_cache = StatCache.objects.filter(ip__in=ips).first()
-        except lib.LogicError as e:
-            messages.warning(self.request, e)
+            peoples_list = peoples_list.filter(street=street_id)
+        peoples_list = peoples_list.select_related(
+            'group', 'street', 'statcache', 'current_tariff'
+        ).only(
+            'group', 'street', 'statcache', 'fio',
+            'street', 'house', 'telephone', 'ballance', 'markers',
+            'username', 'is_active', 'current_tariff'
+        )
         ordering = self.get_ordering()
         if ordering and isinstance(ordering, str):
             ordering = (ordering,)
@@ -75,7 +69,7 @@ class PeoplesListView(OrderedFilteredList):
 
         context = super(PeoplesListView, self).get_context_data(**kwargs)
 
-        context['streets'] = models.AbonStreet.objects.filter(group=gid)
+        context['streets'] = models.AbonStreet.objects.filter(group=gid).only('name')
         context['street_id'] = lib.safe_int(self.request.GET.get('street'))
         context['group'] = group
         return context
@@ -449,9 +443,7 @@ def pick_tariff(request, gid, uname):
 @permission_required('abonapp.delete_abontariff')
 def unsubscribe_service(request, gid, uname, abon_tariff_id):
     try:
-        #abon = get_object_or_404(models.Abon, username=uname)
         abon_tariff = get_object_or_404(models.AbonTariff, pk=int(abon_tariff_id))
-        #abon.disable_on_nas()
         abon_tariff.delete()
         messages.success(request, _('User has been detached from service'))
     except NasFailedResult as e:
@@ -704,7 +696,7 @@ def abon_ping(request):
 
 @login_required
 def vcards(r):
-    abons = models.Abon.objects.exclude(group=None).select_related('group', 'street').only(
+    users = models.Abon.objects.exclude(group=None).select_related('group', 'street').only(
         'username', 'fio', 'group__title', 'telephone',
         'street__name', 'house'
     )
@@ -718,7 +710,7 @@ def vcards(r):
             "END:VCARD\r\n")
 
     def _make_vcard():
-        for ab in abons.iterator():
+        for ab in users.iterator():
             tel = ab.telephone
             if tel:
                 yield tmpl % {
@@ -1215,7 +1207,7 @@ class DhcpLever(SecureApiView):
     @staticmethod
     def on_dhcp_event(data: Dict) -> Optional[str]:
         """
-        data = {
+        :param data = {
             'client_ip': ip_address('127.0.0.1'),
             'client_mac': 'aa:bb:cc:dd:ee:ff',
             'switch_mac': 'aa:bb:cc:dd:ee:ff',

accounts_app/models.py

@@ -94,7 +94,7 @@ class UserProfileManager(MyUserManager):
 
 
 class UserProfile(BaseAccount):
-    avatar = models.ImageField(_('Avatar'), upload_to=os.path.join('user', 'avatar'), null=True, default=None)
+    avatar = models.ImageField(_('Avatar'), upload_to=os.path.join('user', 'avatar'), null=True, default=None, blank=True)
     email = models.EmailField(default='')
     responsibility_groups = models.ManyToManyField(Group, blank=True, verbose_name=_('Responsibility groups'))
 

agent/netflow/mysql_install.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `flowcache` (
+  `last_time` INT(10) UNSIGNED NOT NULL,
+  `abon_id`   INT(11) DEFAULT NULL UNIQUE,
+  `octets`    INT(10) UNSIGNED NOT NULL,
+  `packets`   INT(10) UNSIGNED NOT NULL,
+  KEY `flowcache_abon_id_91e1085d` (`abon_id`)
+)
+  ENGINE = MEMORY
+  DEFAULT CHARSET = utf8;

agent/netflow/netflow_handler.py

@@ -35,9 +35,11 @@ if __name__ == '__main__':
     cursor = db.cursor()
 
     sql = (
-        'SELECT abonent.ip_address, acc.username  FROM abonent '
+        'SELECT INET_ATON(emps.ip) as uip, acc.id FROM abonent '
         'LEFT JOIN base_accounts AS acc ON (acc.id = abonent.baseaccount_ptr_id) '
-        'WHERE abonent.ip_address != 0'
+        'LEFT JOIN abonent_ip_addresses AS ips ON (acc.id = ips.abon_id) '
+        'LEFT JOIN ip_pool_employed_ip AS emps ON (ips.ipleasemodel_id = emps.id) '
+        'WHERE INET_ATON(emps.ip) != 0;'
     )
     ln = cursor.execute(sql)
     with open(tmp_ipuser_file, 'w') as f:
@@ -46,7 +48,7 @@ if __name__ == '__main__':
             row = cursor.fetchone()
             if row is None:
                 break
-            f.write("%d-%s\n" % row)
+            f.write("%d-%d\n" % row)
     db.close()
 
     os.system(

djing/lib/__init__.py

@@ -128,6 +128,7 @@ def process_lock(fn):
                 s.close()
     return wrapped
 
+
 #
 # Raises when IntegrityError in db
 #

statistics/migrations/0003_auto_20180814_1921.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2018-08-14 19:21
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from statistics.fields import UnixDateTimeField
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('abonapp', '0002_auto_20180808_1448'),
+        ('statistics', '0002_auto_20180808_1236'),
+    ]
+
+    operations = [
+        migrations.DeleteModel('StatCache'),
+        migrations.CreateModel(
+            name='StatCache',
+            fields=[
+                ('last_time', UnixDateTimeField()),
+                ('abon', models.OneToOneField(on_delete=models.deletion.CASCADE, primary_key=True, serialize=False, to='abonapp.Abon')),
+                ('octets', models.PositiveIntegerField(default=0)),
+                ('packets', models.PositiveIntegerField(default=0)),
+            ],
+            options={
+                'db_table': 'flowcache',
+            },
+        )
+    ]

statistics/models.py

@@ -14,7 +14,7 @@ def get_dates():
 
 
 class StatManager(models.Manager):
-    def chart(self, username, count_of_parts=12, want_date=date.today()):
+    def chart(self, user, count_of_parts=12, want_date=date.today()):
         def byte_to_mbit(x):
             return ((x / 60) * 8) / 2 ** 20
 
@@ -28,7 +28,7 @@ class StatManager(models.Manager):
             return sum(elements) / len(elements)
 
         try:
-            charts_data = self.filter(uname=username)
+            charts_data = self.filter(abon=user)
             charts_times = tuple(cd.cur_time.timestamp() * 1000 for cd in charts_data)
             charts_octets = tuple(cd.octets for cd in charts_data)
             if len(charts_octets) > 0 and len(charts_octets) == len(charts_times):
@@ -53,7 +53,7 @@ class StatManager(models.Manager):
 
 class StatElem(models.Model):
     cur_time = UnixDateTimeField(primary_key=True)
-    uname = models.CharField(max_length=127, blank=True, null=True, default=None)
+    abon = models.ForeignKey('abonapp.Abon', on_delete=models.CASCADE, null=True, default=None, blank=True)
     ip = MyGenericIPAddressField()
     octets = models.PositiveIntegerField(default=0)
     packets = models.PositiveIntegerField(default=0)
@@ -115,7 +115,8 @@ def getModel(want_date=now()):
 
 class StatCache(models.Model):
     last_time = UnixDateTimeField()
-    ip = MyGenericIPAddressField(primary_key=True)
+    # ip = MyGenericIPAddressField(primary_key=True)
+    abon = models.OneToOneField('abonapp.Abon', on_delete=models.CASCADE, primary_key=True)
     octets = models.PositiveIntegerField(default=0)
     packets = models.PositiveIntegerField(default=0)