Fix syncing users with nas

abonapp/migrations/0002_ip_address_change.py

@@ -1,103 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11 on 2018-06-12 12:05
-from __future__ import unicode_literals
-import os
-import re
-from json import load
-
-import django.core.validators
-from django.db import migrations, models
-from django.core import serializers
-
-
-TMP_FILE = '/tmp/djing_ip_field_abonapp_migrate.json'
-
-
-def backup_info(apps, _):
-    Abon = apps.get_model('abonapp', 'Abon')
-    obs = Abon.objects.all()
-    with open(TMP_FILE, 'w') as f:
-        serializers.serialize('json', obs, stream=f)
-
-
-def restore_info_to_new_scheme(apps, _):
-    Abon = apps.get_model('abonapp', 'Abon')
-    with open(TMP_FILE, 'r') as f:
-        for abon in load(f):
-            Abon.objects.filter(pk=abon['pk']).update(ip_address=abon['fields']['ip_address'])
-    if os.path.isfile(TMP_FILE):
-        os.remove(TMP_FILE)
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('abonapp', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.RunPython(backup_info),
-        migrations.AlterModelOptions(
-            name='abon',
-            options={'ordering': ('fio',), 'permissions': (('can_buy_tariff', 'Buy service perm'), ('can_view_passport', 'Can view passport'), ('can_add_ballance', 'fill account'), ('can_ping', 'Can ping')), 'verbose_name': 'Abon', 'verbose_name_plural': 'Abons'},
-        ),
-        migrations.AlterModelOptions(
-            name='abonlog',
-            options={'ordering': ('-date',), 'permissions': (('can_view_abonlog', 'Can view subscriber logs'),)},
-        ),
-        migrations.AlterModelOptions(
-            name='abonstreet',
-            options={'ordering': ('name',), 'verbose_name': 'Street', 'verbose_name_plural': 'Streets'},
-        ),
-        migrations.AlterModelOptions(
-            name='abontariff',
-            options={'ordering': ('time_start',), 'permissions': (('can_complete_service', 'finish service perm'),), 'verbose_name': 'Abon service', 'verbose_name_plural': 'Abon services'},
-        ),
-        migrations.AlterModelOptions(
-            name='allpaylog',
-            options={'ordering': ('-date_action',)},
-        ),
-        migrations.AlterModelOptions(
-            name='alltimepaylog',
-            options={'ordering': ('-date_add',)},
-        ),
-        migrations.AlterModelOptions(
-            name='passportinfo',
-            options={'ordering': ('series',), 'verbose_name': 'Passport Info', 'verbose_name_plural': 'Passport Info'},
-        ),
-        migrations.AlterModelOptions(
-            name='periodicpayforid',
-            options={'ordering': ('last_pay',)},
-        ),
-        migrations.AlterField(
-            model_name='abon',
-            name='ip_address',
-            field=models.GenericIPAddressField(blank=True, null=True, verbose_name='Ip Address'),
-        ),
-        migrations.AlterField(
-            model_name='additionaltelephone',
-            name='telephone',
-            field=models.CharField(max_length=16, validators=[django.core.validators.RegexValidator('^(\\+[7,8,9,3]\\d{10,11})?$')], verbose_name='Telephone'),
-        ),
-        migrations.AlterField(
-            model_name='passportinfo',
-            name='abon',
-            field=models.OneToOneField(blank=True, null=True, on_delete=models.deletion.CASCADE, to='abonapp.Abon'),
-        ),
-        migrations.AlterField(
-            model_name='passportinfo',
-            name='distributor',
-            field=models.CharField(max_length=64, verbose_name='Distributor'),
-        ),
-        migrations.AlterField(
-            model_name='passportinfo',
-            name='number',
-            field=models.CharField(max_length=6, validators=[django.core.validators.RegexValidator(re.compile('^-?\\d+\\Z', 32), code='invalid', message='Enter a valid integer.')], verbose_name='Pasport number'),
-        ),
-        migrations.AlterField(
-            model_name='passportinfo',
-            name='series',
-            field=models.CharField(max_length=4, validators=[django.core.validators.RegexValidator(re.compile('^-?\\d+\\Z', 32), code='invalid', message='Enter a valid integer.')], verbose_name='Pasport serial'),
-        ),
-        migrations.RunPython(restore_info_to_new_scheme)
-    ]

abonapp/models.py

@@ -250,10 +250,10 @@ class Abon(BaseAccount):
     def get_absolute_url(self):
         return resolve_url('abonapp:abon_home', self.group.id, self.username)
 
-    def add_lease(self, ip: str):
+    def add_lease(self, ip: str, mac_addr=None):
         existed_client_ips = tuple(l.ip for l in self.ip_addresses.all())
         if ip not in existed_client_ips:
-            lease = IpLeaseModel.objects.create_from_ip(ip=ip)
+            lease = IpLeaseModel.objects.create_from_ip(ip=ip, net=None, mac=mac_addr)
             if lease is None:
                 return 'Subnet not found'
             self.ip_addresses.add(lease)

agent/core.py

@@ -96,8 +96,8 @@ class BaseTransmitter(ABC):
         :param users_from_db: QuerySet of all subscribers that can have service
         :return: Tuple of 2 lists that contain list to add users and list to remove users
         """
-        users_struct_list = (ab.build_agent_struct() for ab in users_from_db if ab.is_access())
-        users_struct_set = set(ab for ab in users_struct_list if ab is not None and ab.tariff is not None)
+        users_struct_gen = (ab.build_agent_struct() for ab in users_from_db if ab.is_access())
+        users_struct_set = set(ab for ab in users_struct_gen if ab is not None and ab.tariff is not None)
         users_from_nas = set(self.read_users())
         if len(users_from_nas) < 1:
             print('WARNING: Not have users from NAS')
@@ -108,6 +108,12 @@ class BaseTransmitter(ABC):
     def sync_nas(self, users_from_db: Iterator):
         list_for_add, list_for_del = self._diff_users(users_from_db)
         if len(list_for_del) > 0:
+            print('List for del:')
+            for ld in list_for_del:
+                print('\t', ld.ips)
             self.remove_user_range(list_for_del)
         if len(list_for_add) > 0:
+            print('List for add:')
+            for la in list_for_add:
+                print('\t', la.ips)
             self.add_user_range(list_for_add)

agent/mod_mikrotik.py

@@ -168,7 +168,6 @@ class ApiRos(object):
 
 
 class MikrotikTransmitter(BaseTransmitter, ApiRos, metaclass=type('_ABC_Lazy_mcs', (ABCMeta, LazyInitMetaclass), {})):
-
     def __init__(self, login=None, password=None, ip=None, port=None):
         ip = ip or getattr(local_settings, 'NAS_IP')
         if ip is None or ip == '<NAS IP>':
@@ -473,30 +472,9 @@ class MikrotikTransmitter(BaseTransmitter, ApiRos, metaclass=type('_ABC_Lazy_mcs
         pass
 
     def read_users(self) -> VectorAbon:
-        class ip_mkid_struct(object):
-            __slots__ = ('ip', 'mkid')
-
-            def __init__(self, ip, mkid):
-                self.ip = ip
-                self.mkid = mkid
-
-            def __eq__(self, other):
-                if isinstance(other, ip_mkid_struct):
-                    return self.ip == other.ip
-                return self.ip == str(other)
-
-            def __hash__(self):
-                return hash(self.ip)
         # shapes is ShapeItem
-        all_ips = set(ip_mkid_struct(ip, mkid) for ip, mkid in self.read_ips_iter(LIST_USERS_ALLOWED))
-        queues = (q for q in self.read_queue_iter() if str(q.ip) in all_ips)
-
-        # ips_from_queues = set(str(q.ip) for q in queues)
-
-        # delete ip addresses that are in firewall/address-list and there are no corresponding in queues
-        #diff = tuple(all_ips - ips_from_queues)
-        #if len(diff) > 0:
-        #    self.remove_ip_range(diff)
+        all_ips = set(ip for ip, mkid in self.read_ips_iter(LIST_USERS_ALLOWED))
+        queues = (q for q in self.read_queue_iter() if all_ips.issuperset(q.ips))
         return queues
 
     def lease_free(self, user: AbonStruct, lease):

ip_pool/migrations/0001_initial.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2018-08-07 15:48
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import djing.fields
+import ip_pool.fields
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('group_app', '0002_group_code'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='IpLeaseModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ip', models.GenericIPAddressField(unique=True, verbose_name='Ip address')),
+                ('mac_addr', djing.fields.MACAddressField(blank=True, integer=True, null=True, unique=True, verbose_name='Mac address')),
+                ('lease_time', models.DateTimeField(auto_now_add=True, verbose_name='Lease time')),
+                ('is_dynamic', models.BooleanField(default=False, verbose_name='Is dynamic')),
+                ('is_active', models.BooleanField(default=True, verbose_name='Is active')),
+                ('device_info', models.CharField(blank=True, default=None, max_length=128, null=True)),
+            ],
+            options={
+                'verbose_name': 'Employed ip',
+                'verbose_name_plural': 'Employed ip addresses',
+                'db_table': 'ip_pool_employed_ip',
+                'ordering': ('-id',),
+            },
+        ),
+        migrations.CreateModel(
+            name='NetworkModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('network', ip_pool.fields.GenericIpAddressWithPrefix(help_text='Ip address of network. For example: 192.168.1.0 or fde8:6789:1234:1::', unique=True, verbose_name='IP network')),
+                ('kind', models.CharField(choices=[('inet', 'Internet'), ('guest', 'Guest'), ('trust', 'Trusted'), ('device', 'Devices'), ('admin', 'Admin')], default='guest', max_length=6, verbose_name='Kind of network')),
+                ('description', models.CharField(max_length=64, verbose_name='Description')),
+                ('ip_start', models.GenericIPAddressField(verbose_name='Start work ip range')),
+                ('ip_end', models.GenericIPAddressField(verbose_name='End work ip range')),
+                ('groups', models.ManyToManyField(to='group_app.Group', verbose_name='Groups')),
+            ],
+            options={
+                'verbose_name': 'Network',
+                'verbose_name_plural': 'Networks',
+                'db_table': 'ip_pool_network',
+                'ordering': ('network',),
+            },
+        ),
+        migrations.AddField(
+            model_name='ipleasemodel',
+            name='network',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ip_pool.NetworkModel', verbose_name='Parent network'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='ipleasemodel',
+            unique_together={('ip', 'network')},
+        ),
+    ]

ip_pool/models.py

@@ -1,6 +1,6 @@
 from datetime import timedelta
 from ipaddress import ip_network, ip_address
-from typing import AnyStr
+from typing import Optional
 
 from django.conf import settings
 from django.db.utils import IntegrityError
@@ -85,7 +85,7 @@ class NetworkModel(models.Model):
                 })
                 raise ValidationError(errs)
 
-    def get_scope(self) -> AnyStr:
+    def get_scope(self) -> str:
         net = self.get_network()
         if net.is_global:
             return _('Global')
@@ -138,14 +138,15 @@ class IpLeaseManager(models.Manager):
                 if work_range_start_ip <= net <= work_range_end_ip:
                     return net
 
-    def create_from_ip(self, ip: str, net: NetworkModel, is_dynamic=True):
+    def create_from_ip(self, ip: str, net: Optional[NetworkModel], mac=None, is_dynamic=True):
         # ip = ip_address(ip)
         try:
             return self.create(
                 ip=ip,
                 network=net,
                 is_dynamic=is_dynamic,
-                is_active=True
+                is_active=True,
+                mac_addr=mac
             )
         except IntegrityError as e:
             if 'Duplicate entry' in str(e):

periodic.py

@@ -39,8 +39,12 @@ def main():
     # sync subscribers on NAS
     try:
         tm = Transmitter()
-        users = Abon.objects.filter(is_active=True).exclude(current_tariff=None)
-        tm.sync_nas(users.iterator())
+        users = Abon.objects\
+            .filter(is_active=True)\
+            .exclude(current_tariff=None)\
+            .prefetch_related('ip_addresses')\
+            .iterator()
+        tm.sync_nas(users)
     except NasNetworkError as er:
         print('NetworkTrouble:', er)