Fix nas system bug

abonapp/models.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from ipaddress import ip_address
 from typing import Optional
 
 from django.conf import settings
@@ -14,7 +15,7 @@ from django.utils.translation import ugettext_lazy as _, gettext
 from accounts_app.models import UserProfile, MyUserManager, BaseAccount
 from agent import Transmitter, AbonStruct, TariffStruct, NasFailedResult, NasNetworkError
 from group_app.models import Group
-from djing.lib import ip2int, LogicError
+from djing.lib import LogicError
 from djing.fields import MyGenericIPAddressField
 from djing import IP_ADDR_REGEX
 from tariff_app.models import Tariff, PeriodicPay
@@ -258,7 +259,7 @@ class Abon(BaseAccount):
     # make subscriber from agent structure
     def build_agent_struct(self):
         if self.ip_address:
-            user_ip = ip2int(self.ip_address)
+            user_ip = ip_address(self.ip_address)
         else:
             return
         abon_tariff = self.active_tariff()

abonapp/views.py

@@ -1227,7 +1227,7 @@ class DhcpLever(SecureApiView):
     def on_dhcp_event(data: Dict) -> Optional[str]:
         """
         data = {
-            'client_ip': ip2int('127.0.0.1'),
+            'client_ip': ip_address('127.0.0.1'),
             'client_mac': 'aa:bb:cc:dd:ee:ff',
             'switch_mac': 'aa:bb:cc:dd:ee:ff',
             'switch_port': 3,

agent/core.py

@@ -82,9 +82,11 @@ class BaseTransmitter(metaclass=ABCMeta):
         :param users_from_db: QuerySet of all subscribers that can have service
         :return: Tuple of 2 lists that contain list to add users and list to remove users
         """
-        users_struct_list = [ab.build_agent_struct() for ab in users_from_db if ab.is_access()]
-        users_struct_set = set([ab for ab in users_struct_list if ab is not None and ab.tariff is not None])
+        users_struct_list = (ab.build_agent_struct() for ab in users_from_db if ab.is_access())
+        users_struct_set = set(ab for ab in users_struct_list if ab is not None and ab.tariff is not None)
         users_from_nas = set(self.read_users())
+        if len(users_from_nas) < 1:
+            print('WARNING: Not have users from NAS')
         list_for_del = (users_struct_set ^ users_from_nas) - users_struct_set
         list_for_add = users_struct_set - users_from_nas
         return list_for_add, list_for_del

agent/mod_mikrotik.py

@@ -4,14 +4,14 @@ import socket
 import binascii
 from abc import ABCMeta
 from hashlib import md5
+from ipaddress import _BaseAddress
 from typing import Iterable, Optional, Tuple
-from .core import BaseTransmitter, NasFailedResult, NasNetworkError
+from django.conf import settings
+from .core import NasFailedResult, NasNetworkError, BaseTransmitter
 from djing.lib import Singleton
-from .structs import TariffStruct, AbonStruct, IpStruct, VectorAbon, VectorTariff
+from .structs import TariffStruct, AbonStruct, VectorAbon, VectorTariff
 from . import settings as local_settings
-from django.conf import settings
 from djing import ping
-from agent.core import NasNetworkError, NasFailedResult
 
 DEBUG = getattr(settings, 'DEBUG', False)
 
@@ -221,20 +221,28 @@ class TransmitterManager(BaseTransmitter, metaclass=ABCMeta):
                 res = float(re.sub(r'[a-zA-Z]', '', text_speed)) / 1000 ** 2
             return res
 
-        speeds = info['=max-limit'].split('/')
+        speed_out, speed_in = info['=max-limit'].split('/')
         t = TariffStruct(
-            speed_in=parse_speed(speeds[1]),
-            speed_out=parse_speed(speeds[0])
+            speed_in=parse_speed(speed_in),
+            speed_out=parse_speed(speed_out)
         )
         try:
+            target = info.get('=target')
+            if target is None:
+                target = info.get('=target-addresses')
+            name = info.get('=name')
+            disabled = info.get('=disabled')
+            if disabled is not None:
+                disabled = True if disabled == 'true' else False
+            if target is not None and name is not None:
+                target_ip, target_net = target.split('/')
                 a = AbonStruct(
-                uid=int(info['=name'][3:]),
-                # FIXME: тут в разных микротиках или =target-addresses или =target
-                ip=info['=target'][:-3],
+                    uid=int(name[3:]),
+                    ip=target_ip,
                     tariff=t,
-                is_active=False if info['=disabled'] == 'false' else True
+                    is_active=disabled or False
                 )
-            a.queue_id = info['=.id']
+                a.queue_id = info.get('=.id')
                 return a
         except ValueError:
             pass
@@ -330,15 +338,9 @@ class QueueManager(TransmitterManager, metaclass=ABCMeta):
             return self._exec_cmd(('/queue/simple/enable', '=.id=*' + getattr(q, 'queue_id', '')))
 
 
-class IpAddressListObj(IpStruct):
-    def __init__(self, ip, mk_id):
-        super(IpAddressListObj, self).__init__(ip)
-        self.mk_id = str(mk_id).replace('*', '')
-
-
 class IpAddressListManager(TransmitterManager, metaclass=ABCMeta):
-    def add(self, list_name: str, ip: IpStruct):
-        if not isinstance(ip, IpStruct):
+    def add(self, list_name: str, ip):
+        if not issubclass(ip.__class__, _BaseAddress):
             raise TypeError
         commands = (
             '/ip/firewall/address-list/add',
@@ -350,19 +352,19 @@ class IpAddressListManager(TransmitterManager, metaclass=ABCMeta):
     def remove(self, mk_id):
         return self._exec_cmd((
             '/ip/firewall/address-list/remove',
-            '=.id=*' + str(mk_id).replace('*', '')
+            '=.id=%s' % mk_id
         ))
 
-    def remove_range(self, items: Iterable[IpAddressListObj]):
-        ids = tuple(ip.mk_id for ip in items if isinstance(ip, IpAddressListObj))
+    def remove_range(self, items):
+        ids = tuple(ip_mkid.mkid for ip_mkid in items)
         if len(ids) > 0:
             return self._exec_cmd([
                 '/ip/firewall/address-list/remove',
-                '=numbers=*%s' % ',*'.join(ids)
+                '=numbers=%s' % ','.join(ids)
             ])
 
-    def find(self, ip: IpStruct, list_name: str):
-        if not isinstance(ip, IpStruct):
+    def find(self, ip, list_name: str):
+        if not issubclass(ip.__class__, _BaseAddress):
             raise TypeError
         return self._exec_cmd((
             '/ip/firewall/address-list/print', 'where',
@@ -378,7 +380,7 @@ class IpAddressListManager(TransmitterManager, metaclass=ABCMeta):
         ))
         for code, dat in ips:
             if dat != {}:
-                yield IpAddressListObj(dat['=address'], dat['=.id'])
+                yield dat.get('=address'), dat.get('=.id')
 
     def disable(self, user: AbonStruct):
         r = IpAddressListManager.find(self, user.ip, LIST_USERS_ALLOWED)
@@ -415,7 +417,7 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager):
                 IpAddressListManager.remove(self, ip_list_entity[0]['=.id'])
 
     def add_user(self, user: AbonStruct, *args):
-        if not isinstance(user.ip, IpStruct):
+        if not issubclass(user.ip.__class__, _BaseAddress):
             raise TypeError
         if user.tariff is None:
             return
@@ -437,7 +439,7 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager):
             IpAddressListManager.remove(self, firewall_ip_list_obj[0]['=.id'])
 
     def update_user(self, user: AbonStruct, *args):
-        if not isinstance(user.ip, IpStruct):
+        if not issubclass(user.ip.__class__, _BaseAddress):
             raise TypeError
 
         find_res = IpAddressListManager.find(self, user.ip, LIST_USERS_ALLOWED)
@@ -507,14 +509,30 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager):
         pass
 
     def read_users(self) -> Iterable[AbonStruct]:
+
+        class ip_mkid_struct(object):
+            __slots__ = ('ip', 'mkid')
+
+            def __init__(self, ip, mkid):
+                self.ip = ip
+                self.mkid = mkid
+
+            def __eq__(self, other):
+                if isinstance(other, ip_mkid_struct):
+                    return self.ip == other.ip
+                return self.ip == str(other)
+
+            def __hash__(self):
+                return hash(self.ip)
+
         # shapes is ShapeItem
-        allowed_ips = set(IpAddressListManager.read_ips_iter(self, LIST_USERS_ALLOWED))
-        queues = tuple(q for q in QueueManager.read_queue_iter(self) if q.ip in allowed_ips)
+        all_ips = set(ip_mkid_struct(ip, mkid) for ip, mkid in IpAddressListManager.read_ips_iter(self, LIST_USERS_ALLOWED))
+        queues = tuple(q for q in QueueManager.read_queue_iter(self) if str(q.ip) in all_ips)
 
-        ips_from_queues = set(q.ip for q in queues)
+        ips_from_queues = set(str(q.ip) for q in queues)
 
         # delete ip addresses that are in firewall/address-list and there are no corresponding in queues
-        diff = tuple(allowed_ips - ips_from_queues)
+        diff = tuple(all_ips - ips_from_queues)
         if len(diff) > 0:
             IpAddressListManager.remove_range(self, diff)
 

agent/structs.py

@@ -1,78 +1,23 @@
-# -*- coding: utf-8 -*-
-from abc import ABCMeta, abstractmethod
-from struct import pack, unpack, calcsize
-from typing import Iterable, Optional
-from djing.lib import int2ip, ip2int
+from abc import ABCMeta
+from ipaddress import ip_address, _BaseAddress
+from typing import Iterable
 
 
 class BaseStruct(object, metaclass=ABCMeta):
-    @abstractmethod
-    def serialize(self) -> Optional[bytes]:
-        """make binary"""
-
-    @abstractmethod
-    def deserialize(self, data: bytes, *args):
-        """restore from binary"""
 
     def __ne__(self, other):
         return not self == other
 
 
-class IpStruct(BaseStruct):
-    def __init__(self, ip):
-        if type(ip) is int:
-            self.__ip = ip
-        else:
-            self.__ip = ip2int(str(ip))
-
-    def serialize(self) -> Optional[bytes]:
-        dt = pack("!I", int(self.__ip))
-        return dt
-
-    def deserialize(self, data: bytes, *args):
-        dt = unpack("!I", data)
-        self.__ip = int(dt[0])
-        return self
-
-    def get_int(self):
-        return self.__ip
-
-    def __eq__(self, other):
-        if not isinstance(other, IpStruct):
-            raise TypeError('Instance must be IpStruct')
-        return self.__ip == other.__ip
-
-    def __int__(self):
-        return self.__ip
-
-    def __str__(self):
-        return int2ip(self.__ip)
-
-    def __hash__(self):
-        return hash(self.__ip)
-
-
-# Как обслуживается абонент
 class TariffStruct(BaseStruct):
     def __init__(self, tariff_id=0, speed_in=None, speed_out=None):
         self.tid = int(tariff_id)
         self.speedIn = speed_in or 0
         self.speedOut = speed_out or 0
 
-    def serialize(self) -> Optional[bytes]:
-        dt = pack("!Iff", int(self.tid), float(self.speedIn), float(self.speedOut))
-        return dt
-
-    # Да, если все значения нулевые
+    # Yes, if all variables is zeroed
     def is_empty(self):
-        return self.tid == 0 and self.speedIn == 0.001 and self.speedOut == 0.001
-
-    def deserialize(self, data: bytes, *args):
-        dt = unpack("!Iff", data)
-        self.tid = int(dt[0])
-        self.speedIn = float(dt[1])
-        self.speedOut = float(dt[2])
-        return self
+        return self.tid == 0 and self.speedIn == 0 and self.speedOut == 0
 
     def __eq__(self, other):
         # не сравниваем id, т.к. тарифы с одинаковыми скоростями для NAS одинаковы
@@ -88,35 +33,19 @@ class TariffStruct(BaseStruct):
         return hash(str(self.speedIn) + str(self.speedOut))
 
 
-# Абонент из базы
+# Abon from database
 class AbonStruct(BaseStruct):
+    __slots__ = ('ip', 'uid', 'tariff', 'is_active', 'queue_id')
+
     def __init__(self, uid=0, ip=None, tariff=None, is_active=True):
         self.uid = int(uid or 0)
-        self.ip = IpStruct(ip)
+        if issubclass(ip.__class__, _BaseAddress):
+            self.ip = ip
+        else:
+            self.ip = ip_address(ip)
         self.tariff = tariff
         self.is_active = is_active
 
-    def serialize(self) -> Optional[bytes]:
-        if self.tariff is None:
-            return
-        if not isinstance(self.tariff, TariffStruct):
-            raise TypeError('Instance must be TariffStruct')
-        if not isinstance(self.ip, IpStruct):
-            raise TypeError('Instance must be IpStruct')
-        dt = pack("!LII?", self.uid, int(self.ip), self.tariff.tid, self.is_active)
-        return dt
-
-    def deserialize(self, data: bytes, tariff=None):
-        dt = unpack("!LII?", data)
-        self.uid = dt[0]
-        self.ip = IpStruct(dt[1])
-        if tariff is not None:
-            if not isinstance(tariff, TariffStruct):
-                raise TypeError
-            self.tariff = tariff
-        self.is_active = dt['3']
-        return self
-
     def __eq__(self, other):
         if not isinstance(other, AbonStruct):
             raise TypeError
@@ -131,24 +60,12 @@ class AbonStruct(BaseStruct):
         return hash(int(self.ip) + hash(self.tariff)) if self.tariff is not None else 0
 
 
-# Правило шейпинга в фаере, или ещё можно сказать услуга абонента на NAS
+# Shape rule from NAS(Network Access Server)
 class ShapeItem(BaseStruct):
     def __init__(self, abon, sid):
         self.abon = abon
         self.sid = sid
 
-    def serialize(self) -> Optional[bytes]:
-        abon_pack = self.abon.serialize()
-        dt = pack('!L', self.sid)
-        return dt + abon_pack
-
-    def deserialize(self, data: bytes, *args):
-        sz = calcsize('!L')
-        dt = unpack('!L', data[:sz])
-        self.sid = dt
-        self.abon.deserialize(data[sz:])
-        return self
-
     def __eq__(self, other):
         if not isinstance(other, ShapeItem):
             raise TypeError

agent/utils.py

@@ -1,16 +0,0 @@
-import socket
-import struct
-
-
-def ip2int(addr):
-    try:
-        return struct.unpack("!I", socket.inet_aton(addr))[0]
-    except:
-        return 0
-
-
-def int2ip(addr):
-    try:
-        return socket.inet_ntoa(struct.pack("!I", addr))
-    except:
-        return ''

dhcp_lever.py

@@ -16,7 +16,7 @@ def die(text):
 
 '''
 obj = {
-    'client_ip': ip2int('127.0.0.1'),
+    'client_ip': ip_address('127.0.0.1'),
     'client_mac': 'aa:bb:cc:dd:ee:ff',
     'switch_mac': 'aa:bb:cc:dd:ee:ff',
     'switch_port': 3,

djing/fields.py

@@ -1,11 +1,11 @@
 #
 # I got it on https://github.com/django-macaddress/django-macaddress
 #
+from ipaddress import ip_address
+from netaddr import EUI, AddrFormatError
 from django.core.exceptions import ValidationError
 from django.db import models
-from netaddr import EUI, AddrFormatError
 
-from djing.lib import ip2int, int2ip
 from .formfields import MACAddressField as MACAddressFormField
 from . import default_dialect
 import warnings
@@ -121,7 +121,7 @@ class MyGenericIPAddressField(models.GenericIPAddressField):
     def get_prep_value(self, value):
         # strIp to Int
         value = super(MyGenericIPAddressField, self).get_prep_value(value)
-        return ip2int(value)
+        return int(ip_address(value))
 
     def to_python(self, value):
         return value
@@ -131,7 +131,8 @@ class MyGenericIPAddressField(models.GenericIPAddressField):
 
     @staticmethod
     def from_db_value(value, expression, connection, context):
-        return int2ip(value) if value != 0 else None
+        if value:
+            return str(ip_address(value))
 
     def int_ip(self):
-        return ip2int(self)
+        return int(ip_address(self))

periodic.py

@@ -39,9 +39,9 @@ def main():
     try:
         tm = Transmitter()
         users = Abon.objects.filter(is_active=True).exclude(current_tariff=None)
-        tm.sync_nas(users)
-    except NasNetworkError as e:
-        print('NetworkTrouble:', e)
+        tm.sync_nas(users.iterator())
+    except NasNetworkError as er:
+        print('NetworkTrouble:', er)
 
     # manage periodic pays
     ppays = PeriodicPayForId.objects.filter(next_pay__lt=now) \