From a643bf995aeb74c4e72a6efb5ff018fac92a3859 Mon Sep 17 00:00:00 2001 From: Dmitry Novikov Date: Wed, 25 Jul 2018 04:28:16 +0300 Subject: [PATCH] Fix nas system bug --- abonapp/models.py | 5 +- abonapp/views.py | 2 +- agent/core.py | 6 ++- agent/mod_mikrotik.py | 94 +++++++++++++++++++++--------------- agent/structs.py | 109 +++++------------------------------------- agent/utils.py | 16 ------- dhcp_lever.py | 2 +- djing/fields.py | 11 +++-- periodic.py | 6 +-- 9 files changed, 87 insertions(+), 164 deletions(-) delete mode 100644 agent/utils.py diff --git a/abonapp/models.py b/abonapp/models.py index 4227243..49f801c 100644 --- a/abonapp/models.py +++ b/abonapp/models.py @@ -1,4 +1,5 @@ from datetime import datetime +from ipaddress import ip_address from typing import Optional from django.conf import settings @@ -14,7 +15,7 @@ from django.utils.translation import ugettext_lazy as _, gettext from accounts_app.models import UserProfile, MyUserManager, BaseAccount from agent import Transmitter, AbonStruct, TariffStruct, NasFailedResult, NasNetworkError from group_app.models import Group -from djing.lib import ip2int, LogicError +from djing.lib import LogicError from djing.fields import MyGenericIPAddressField from djing import IP_ADDR_REGEX from tariff_app.models import Tariff, PeriodicPay @@ -258,7 +259,7 @@ class Abon(BaseAccount): # make subscriber from agent structure def build_agent_struct(self): if self.ip_address: - user_ip = ip2int(self.ip_address) + user_ip = ip_address(self.ip_address) else: return abon_tariff = self.active_tariff() diff --git a/abonapp/views.py b/abonapp/views.py index e1c72ee..e408283 100644 --- a/abonapp/views.py +++ b/abonapp/views.py @@ -1227,7 +1227,7 @@ class DhcpLever(SecureApiView): def on_dhcp_event(data: Dict) -> Optional[str]: """ data = { - 'client_ip': ip2int('127.0.0.1'), + 'client_ip': ip_address('127.0.0.1'), 'client_mac': 'aa:bb:cc:dd:ee:ff', 'switch_mac': 'aa:bb:cc:dd:ee:ff', 'switch_port': 3, diff --git a/agent/core.py b/agent/core.py index 3505597..c7aa5be 100644 --- a/agent/core.py +++ b/agent/core.py @@ -82,9 +82,11 @@ class BaseTransmitter(metaclass=ABCMeta): :param users_from_db: QuerySet of all subscribers that can have service :return: Tuple of 2 lists that contain list to add users and list to remove users """ - users_struct_list = [ab.build_agent_struct() for ab in users_from_db if ab.is_access()] - users_struct_set = set([ab for ab in users_struct_list if ab is not None and ab.tariff is not None]) + users_struct_list = (ab.build_agent_struct() for ab in users_from_db if ab.is_access()) + users_struct_set = set(ab for ab in users_struct_list if ab is not None and ab.tariff is not None) users_from_nas = set(self.read_users()) + if len(users_from_nas) < 1: + print('WARNING: Not have users from NAS') list_for_del = (users_struct_set ^ users_from_nas) - users_struct_set list_for_add = users_struct_set - users_from_nas return list_for_add, list_for_del diff --git a/agent/mod_mikrotik.py b/agent/mod_mikrotik.py index 5a53711..2126dca 100644 --- a/agent/mod_mikrotik.py +++ b/agent/mod_mikrotik.py @@ -4,19 +4,19 @@ import socket import binascii from abc import ABCMeta from hashlib import md5 +from ipaddress import _BaseAddress from typing import Iterable, Optional, Tuple -from .core import BaseTransmitter, NasFailedResult, NasNetworkError +from django.conf import settings +from .core import NasFailedResult, NasNetworkError, BaseTransmitter from djing.lib import Singleton -from .structs import TariffStruct, AbonStruct, IpStruct, VectorAbon, VectorTariff +from .structs import TariffStruct, AbonStruct, VectorAbon, VectorTariff from . import settings as local_settings -from django.conf import settings from djing import ping -from agent.core import NasNetworkError, NasFailedResult DEBUG = getattr(settings, 'DEBUG', False) LIST_USERS_ALLOWED = 'DjingUsersAllowed' -#LIST_USERS_BLOCKED = 'DjingUsersBlocked' +# LIST_USERS_BLOCKED = 'DjingUsersBlocked' class ApiRos(metaclass=Singleton): @@ -221,21 +221,29 @@ class TransmitterManager(BaseTransmitter, metaclass=ABCMeta): res = float(re.sub(r'[a-zA-Z]', '', text_speed)) / 1000 ** 2 return res - speeds = info['=max-limit'].split('/') + speed_out, speed_in = info['=max-limit'].split('/') t = TariffStruct( - speed_in=parse_speed(speeds[1]), - speed_out=parse_speed(speeds[0]) + speed_in=parse_speed(speed_in), + speed_out=parse_speed(speed_out) ) try: - a = AbonStruct( - uid=int(info['=name'][3:]), - # FIXME: тут в разных микротиках или =target-addresses или =target - ip=info['=target'][:-3], - tariff=t, - is_active=False if info['=disabled'] == 'false' else True - ) - a.queue_id = info['=.id'] - return a + target = info.get('=target') + if target is None: + target = info.get('=target-addresses') + name = info.get('=name') + disabled = info.get('=disabled') + if disabled is not None: + disabled = True if disabled == 'true' else False + if target is not None and name is not None: + target_ip, target_net = target.split('/') + a = AbonStruct( + uid=int(name[3:]), + ip=target_ip, + tariff=t, + is_active=disabled or False + ) + a.queue_id = info.get('=.id') + return a except ValueError: pass @@ -330,15 +338,9 @@ class QueueManager(TransmitterManager, metaclass=ABCMeta): return self._exec_cmd(('/queue/simple/enable', '=.id=*' + getattr(q, 'queue_id', ''))) -class IpAddressListObj(IpStruct): - def __init__(self, ip, mk_id): - super(IpAddressListObj, self).__init__(ip) - self.mk_id = str(mk_id).replace('*', '') - - class IpAddressListManager(TransmitterManager, metaclass=ABCMeta): - def add(self, list_name: str, ip: IpStruct): - if not isinstance(ip, IpStruct): + def add(self, list_name: str, ip): + if not issubclass(ip.__class__, _BaseAddress): raise TypeError commands = ( '/ip/firewall/address-list/add', @@ -350,19 +352,19 @@ class IpAddressListManager(TransmitterManager, metaclass=ABCMeta): def remove(self, mk_id): return self._exec_cmd(( '/ip/firewall/address-list/remove', - '=.id=*' + str(mk_id).replace('*', '') + '=.id=%s' % mk_id )) - def remove_range(self, items: Iterable[IpAddressListObj]): - ids = tuple(ip.mk_id for ip in items if isinstance(ip, IpAddressListObj)) + def remove_range(self, items): + ids = tuple(ip_mkid.mkid for ip_mkid in items) if len(ids) > 0: return self._exec_cmd([ '/ip/firewall/address-list/remove', - '=numbers=*%s' % ',*'.join(ids) + '=numbers=%s' % ','.join(ids) ]) - def find(self, ip: IpStruct, list_name: str): - if not isinstance(ip, IpStruct): + def find(self, ip, list_name: str): + if not issubclass(ip.__class__, _BaseAddress): raise TypeError return self._exec_cmd(( '/ip/firewall/address-list/print', 'where', @@ -378,7 +380,7 @@ class IpAddressListManager(TransmitterManager, metaclass=ABCMeta): )) for code, dat in ips: if dat != {}: - yield IpAddressListObj(dat['=address'], dat['=.id']) + yield dat.get('=address'), dat.get('=.id') def disable(self, user: AbonStruct): r = IpAddressListManager.find(self, user.ip, LIST_USERS_ALLOWED) @@ -415,7 +417,7 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager): IpAddressListManager.remove(self, ip_list_entity[0]['=.id']) def add_user(self, user: AbonStruct, *args): - if not isinstance(user.ip, IpStruct): + if not issubclass(user.ip.__class__, _BaseAddress): raise TypeError if user.tariff is None: return @@ -437,7 +439,7 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager): IpAddressListManager.remove(self, firewall_ip_list_obj[0]['=.id']) def update_user(self, user: AbonStruct, *args): - if not isinstance(user.ip, IpStruct): + if not issubclass(user.ip.__class__, _BaseAddress): raise TypeError find_res = IpAddressListManager.find(self, user.ip, LIST_USERS_ALLOWED) @@ -507,14 +509,30 @@ class MikrotikTransmitter(QueueManager, IpAddressListManager): pass def read_users(self) -> Iterable[AbonStruct]: + + class ip_mkid_struct(object): + __slots__ = ('ip', 'mkid') + + def __init__(self, ip, mkid): + self.ip = ip + self.mkid = mkid + + def __eq__(self, other): + if isinstance(other, ip_mkid_struct): + return self.ip == other.ip + return self.ip == str(other) + + def __hash__(self): + return hash(self.ip) + # shapes is ShapeItem - allowed_ips = set(IpAddressListManager.read_ips_iter(self, LIST_USERS_ALLOWED)) - queues = tuple(q for q in QueueManager.read_queue_iter(self) if q.ip in allowed_ips) + all_ips = set(ip_mkid_struct(ip, mkid) for ip, mkid in IpAddressListManager.read_ips_iter(self, LIST_USERS_ALLOWED)) + queues = tuple(q for q in QueueManager.read_queue_iter(self) if str(q.ip) in all_ips) - ips_from_queues = set(q.ip for q in queues) + ips_from_queues = set(str(q.ip) for q in queues) # delete ip addresses that are in firewall/address-list and there are no corresponding in queues - diff = tuple(allowed_ips - ips_from_queues) + diff = tuple(all_ips - ips_from_queues) if len(diff) > 0: IpAddressListManager.remove_range(self, diff) diff --git a/agent/structs.py b/agent/structs.py index 39c7bac..23a545b 100644 --- a/agent/structs.py +++ b/agent/structs.py @@ -1,78 +1,23 @@ -# -*- coding: utf-8 -*- -from abc import ABCMeta, abstractmethod -from struct import pack, unpack, calcsize -from typing import Iterable, Optional -from djing.lib import int2ip, ip2int +from abc import ABCMeta +from ipaddress import ip_address, _BaseAddress +from typing import Iterable class BaseStruct(object, metaclass=ABCMeta): - @abstractmethod - def serialize(self) -> Optional[bytes]: - """make binary""" - - @abstractmethod - def deserialize(self, data: bytes, *args): - """restore from binary""" def __ne__(self, other): return not self == other -class IpStruct(BaseStruct): - def __init__(self, ip): - if type(ip) is int: - self.__ip = ip - else: - self.__ip = ip2int(str(ip)) - - def serialize(self) -> Optional[bytes]: - dt = pack("!I", int(self.__ip)) - return dt - - def deserialize(self, data: bytes, *args): - dt = unpack("!I", data) - self.__ip = int(dt[0]) - return self - - def get_int(self): - return self.__ip - - def __eq__(self, other): - if not isinstance(other, IpStruct): - raise TypeError('Instance must be IpStruct') - return self.__ip == other.__ip - - def __int__(self): - return self.__ip - - def __str__(self): - return int2ip(self.__ip) - - def __hash__(self): - return hash(self.__ip) - - -# Как обслуживается абонент class TariffStruct(BaseStruct): def __init__(self, tariff_id=0, speed_in=None, speed_out=None): self.tid = int(tariff_id) self.speedIn = speed_in or 0 self.speedOut = speed_out or 0 - def serialize(self) -> Optional[bytes]: - dt = pack("!Iff", int(self.tid), float(self.speedIn), float(self.speedOut)) - return dt - - # Да, если все значения нулевые + # Yes, if all variables is zeroed def is_empty(self): - return self.tid == 0 and self.speedIn == 0.001 and self.speedOut == 0.001 - - def deserialize(self, data: bytes, *args): - dt = unpack("!Iff", data) - self.tid = int(dt[0]) - self.speedIn = float(dt[1]) - self.speedOut = float(dt[2]) - return self + return self.tid == 0 and self.speedIn == 0 and self.speedOut == 0 def __eq__(self, other): # не сравниваем id, т.к. тарифы с одинаковыми скоростями для NAS одинаковы @@ -88,35 +33,19 @@ class TariffStruct(BaseStruct): return hash(str(self.speedIn) + str(self.speedOut)) -# Абонент из базы +# Abon from database class AbonStruct(BaseStruct): + __slots__ = ('ip', 'uid', 'tariff', 'is_active', 'queue_id') + def __init__(self, uid=0, ip=None, tariff=None, is_active=True): self.uid = int(uid or 0) - self.ip = IpStruct(ip) + if issubclass(ip.__class__, _BaseAddress): + self.ip = ip + else: + self.ip = ip_address(ip) self.tariff = tariff self.is_active = is_active - def serialize(self) -> Optional[bytes]: - if self.tariff is None: - return - if not isinstance(self.tariff, TariffStruct): - raise TypeError('Instance must be TariffStruct') - if not isinstance(self.ip, IpStruct): - raise TypeError('Instance must be IpStruct') - dt = pack("!LII?", self.uid, int(self.ip), self.tariff.tid, self.is_active) - return dt - - def deserialize(self, data: bytes, tariff=None): - dt = unpack("!LII?", data) - self.uid = dt[0] - self.ip = IpStruct(dt[1]) - if tariff is not None: - if not isinstance(tariff, TariffStruct): - raise TypeError - self.tariff = tariff - self.is_active = dt['3'] - return self - def __eq__(self, other): if not isinstance(other, AbonStruct): raise TypeError @@ -131,24 +60,12 @@ class AbonStruct(BaseStruct): return hash(int(self.ip) + hash(self.tariff)) if self.tariff is not None else 0 -# Правило шейпинга в фаере, или ещё можно сказать услуга абонента на NAS +# Shape rule from NAS(Network Access Server) class ShapeItem(BaseStruct): def __init__(self, abon, sid): self.abon = abon self.sid = sid - def serialize(self) -> Optional[bytes]: - abon_pack = self.abon.serialize() - dt = pack('!L', self.sid) - return dt + abon_pack - - def deserialize(self, data: bytes, *args): - sz = calcsize('!L') - dt = unpack('!L', data[:sz]) - self.sid = dt - self.abon.deserialize(data[sz:]) - return self - def __eq__(self, other): if not isinstance(other, ShapeItem): raise TypeError diff --git a/agent/utils.py b/agent/utils.py deleted file mode 100644 index 88f41f5..0000000 --- a/agent/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -import socket -import struct - - -def ip2int(addr): - try: - return struct.unpack("!I", socket.inet_aton(addr))[0] - except: - return 0 - - -def int2ip(addr): - try: - return socket.inet_ntoa(struct.pack("!I", addr)) - except: - return '' diff --git a/dhcp_lever.py b/dhcp_lever.py index e649194..2b7fc25 100755 --- a/dhcp_lever.py +++ b/dhcp_lever.py @@ -16,7 +16,7 @@ def die(text): ''' obj = { - 'client_ip': ip2int('127.0.0.1'), + 'client_ip': ip_address('127.0.0.1'), 'client_mac': 'aa:bb:cc:dd:ee:ff', 'switch_mac': 'aa:bb:cc:dd:ee:ff', 'switch_port': 3, diff --git a/djing/fields.py b/djing/fields.py index c982beb..b1e5de4 100644 --- a/djing/fields.py +++ b/djing/fields.py @@ -1,11 +1,11 @@ # # I got it on https://github.com/django-macaddress/django-macaddress # +from ipaddress import ip_address +from netaddr import EUI, AddrFormatError from django.core.exceptions import ValidationError from django.db import models -from netaddr import EUI, AddrFormatError -from djing.lib import ip2int, int2ip from .formfields import MACAddressField as MACAddressFormField from . import default_dialect import warnings @@ -121,7 +121,7 @@ class MyGenericIPAddressField(models.GenericIPAddressField): def get_prep_value(self, value): # strIp to Int value = super(MyGenericIPAddressField, self).get_prep_value(value) - return ip2int(value) + return int(ip_address(value)) def to_python(self, value): return value @@ -131,7 +131,8 @@ class MyGenericIPAddressField(models.GenericIPAddressField): @staticmethod def from_db_value(value, expression, connection, context): - return int2ip(value) if value != 0 else None + if value: + return str(ip_address(value)) def int_ip(self): - return ip2int(self) + return int(ip_address(self)) diff --git a/periodic.py b/periodic.py index 0657bb5..8083caf 100755 --- a/periodic.py +++ b/periodic.py @@ -39,9 +39,9 @@ def main(): try: tm = Transmitter() users = Abon.objects.filter(is_active=True).exclude(current_tariff=None) - tm.sync_nas(users) - except NasNetworkError as e: - print('NetworkTrouble:', e) + tm.sync_nas(users.iterator()) + except NasNetworkError as er: + print('NetworkTrouble:', er) # manage periodic pays ppays = PeriodicPayForId.objects.filter(next_pay__lt=now) \