From cd1b3da65075efc5608bff0532dd81295301697b Mon Sep 17 00:00:00 2001 From: bashmak Date: Sat, 22 Apr 2017 15:36:23 +0300 Subject: [PATCH 01/97] =?UTF-8?q?=D0=AD=D1=82=D0=B8=20=D1=8E=D0=BD=D0=B8?= =?UTF-8?q?=D1=82=D1=8B=20=D0=B1=D1=83=D0=B4=D1=83=D1=82=20=D0=BE=D1=82?= =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D1=8F=D1=82=D1=8C=20=D0=BD=D0=B0?= =?UTF-8?q?=D0=BA=D0=BE=D0=BF=D0=B8=D0=B2=D1=88=D1=83=D1=8E=D1=81=D1=8F=20?= =?UTF-8?q?=D0=B8=D0=BD=D1=84=D1=83=20=D1=81=20netflow=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80=20=D0=B1=D0=B0=D0=B7=20?= =?UTF-8?q?=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- systemd_units/djing_rotate.service | 12 ++++++++++++ systemd_units/djing_rotate.timer | 11 +++++++++++ 2 files changed, 23 insertions(+) create mode 100644 systemd_units/djing_rotate.service create mode 100644 systemd_units/djing_rotate.timer diff --git a/systemd_units/djing_rotate.service b/systemd_units/djing_rotate.service new file mode 100644 index 0000000..4e1678b --- /dev/null +++ b/systemd_units/djing_rotate.service @@ -0,0 +1,12 @@ +[Unit] +Description=A job for rotate djing netflow data + +[Service] +Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/bin" +Type=oneshot +ExecStart=/bin/bash -c "kill -HUP `cat /run/flow.pid.6343`" +User=root +Group=root + +[Install] +WantedBy=multi-user.target diff --git a/systemd_units/djing_rotate.timer b/systemd_units/djing_rotate.timer new file mode 100644 index 0000000..c0be202 --- /dev/null +++ b/systemd_units/djing_rotate.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run every one minute rotate flows for djing + +[Timer] +OnCalendar=*-*-* *:*:59 +Persistent=true +RandomizedDelaySec=5 +Unit=djing_rotate.service + +[Install] +WantedBy=timers.target From 0d361c7507ee4da76c8127b38203d26d7d9a25cd Mon Sep 17 00:00:00 2001 From: bashmak Date: Sat, 22 Apr 2017 15:36:48 +0300 Subject: [PATCH 02/97] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D0=BB=20=D1=81=D0=BA=D1=80=D0=B8=D0=BF=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/netflow/netflow_handler.sh | 13 +++++++++---- agent/netflow/start_netflow.sh | 8 +++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/agent/netflow/netflow_handler.sh b/agent/netflow/netflow_handler.sh index db275fa..3cc703a 100755 --- a/agent/netflow/netflow_handler.sh +++ b/agent/netflow/netflow_handler.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash FNAME="$1" @@ -9,9 +9,14 @@ fi CUR_DIR=`dirname $0` -DUMP_FILE="$CUR_DIR/$FNAME" +DUMP_FILE="/tmp/djing_flow/$FNAME" PATH=/usr/local/sbin:/usr/local/bin:/usr/bin +TMP_DUMP=/tmp/djing_flow/djing_flow_dump.tmp +cd $CUR_DIR +mkdir -p /tmp/djing_flow +mv $DUMP_FILE $TMP_DUMP -flow-print -f3 < ${DUMP_FILE} | ${CUR_DIR}/to_mysql \ -| mysql -uroot -p jungagent --password=ps +./djing_flow < $TMP_DUMP | /usr/bin/mysql -uDB_USER -h -p djingdb --password=PASSWORD + +rm $TMP_DUMP diff --git a/agent/netflow/start_netflow.sh b/agent/netflow/start_netflow.sh index d7fafba..7e2dd9a 100755 --- a/agent/netflow/start_netflow.sh +++ b/agent/netflow/start_netflow.sh @@ -1,7 +1,5 @@ -#!/bin/sh +#!/usr/bin/env bash -PATH=/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/lib/jvm/default/bin:/home/dn/bin +PATH=/usr/local/sbin:/usr/local/bin:/usr/bin -flow-capture -R /home/dn/bin/netflow_handler.py -w /var/db/flows -n1 -N0 0.0.0.0/0.0.0.0/8888 - -softflowd -v 5 -i wlp3s0 -n 127.0.0.1:8888 +flow-capture -R /srv/http/djing/agent/netflow/netflow_handler.sh -p /run/flow.pid -w /tmp/djing_flow -n1 -N0 0/0/6343 From 4c7b73422d5792cae9ba03e5fcbe9c5e1f7cd07d Mon Sep 17 00:00:00 2001 From: bashmak Date: Sat, 22 Apr 2017 15:37:06 +0300 Subject: [PATCH 03/97] =?UTF-8?q?=D1=83=D0=B4=D0=B0=D0=BB=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=BB=D0=B8=D1=88=D0=BD=D0=B5=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/netflow/to_mysql.c | 125 -------------------------------------- agent/netflow/to_mysql.py | 66 -------------------- 2 files changed, 191 deletions(-) delete mode 100644 agent/netflow/to_mysql.c delete mode 100755 agent/netflow/to_mysql.py diff --git a/agent/netflow/to_mysql.c b/agent/netflow/to_mysql.c deleted file mode 100644 index e57f41d..0000000 --- a/agent/netflow/to_mysql.c +++ /dev/null @@ -1,125 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include - - -#define FLOW_COLS 8 -#define uint unsigned int - - -uint32_t ip2int(const char* ip) -{ - uint32_t res = 0; - inet_pton(AF_INET, ip, &res); - return htonl(res); -} - - -uint str_split(char* str, const char* delimiter, char** pInChunks) -{ - char* dat = strtok(str, " "); - register uint n=0; - while(dat) - { - pInChunks[n++] = dat; - dat = strtok(NULL, " "); - } - return n; -} - - -void curtime(char* pInStrTime, const uint maxlen) -{ - time_t rawtime; - time( &rawtime ); - strftime(pInStrTime, maxlen, "flowstat_%d%m%Y", localtime( &rawtime )); -} - - -void convert(char* query, char* pInRes) -{ - char* chunks[FLOW_COLS] = {NULL}; - - int chunk_count = str_split(query, " ", chunks); - - if(chunk_count < 7) - { - printf("Too short input line\n"); - exit(1); - } - - uint32_t src_ip = ip2int(chunks[0]); - uint32_t dst_ip = ip2int(chunks[1]); - uint proto = atoi(chunks[2]); - uint16_t src_port = ip2int(chunks[3]); - uint16_t dst_port = ip2int(chunks[4]); - uint octets = atoi(chunks[5]); - uint packets = atoi(chunks[6]); - - sprintf(pInRes, ",(%u,%u,%u,%u,%u,%u,%u)\0", - src_ip, dst_ip, proto, src_port, dst_port, octets, packets); -} - - -int main() -{ - char buf_result_convert[0xff] = {0}; - FILE* f = stdin; - char* input_line = malloc(0xff); - size_t input_line_len = 0; - ssize_t read_len = 0; - char table_name[19] = {0}; - - curtime(table_name, 19); - - printf("CREATE TABLE IF NOT EXISTS %s (\n", table_name); - printf("`id` int(10) AUTO_INCREMENT NOT NULL,\n"); - printf("`src_ip` INT(10) UNSIGNED NOT NULL,\n"); - printf("`dst_ip` INT(10) UNSIGNED NOT NULL,\n"); - printf("`proto` smallint(2) unsigned NOT NULL DEFAULT 0,\n"); - printf("`src_port` smallint(5) unsigned NOT NULL DEFAULT 0,\n"); - printf("`dst_port` smallint(5) unsigned NOT NULL DEFAULT 0,\n"); - printf("`octets` INT unsigned NOT NULL DEFAULT 0,\n"); - printf("`packets` INT unsigned NOT NULL DEFAULT 0,\n"); - printf("PRIMARY KEY (`id`)\n"); - printf(") ENGINE=MyISAM DEFAULT CHARSET=utf8;\n"); - - char ins_sql[0xff] = {0}; - sprintf(ins_sql, "INSERT INTO %s(`src_ip`, `dst_ip`, `proto`, `src_port`, `dst_port`, `octets`, `packets`) VALUES", table_name); - - // always none - read_len = getline(&input_line, &input_line_len, f); - - while(true) - { - register uint n=0xfff; - read_len = getline(&input_line, &input_line_len, f); - if(read_len <= 0) - break; - convert(input_line, buf_result_convert); - - printf("%s\n", ins_sql); - - // without first comma - printf("%s\n", buf_result_convert+1); - - while(n>0) - { - read_len = getline(&input_line, &input_line_len, f); - if(read_len <= 0) - break; - convert(input_line, buf_result_convert); - printf("%s\n", buf_result_convert); - n--; - } - putc(';', stdout); - } - - free(input_line); - return 0; -} diff --git a/agent/netflow/to_mysql.py b/agent/netflow/to_mysql.py deleted file mode 100755 index bc7a346..0000000 --- a/agent/netflow/to_mysql.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/env python3 -import sys -import socket -import struct -from re import sub -from django.utils import timezone - - -def ip2int(strip): - return struct.unpack("!I", socket.inet_aton(strip))[0] - - -def convert(query): - dat = sub(r'\s+', ' ', query.strip('\n')).split(' ') - - if len(dat) == 1: - return - - src_ip = ip2int(dat[0]) - dst_ip = ip2int(dat[1]) - proto = int(dat[2]) - src_port = int(dat[3]) - dst_port = int(dat[4]) - octets = int(dat[5]) - packets = int(dat[6]) - - sql = ",(%d,%d,%d,%d,%d,%d,%d)" % ( - src_ip, dst_ip, proto, src_port, dst_port, octets, packets - ) - return sql - - -if __name__ == '__main__': - f = sys.stdin - table_name = "flowstat_%s" % timezone.now().strftime("%d%m%Y") - print(("CREATE TABLE IF NOT EXISTS %s (" % table_name)) - print("`id` int(10) AUTO_INCREMENT NOT NULL,") - print("`src_ip` INT(10) UNSIGNED NOT NULL,") - print("`dst_ip` INT(10) UNSIGNED NOT NULL,") - print("`proto` smallint(2) unsigned NOT NULL DEFAULT 0,") - print("`src_port` smallint(5) unsigned NOT NULL DEFAULT 0,") - print("`dst_port` smallint(5) unsigned NOT NULL DEFAULT 0,") - print("`octets` INT unsigned NOT NULL DEFAULT 0,") - print("`packets` INT unsigned NOT NULL DEFAULT 0,") - print("PRIMARY KEY (`id`)") - print(") ENGINE=MyISAM DEFAULT CHARSET=utf8;") - ins_sql = r"INSERT INTO %s(`src_ip`, `dst_ip`, `proto`, `src_port`, `dst_port`, `octets`, `packets`) VALUES" % table_name - - # always none - f.readline() - - while True: - n = 0xfff - rs = convert(f.readline()) - if not rs: exit() - # without first comma - print(ins_sql) - print((rs[1:])) - while n > 0: - rs = convert(f.readline()) - if not rs: exit() - print(rs) - n -= 1 - print(';') - - f.close() From fb699053079a2293e29d4e830322d4e126780959 Mon Sep 17 00:00:00 2001 From: http Date: Wed, 3 May 2017 13:20:28 +0300 Subject: [PATCH 04/97] =?UTF-8?q?=D1=83=D0=B1=D1=80=D0=B0=D0=BB=20=D0=BB?= =?UTF-8?q?=D0=B8=D1=88=D0=BD=D1=8E=D1=8E=20=D1=81=D1=82=D0=B0=D1=82=D0=B8?= =?UTF-8?q?=D0=BA=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/404.html | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/404.html b/templates/404.html index 6f613a5..0eb1ccd 100644 --- a/templates/404.html +++ b/templates/404.html @@ -8,7 +8,6 @@ -