16,741 bytes added,
02:24, 14 December 2016 = Load Balancing =
== Description ==
Load balancing between multiple ISP's in case if there is no BGP/OSPF/RIP can be achieved by using following software:
# FreeBSD as base system;
# IPFW firewall;
# Optional MPD5 as PPPoE client.
== How it all works? ==
Current config:
FIB 0: ADSL PPPOE
FIB 1: ADSL PPPOE
FIB 2: DHCP client
* DHCP client on FIB starts via RC.D script (see below);
* watchdog script periodically check's FIB 2 connection and reload rules on status change(see below);
* PPPOE connections managed by MPD5 daemon;;
* When connection established "/usr/local/etc/mpd5/up-script_fib_X.sh" script will be executed in order to reload firewall rules;
* If one or more PPPoE connection lost MPD5 will DOWN adsl interface and execute "/usr/local/etc/mpd5/down-script_fib_X.sh" script in order to reload firewall rules;
* arpalert daemon waits for new MAC's to appear and launches script which will put IP's to IPFW tables. This is actual load balancing.
== Why configuration is so complicated? ==
If we use simple 'prob' and 'setfib' rules of IPFW some services may work incorrect(FTP, PPTP, HTTPS and etc.).
== Configuration ==
=== IPFW firewall ===
Kernel options:
<pre>
options LIBALIAS
options ROUTETABLES=5
options IPFIREWALL
options IPDIVERT
options DUMMYNET
options IPFIREWALL_NAT
options IPFIREWALL_FORWARD
options IPFIREWALL_DEFAULT_TO_ACCEPT
options IPFIREWALL_VERBOSE
options IPFIREWALL_VERBOSE_LIMIT=50
</pre>
/etc/sysctl.conf:
<pre>
net.inet.ip.fw.autoinc_step=5
net.inet.ip.fw.one_pass=0
net.inet.ip.fw.verbose=1
net.inet.ip.fw.dyn_short_lifetime=25
net.inet.ip.fw.verbose_limit=1000000
</pre>
/etc/firewall:
<pre>
#!/bin/sh
DELAY=`/usr/bin/jot -r 1 0 9`
# Delay
echo "${DELAY} seconds delay..."
sleep ${DELAY}
PIDS=`pgrep -f "/bin/sh /etc/firewall" | wc -l`
#Check if copy of process is already running:
if [ ${PIDS} -gt 1 ]; then
echo "Another copy is already running."
exit 1
fi
#Flush out list before we begin.
ipfw -q -f flush
ipfw -q -f nat flush
ipfw -q pipe flush
ipfw -q queue flush
################################################################################
#Set rules command prefix
cmd="ipfw -q"
pif0="adsl1"
pif0ip=`ifconfig $pif0 inet | grep inet | awk '{print $2}'`
pif0gw=`ifconfig $pif0 inet | grep inet | awk '{print $4}'`
pif1="adsl2"
pif1ip=`ifconfig $pif1 inet | grep inet | awk '{print $2}'`
pif1gw=`ifconfig $pif1 inet | grep inet | awk '{print $4}'`
pif2="vlan200"
pif2ip="10.61.168.231"
pif2gw="10.61.168.1"
if [ ! -z "$pif0ip" ]; then
pif0status="UP"
else
pif0status="DOWN"
fi
if [ ! -z "$pif1ip" ]; then
pif1status="UP"
else
pif1status="DOWN"
fi
if [ -f /vlan200.status ]; then
pif2status=`cat /vlan200.status`
else
pif2status="DOWN"
fi
# Override
#pif0status="DOWN"
#pif1status="DOWN"
#pif2status="DOWN"
pifs="adsl*"
lif="lan0"
zuznet="192.168.1.0/24"
== How it all works? ==
Current config:
FIB 0: ADSL PPPOE
FIB 1: ADSL PPPOE
FIB 2: DHCP client
* DHCP client on FIB starts via RC.D script (see below);
* watchdog script periodically check's FIB 2 connection and reload rules on status change(see below);
* PPPOE connections managed by MPD5 daemon;;
* When connection established "/usr/local/etc/mpd5/up-script_fib_X.sh" script will be executed in order to reload firewall rules;
* If one or more PPPoE connection lost MPD5 will DOWN adsl interface and execute "/usr/local/etc/mpd5/down-script_fib_X.sh" script in order to reload firewall rules;
* arpalert daemon waits for new MAC's to appear and launches script which will put IP's to IPFW tables. This is actual load balancing.
################################################################################
$cmd add 100 allow ip from any to any via lo0
$cmd add deny ip from any to 127.0.0.0/8
$cmd add deny ip from 127.0.0.0/8 to any
$cmd add 1100 skipto 2040 ip from any to any out xmit $lif tagged 101 keep-state
$cmd add skipto 2080 ip from any to any out xmit $lif tagged 102 keep-state
$cmd add skipto 2100 ip from any to any out xmit $lif tagged 120 keep-state
$cmd add 1500 deny gre from 192.168.1.XXX to any // This rule will drop first GRE packet from PPTP SERVER when VPN initiates
# Next few IF's will keep firewall rules consistent in cases if ISP connections will go down
if [ $pif0status == "UP" ] && [ $pif1status == "UP" ]; then
$cmd add 2000 skipto 2040 ip from table\(101\) to any in recv $lif // Smart Load balancing using arpalert
$cmd add skipto 2080 ip from table\(102\) to any in recv $lif // Smart Load balancing using arpalert
fi
if [ $pif2status == "UP" ]; then
$cmd add skipto 2100 ip from table\(120\) to any in recv $lif // Smart Load balancing using arpalert
fi
if [ $pif0status == "UP" ]; then
$cmd add 2040 setfib 0 ip from any to any via $lif keep-state
$cmd add 2050 allow tag 101 ip from any to any via $lif
fi
if [ $pif1status == "UP" ]; then
$cmd add 2080 setfib 1 ip from any to any via $lif keep-state
$cmd add 2090 allow tag 102 ip from any to any via $lif
fi
if [ $pif2status == "UP" ]; then
$cmd add 2100 setfib 2 ip from any to any via $lif keep-state
$cmd add 2110 allow tag 120 ip from any to any via $lif
fi
################################################################################
$cmd add 3050 deny ip from any to 192.168.0.0/16 in recv $pifs
$cmd add deny ip from 192.168.0.0/16 to any in recv $pifs
$cmd add deny ip from any to 172.16.0.0/12 in recv $pifs
$cmd add deny ip from 172.16.0.0/12 to any in recv $pifs
$cmd add deny ip from any to 10.0.0.0/8 in recv $pifs
$cmd add deny ip from 10.0.0.0/8 to any in recv $pifs
$cmd add deny ip from any to 169.254.0.0/16 in recv $pifs
$cmd add deny ip from 169.254.0.0/16 to any in recv $pifs
################################################################################
if [ $pif1status == "UP" ]; then
$cmd add 11000 fwd $pif1gw all from $pif1ip to any via $pif0 // Send reply to necessary interface
fi
if [ $pif2status == "UP" ]; then
$cmd add 11100 fwd $pif2gw all from $pif2ip to any via $pif0 // Send reply to necessary interface
fi
################################################################################
$cmd add 13000 skipto 31000 log ip from any to $pif0ip 22 in recv $pif0 // sshd
$cmd add skipto 31000 log ip from any to $pif1ip 22 in recv $pif1 // sshd
$cmd add skipto 31000 log ip from any to $pif2ip 22 in recv $pif2 // sshd
$cmd add skipto 31000 log ip from any to $pif0ip 1723 in recv $pif0 // PPTP server
$cmd add skipto 31000 log ip from any to $pif1ip 1723 in recv $pif1 // PPTP server
$cmd add skipto 31000 log ip from any to $pif2ip 1723 in recv $pif2 // PPTP server
$cmd add skipto 31000 gre from any to $pif0ip in recv $pif0 // PPTP server
$cmd add skipto 31000 gre from any to $pif1ip in recv $pif1 // PPTP server
$cmd add skipto 31000 gre from any to $pif2ip in recv $pif2 // PPTP server
################################################################################
if [ $pif0status == "UP" ]; then
$cmd add 20000 deny log ip from any to any in via $pif0 setup // Reject and Log all setup of incoming connections from the outside
$cmd add deny log ip from any to any in via $pif1 setup // Reject and Log all setup of incoming connections from the outside
$cmd add deny log ip from any to any in via $pif2 setup // Reject and Log all setup of incoming connections from the outside
fi
################################################################################
if [ $pif0status == "UP" ]; then
$cmd nat 101 config if $pif0 same_ports reset \
redirect_port tcp 192.168.1.XXX:1723 $pif0ip:1723 \
redirect_proto gre 192.168.1.XXX $pif0ip
fi
if [ $pif1status == "UP" ]; then
$cmd nat 102 config if $pif1 same_ports reset \
redirect_port tcp 192.168.1.XXX:1723 $pif1ip:1723 \
redirect_proto gre 192.168.1.XXX $pif1ip
fi
if [ $pif2status == "UP" ]; then
$cmd nat 120 config if $pif2 same_ports reset \
redirect_port tcp 192.168.1.XXX:1723 $pif2ip:1723 \
redirect_proto gre 192.168.1.XXX $pif2ip
fi
################################################################################
# NAT
if [ $pif0status == "UP" ]; then
$cmd add 31000 nat 101 ip from any to any via $pif0 // $pif0 nat
$cmd add skipto 35000 tag 101 ip from any to any in recv $pif0
fi
if [ $pif1status == "UP" ]; then
$cmd add 31500 nat 102 ip from any to any via $pif1 // $pif1 nat
$cmd add skipto 35000 tag 102 ip from any to any in recv $pif1
fi
if [ $pif2status == "UP" ]; then
$cmd add 32500 nat 120 ip from any to any via $pif2 // $pif2 nat
$cmd add skipto 35000 tag 120 ip from any to any in recv $pif2
fi
################################################################################
$cmd add 35000 allow tcp from any to any established // Allow TCP through if setup succeeded
$cmd add 50000 allow all from any to any
$cmd add 65534 deny all from any to any
################################################################################
exit 0
</pre>
=== MPD5 as PPPoE client ===
Default route will be set using up-script_fib_0.sh. This is workaround to avoid identical gateways in case of using two lines from on ISP. Use different fib's for each ISP.
/usr/local/etc/mpd5/mpd.conf:
<pre>
startup:
# Name password password_user
set user myuser mypass admin
# console on localhost
set console self 127.0.0.1 5005
set console open
set web self 0.0.0.0 5006
set web open
# default settings
default:
load pppoe_client_101
load pppoe_client_102
pppoe_client_101:
create bundle static B1
set iface name adsl1
set iface enable tcpmssfix
set iface up-script /usr/local/etc/mpd5/up-script_fib_0.sh
set iface down-script /usr/local/etc/mpd5/down-script_fib_0.sh
# Default route is added via up-script
# set iface route default
set ipcp ranges 0.0.0.0/0 0.0.0.0/0
create link static L1 pppoe
set link action bundle B1
set auth authname PPPOE_USERNAME
set auth password PPPOE_PASSWORD
set link max-redial 0
set link mtu 1492
set link mru 1492
set link keep-alive 10 60
# interface to PPPoE
set pppoe iface vlan101
# set pppoe iface pif0
set pppoe service ""
# start connection
open
pppoe_client_102:
create bundle static B2
set iface name adsl2
set iface enable tcpmssfix
set iface up-script /usr/local/etc/mpd5/up-script_fib_1.sh
set iface down-script /usr/local/etc/mpd5/down-script_fib_1.sh
# Default route is added via up-script
# set iface route default
set ipcp ranges 0.0.0.0/0 0.0.0.0/0
create link static L2 pppoe
set link action bundle B2
set auth authname PPPOE_USERNAME
set auth password PPPOE_PASSWORD
set link max-redial 0
set link mtu 1492
set link mru 1492
set link keep-alive 10 60
# interface to PPPoE
set pppoe iface vlan102
# set pppoe iface pif0
set pppoe service ""
# start connection
open
</pre>
/usr/local/etc/mpd5/up-script_fib_0.sh:
<pre>
#!/bin/sh
echo "`date` UP $@" >> /tmp/firewall.sh.log
# Manually adding default route to avoid error with identical destination gateways
EXT_IF_IP=$4
setfib 0 route add default -interface $1
/etc/firewall &
exit 0
</pre>
/usr/local/etc/mpd5/down-script_fib_0.sh:
<pre>
#!/bin/sh
echo "`date` DOWN $@" >> /tmp/firewall.sh.log
# Manually adding default route to avoid error with identical destination gateways
setfib 0 route del default -interface $1
/etc/firewall &
exit 0
</pre>
== setfib + dhclient ==
/etc/rc.conf:
# setfib 2 dhclient vlan200
vlan200_enable="YES"
/usr/local/etc/rc.d/vlan200:
<pre>
#!/bin/sh
# PROVIDE: sumtel-watchdog
# REQUIRE: DAEMON
# KEYWORD: shutdown
#
# Add the following lines to /etc/rc.conf.local or /etc/rc.conf to
# enable vlan200:
#
# vlan200_enable (bool): Set to NO by default. Set it to YES to
# enable vlan200.
#
. /etc/rc.subr
name="vlan200"
rcvar="vlan200_enable"
pidfile="/var/run/${name}.pid"
start_cmd=${name}_start
stop_cmd=${name}_stop
vlan200_start() {
/usr/sbin/setfib 2 /sbin/dhclient vlan200
}
vlan200_stop() {
/bin/pgrep -f "dhclient: vlan200" | /usr/bin/xargs kill
}
load_rc_config $name
: ${vlan200_enable="NO"}
run_rc_command "$1"
</pre>
== watchdog script ==
Following script checks internet connectivity and reloads IPFW rules if status has changed.
/etc/rc.conf:
sumtelwatchdog_enable="YES"
usr/local/etc/rc.d/sumtelwatchdog:
<pre>
#!/bin/sh
# PROVIDE: sumtel-watchdog
# REQUIRE: DAEMON
# KEYWORD: shutdown
#
# Add the following lines to /etc/rc.conf.local or /etc/rc.conf to
# enable sumtelwatchdog:
#
# sumtelwatchdog_enable (bool): Set to NO by default. Set it to YES to
# enable sumtelwatchdog.
#
. /etc/rc.subr
name="sumtelwatchdog"
rcvar="sumtelwatchdog_enable"
pidfile="/var/run/${name}.pid"
start_cmd=${name}_start
stop_cmd=${name}_stop
sumtelwatchdog_start() {
/usr/sbin/daemon -f -p /var/run/${name}.pid /root/${name}.sh
}
sumtelwatchdog_stop() {
kill `cat /var/run/${name}.pid`
rm -f /var/run/${name}.pid
}
load_rc_config $name
: ${sumtelwatchdog_enable="NO"}
run_rc_command "$1"
</pre>
/root/sumtelwatchdog.sh:
<pre>
#!/bin/sh
# Variables
HOST1="8.8.8.8"
HOST2="172.29.61.11"
ADDRESS="ya.ru"
FIB="2"
FILE="/vlan200.status"
# Infinite loop
# If all three checks will fail then status will be "DOWN"
while :
do
setfib ${FIB} dig @${HOST1} ${ADDRESS} 1>/dev/null 2>/dev/null
STATUS1=`echo $?`
sleep 20
setfib ${FIB} dig @${HOST2} ${ADDRESS} 1>/dev/null 2>/dev/null
STATUS2=`echo $?`
sleep 20
if [ ${STATUS1} -ne 0 ] && [ ${STATUS2} -ne 0 ];
then
STATUS=`cat ${FILE}`
if [ ${STATUS} != "DOWN" ]; then
echo "DOWN" > ${FILE}
/etc/firewall
fi
else
STATUS=`cat ${FILE}`
if [ ${STATUS} != "UP" ]; then
echo "UP" > ${FILE}
/etc/firewall
fi
fi
done
exit 0
</pre>
== arpalert ==
Purpose of this following configuration is to add LAN IP's to IPFW tables.
/usr/local/etc/rc.d/arpalert:
<pre>
#!/bin/sh
# PROVIDE: arpalert
# REQUIRE: DAEMON
# KEYWORD: shutdown
#
# Add the following lines to /etc/rc.conf.local or /etc/rc.conf to
# enable arpalert:
#
# arpalert_enable (bool): Set to NO by default. Set it to YES to
# enable arpalert.
#
. /etc/rc.subr
name="arpalert"
rcvar=arpalert_enable
command="/usr/local/sbin/${name}"
required_files="/usr/local/etc/arpalert/${name}.conf"
load_rc_config $name
: ${arpalert_enable="NO"}
run_rc_command "$1"
</pre>
/usr/local/etc/arpalert/arpalert.conf (only changed values):
<pre>
user = root
interface = lan0
action on detect = "/usr/local/etc/arpalert/script.pl"
mac timeout = 1200
</pre>
/usr/local/etc/arpalert/script.pl:
<pre>
#!/usr/bin/perl
use strict;
use warnings;
#Check if copy of process is already running:
my $pids=`pgrep -f "/usr/bin/perl /usr/local/etc/arpalert/script.pl" | wc -l`;
if ( $pids > 1 ) {
print "Another copy is already running.\n";
exit 1;
}
# Variables and initial data
my $interface="lan0";
my $prob;
my %arp_hash;
my %tables;
my @arp_out=`/usr/sbin/arp -a -i $interface`;
# Writing timestamp
`date >> /tmp/table_balance.log`;
# Output to file
open (LOGFILE, '>> /tmp/table_balance.log');
# Storing IP from 'arp' command in %arp_hash
foreach (@arp_out) {
$_ =~ m|\((\S+?)\)|; # Match string
$arp_hash{$1}='';
}
foreach my $table (101, 102, 120) {
# Storing IP from 'ipfw table' command in %tables
foreach (`/sbin/ipfw table $table list`) {
$_ =~ m|^(\S+?)\/|;
$tables{"$table"}->{"$1"} = '1';
}
# Checking existance of IP in %arp_hash
foreach my $ip (keys %{$tables{$table}}) {
if (exists $arp_hash{$ip}) {
$arp_hash{$ip} = "$table";
} else {
print LOGFILE "Removing IP:$ip from table $table\n";
print `/sbin/ipfw table $table delete $ip`;
delete $tables{$table}->{$ip};
}
}
}
# Adding new IP addreses to tables
foreach my $ip (keys %arp_hash) {
if ($arp_hash{$ip} eq '') {
$prob = rand(100);
if ( $prob > 66 ) {
print LOGFILE "Adding IP:$ip to the table 101. prob=$prob\n";
print `/sbin/ipfw table 101 add $ip`;
} elsif ( $prob < 33 ) {
print LOGFILE "Adding IP:$ip to the table 102. prob=$prob\n";
print `/sbin/ipfw table 102 add $ip`;
} else {
print LOGFILE "Adding IP:$ip to the table 120. prob=$prob\n";
print `/sbin/ipfw table 120 add $ip`;
}
}
}
# Debug info
#foreach my $ip (keys %arp_hash) {
# print "key = '$ip' value='$arp_hash{$ip}' \n" if ($arp_hash{$ip} ne '');
#}
print "Done.\n";
# Closing file
close (LOGFILE);
exit 0
</pre>