0.
#udev rules
SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}!="", ATTR{phys_port_name}!="", \
IMPORT{program}="/etc/infiniband/vf-net-link-name.sh $attr{phys_switch_id} $attr{phys_port_name}" \
NAME="$env{NAME}", RUN+="/sbin/mlnx_net_rules $env{NAME}"
SUBSYSTEM=="net", ACTION=="add", RUN+="/sbin/mlnx_net_rules $env{NAME}"
SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}=="e8a8470003da341c", \
ATTR{phys_port_name}!="", NAME="eth2_$attr{phys_port_name}"
SUBSYSTEM=="net", ACTION=="add", ATTR{address}="1c:34:da:47:a8:e8", NAME="eth2"
1.
#enable sriov
echo 2 > /sys/class/net/eth3/device/sriov_numvfs
lspci | grep -i mellanox
2.
#unbind VFs
echo 0000:00:08.2 > /sys/bus/pci/drivers/mlx5_core/unbind
echo 0000:00:08.3 > /sys/bus/pci/drivers/mlx5_core/unbind
3.
#enable switchdev
devlink dev eswitch set pci/0000:00:08.0 mode switchdev
4.
#set VFs mac address
ip link set dev eth3 vf 0 mac e4:11:22:33:44:50
ip link set dev eth3 vf 1 mac e4:11:22:33:44:51
5.
#To change the encapsulation value make sure all VFs are unbound and no rules are set
echo basic > /sys/class/net/eth2/compat/devlink/encap
6.
#bind VFs
echo 0000:00:08.2 > /sys/bus/pci/drivers/mlx5_core/bind
echo 0000:00:08.3 > /sys/bus/pci/drivers/mlx5_core/bind
7.
#configure openvswitch
systemctl start openvswitch
ovs-vsctl add-br ovs-sriov
ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
ovs-vsctl set Open_vSwitch . other_config:max-idle=300000
systemctl restart openvswitch
8.
#add representor to OVS
ovs-vsctl add-port ovs-sriov eth3
ovs-vsctl add-port ovs-sriov eth5
ovs-vsctl add-port ovs-sriov eth6
ip link set eth3 up
ip link set eth5 up
ip link set eth6 up
On the 2 hosts:
echo basic > /sys/class/net/eth4/compat/devlink/encap
On the 1st host:
ip addr add 1.1.1.77/24 dev eth4
ovs-vsctl add-port ovs-sriov vxlan0 -- set interface vxlan0 type=vxlan options:local_ip=1.1.1.77 options:remote_ip=1.1.1.88 options:key=98
On the 2nd host:
ip addr add 1.1.1.88/24 dev eth4
ovs-vsctl add-port ovs-sriov vxlan0 -- set interface vxlan0 type=vxlan options:local_ip=1.1.1.88 options:remote_ip=1.1.1.77 options:key=98
ovs-vsctl -- set Bridge ovs-sriov mirrors=@m \
-- --id=@vxlan0 get Port vxlan0 \
-- --id=@eth5 get Port eth5 \
-- --id=@m create Mirror name=mirror_test select-dst-port=@vxlan0 select-src-port=@vxlan0 output-port=@eth5
ovs-vsctl list Mirror