Skip to content

Networking

Create bridge on main interface

All nodes on which the configuration is executed are restarted.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
oc apply -f - <<EOF
apiVersion: nmstate.io/v1alpha1
kind: NodeNetworkConfigurationPolicy
metadata:
  name: br1-ens3-policy-workers
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  desiredState:
    interfaces:
      - name: br1
        description: Linux bridge with ens3 as a port
        type: linux-bridge
        state: up
        ipv4:
          enabled: true
          dhcp: true
        bridge:
          options:
            stp:
              enabled: false
          port:
            - name: ens3
EOF

Create Network Attachment Definition

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
cat << EOF | oc apply -f -
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: tuning-bridge-fixed
  annotations:
    k8s.v1.cni.cncf.io/resourceName: bridge.network.kubevirt.io/br1
spec:
  config: '{
    "cniVersion": "0.3.1",
    "name": "br1",
    "plugins": [
      {
        "type": "cnv-bridge",
        "bridge": "br1"
      },
      {
        "type": "cnv-tuning"
      }
    ]
  }'
EOF

Debugging purpose

Create br1 via nmcli

1
2
3
4
5
6
7
8
nmcli con show --active
nmcli con add type bridge ifname br1 con-name br1
nmcli con add type bridge-slave ifname ens3 master br1
nmcli con modify br1 bridge.stp no
nmcli con down 'Wired connection 1'
nmcli con up br1
nmcli con mod br1 connection.autoconnect yes
nmcli con mod 'Wired connection 1' connection.autoconnect no

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
[root@compute-0 ~]# nmcli con show
NAME                UUID                                  TYPE      DEVICE
br1                 2ae82518-2ff3-4d49-b95c-fc8fbf029d48  bridge    br1
bridge-slave-ens3   faac459f-ce51-4ce9-8616-ea9d23aff675  ethernet  ens3
Wired connection 1  e158d160-1743-3b00-9f67-258849993562  ethernet  --
[root@compute-0 ~]# nmcli -f bridge con show br1
bridge.mac-address:                     --
bridge.stp:                             no
bridge.priority:                        32768
bridge.forward-delay:                   15
bridge.hello-time:                      2
bridge.max-age:                         20
bridge.ageing-time:                     300
bridge.group-forward-mask:              0
bridge.multicast-snooping:              yes
bridge.vlan-filtering:                  no
bridge.vlan-default-pvid:               1
bridge.vlans:                           --
[root@compute-0 ~]# ip a show dev ens3
2: ens3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master br1 state UP group default qlen 1000
    link/ether 52:54:00:a8:34:0d brd ff:ff:ff:ff:ff:ff
[root@compute-0 ~]# ip a show dev br1
17: br1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 52:54:00:a8:34:0d brd ff:ff:ff:ff:ff:ff
    inet 192.168.52.13/24 brd 192.168.52.255 scope global dynamic noprefixroute br1
       valid_lft 3523sec preferred_lft 3523sec
    inet6 fe80::70f0:71c5:53ea:71ee/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

Connection problem with kubevirt.io/allow-pod-bridge-network-live-migration after live migration

HCP Cluster sendling

1
2
3
4
oc get nodes
NAME                      STATUS   ROLES    AGE   VERSION
sendling-d0c14274-6nbvl   Ready    worker   11d   v1.27.8+4fab27b
sendling-d0c14274-sz7rb   Ready    worker   11d   v1.27.8+4fab27b
Ping check details node/sendling-d0c14274-6nbvl 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
oc debug node/sendling-d0c14274-6nbvl
Starting pod/sendling-d0c14274-6nbvl-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.128.8.133
If you don't see a command prompt, try pressing enter.
sh-4.4# ping www.google.de
PING www.google.de (172.253.62.94) 56(84) bytes of data.
64 bytes from bc-in-f94.1e100.net (172.253.62.94): icmp_seq=1 ttl=99 time=112 ms
64 bytes from bc-in-f94.1e100.net (172.253.62.94): icmp_seq=2 ttl=99 time=98.3 ms
^C
--- www.google.de ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 98.310/105.047/111.785/6.745 ms
sh-4.4# exit
exit

Removing debug pod ...
Ping check details node/sendling-d0c14274-sz7rb 
1
2
3
4
5
6
7
$ oc debug node/sendling-d0c14274-sz7rb
Starting pod/sendling-d0c14274-sz7rb-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.131.9.28
If you don't see a command prompt, try pressing enter.
sh-4.4# ping www.google.de
PING www.google.de (172.253.62.94) 56(84) bytes of data.
  • Node sendling-d0c14274-6nbvl - Ping google ✅
  • Node sendling-d0c14274-sz7rb - Ping google ❌
1
2
3
4
5
6
7
$ oc get pods -l kubevirt.io=virt-launcher -o wide -n rbohne-hcp-sendling
NAME                                          READY   STATUS      RESTARTS   AGE     IP             NODE                 NOMINATED NODE   READINESS GATES
virt-launcher-sendling-d0c14274-6nbvl-pb6zd   1/1     Running     0          6d2h    10.128.8.133   inf8                 <none>           1/1
virt-launcher-sendling-d0c14274-sz7rb-cw5vj   1/1     Running     0          3d20h   10.131.9.28    ucs-blade-server-1   <none>           1/1
virt-launcher-sendling-d0c14274-sz7rb-mbmv8   0/1     Completed   0          3d20h   10.131.9.28    ucs-blade-server-3   <none>           1/1
virt-launcher-sendling-d0c14274-sz7rb-nb25r   0/1     Completed   0          6d2h    10.131.9.28    ucs-blade-server-1   <none>           1/1
$

Checkout node routing

Host subnets:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
$ oc get nodes -o custom-columns="NODE:.metadata.name,node-subnets:.metadata.annotations.k8s\.ovn\.org/node-subnets"
NODE                 node-subnets
...
inf8                 {"default":["10.131.8.0/21"]}
ucs-blade-server-1   {"default":["10.131.0.0/21"]}
ucs-blade-server-3   {"default":["10.130.8.0/21"]}
...

$ oc get pods -n openshift-ovn-kubernetes -o wide -l  app=ovnkube-node
NAME                 READY   STATUS    RESTARTS       AGE    IP             NODE                 NOMINATED NODE   READINESS GATES
...
ovnkube-node-9xt5n   8/8     Running   8              2d7h   10.32.96.101   ucs-blade-server-1   <none>           <none>
ovnkube-node-hhsx5   8/8     Running   8              2d7h   10.32.96.8     inf8                 <none>           <none>
ovnkube-node-qx9bh   8/8     Running   9 (2d6h ago)   2d7h   10.32.96.103   ucs-blade-server-3   <none>           <none>
...

$ oc exec -n openshift-ovn-kubernetes -c ovn-controller ovnkube-node-9xt5n -- ovn-nbctl lr-route-list ovn_cluster_router
IPv4 Routes
Route Table <main>:
             10.128.8.133                100.88.0.9 dst-ip
             10.129.8.107              10.129.8.107 dst-ip rtos-ucs-blade-server-1 ecmp
             10.129.8.107                100.88.0.8 dst-ip ecmp
             10.130.10.29              10.130.10.29 dst-ip rtos-ucs-blade-server-1
              10.131.8.41               10.131.8.41 dst-ip rtos-ucs-blade-server-1
              10.131.9.28               10.131.9.28 dst-ip rtos-ucs-blade-server-1 ecmp
              10.131.9.28                100.88.0.8 dst-ip ecmp
              10.131.9.44               10.131.9.44 dst-ip rtos-ucs-blade-server-1
               100.64.0.2                100.88.0.2 dst-ip
               100.64.0.3                100.88.0.3 dst-ip
               100.64.0.4                100.88.0.4 dst-ip
               100.64.0.5                100.64.0.5 dst-ip
               100.64.0.6                100.88.0.6 dst-ip
               100.64.0.8                100.88.0.8 dst-ip
               100.64.0.9                100.88.0.9 dst-ip
              100.64.0.10               100.88.0.10 dst-ip
            10.128.0.0/21                100.88.0.2 dst-ip
            10.128.8.0/21                100.88.0.6 dst-ip
           10.128.16.0/21               100.88.0.10 dst-ip
            10.129.0.0/21                100.88.0.3 dst-ip
            10.130.0.0/21                100.88.0.4 dst-ip
            10.130.8.0/21                100.88.0.8 dst-ip
            10.131.8.0/21                100.88.0.9 dst-ip
            10.128.0.0/14                100.64.0.5 src-ip

$ oc exec -n openshift-ovn-kubernetes -c ovn-controller ovnkube-node-hhsx5   -- ovn-nbctl lr-route-list ovn_cluster_router
IPv4 Routes
Route Table <main>:
             10.128.8.133              10.128.8.133 dst-ip rtos-inf8
             10.129.8.107                100.88.0.5 dst-ip ecmp
             10.129.8.107                100.88.0.8 dst-ip ecmp
             10.130.10.29                100.88.0.5 dst-ip
              10.131.8.41                100.88.0.5 dst-ip
              10.131.9.28                100.88.0.5 dst-ip ecmp
              10.131.9.28                100.88.0.8 dst-ip ecmp
              10.131.9.44                100.88.0.5 dst-ip
               100.64.0.2                100.88.0.2 dst-ip
               100.64.0.3                100.88.0.3 dst-ip
               100.64.0.4                100.88.0.4 dst-ip
               100.64.0.5                100.88.0.5 dst-ip
               100.64.0.6                100.88.0.6 dst-ip
               100.64.0.8                100.88.0.8 dst-ip
               100.64.0.9                100.64.0.9 dst-ip
              100.64.0.10               100.88.0.10 dst-ip
            10.128.0.0/21                100.88.0.2 dst-ip
            10.128.8.0/21                100.88.0.6 dst-ip
           10.128.16.0/21               100.88.0.10 dst-ip
            10.129.0.0/21                100.88.0.3 dst-ip
            10.130.0.0/21                100.88.0.4 dst-ip
            10.130.8.0/21                100.88.0.8 dst-ip
            10.131.0.0/21                100.88.0.5 dst-ip
            10.128.0.0/14                100.64.0.9 src-ip
$

$ oc exec -n openshift-ovn-kubernetes -c ovn-controller ovnkube-node-qx9bh -- ovn-nbctl lr-route-list ovn_cluster_router
IPv4 Routes
Route Table <main>:
             10.128.8.133                100.88.0.9 dst-ip
             10.129.8.107                100.88.0.5 dst-ip
             10.130.10.29                100.88.0.5 dst-ip
              10.131.8.41                100.88.0.5 dst-ip
              10.131.9.28                100.88.0.5 dst-ip
              10.131.9.44                100.88.0.5 dst-ip
               100.64.0.2                100.88.0.2 dst-ip
               100.64.0.3                100.88.0.3 dst-ip
               100.64.0.4                100.88.0.4 dst-ip
               100.64.0.5                100.88.0.5 dst-ip
               100.64.0.6                100.88.0.6 dst-ip
               100.64.0.8                100.64.0.8 dst-ip
               100.64.0.9                100.88.0.9 dst-ip
              100.64.0.10               100.88.0.10 dst-ip
            10.128.0.0/21                100.88.0.2 dst-ip
            10.128.8.0/21                100.88.0.6 dst-ip
           10.128.16.0/21               100.88.0.10 dst-ip
            10.129.0.0/21                100.88.0.3 dst-ip
            10.130.0.0/21                100.88.0.4 dst-ip
            10.131.0.0/21                100.88.0.5 dst-ip
            10.131.8.0/21                100.88.0.9 dst-ip
            10.128.0.0/14                100.64.0.8 src-ip
2024-01-03 2020-05-06 Contributors: Robert Bohne