feat: native asap2 integration (#458)
* wip asap2 docs
* simplified asap2. a lot.
* fix: added asap2 chart changes
* fix agent name
* chore: bump neutron
diff --git a/charts/neutron/Chart.yaml b/charts/neutron/Chart.yaml
index 52b6060..f5f34be 100644
--- a/charts/neutron/Chart.yaml
+++ b/charts/neutron/Chart.yaml
@@ -9,4 +9,4 @@
sources:
- https://opendev.org/openstack/neutron
- https://opendev.org/openstack/openstack-helm
-version: 0.3.8
+version: 0.3.13
diff --git a/charts/neutron/requirements.lock b/charts/neutron/requirements.lock
index 39a58b5..62d9333 100644
--- a/charts/neutron/requirements.lock
+++ b/charts/neutron/requirements.lock
@@ -3,4 +3,4 @@
repository: file://../../openstack-helm-infra/helm-toolkit
version: 0.2.53
digest: sha256:f8f4fbba6f638b79447f7e458933b07deb792ae30a14df5900bde542cf0e64a6
-generated: "2023-05-16T04:42:11.158271588Z"
+generated: "2023-06-29T17:37:44.246907398Z"
diff --git a/charts/neutron/templates/bin/_health-probe.py.tpl b/charts/neutron/templates/bin/_health-probe.py.tpl
index 266c3d8..897b735 100644
--- a/charts/neutron/templates/bin/_health-probe.py.tpl
+++ b/charts/neutron/templates/bin/_health-probe.py.tpl
@@ -68,9 +68,14 @@
target = oslo_messaging.Target(
topic=cfg.CONF.agent_queue_name,
server=_get_hostname(use_fqdn))
- client = oslo_messaging.RPCClient(transport, target,
- timeout=rpc_timeout,
- retry=rpc_retries)
+ if hasattr(oslo_messaging, 'get_rpc_client'):
+ client = oslo_messaging.get_rpc_client(transport, target,
+ timeout=rpc_timeout,
+ retry=rpc_retries)
+ else:
+ client = oslo_messaging.RPCClient(transport, target,
+ timeout=rpc_timeout,
+ retry=rpc_retries)
client.call(context.RequestContext(),
'pod_health_probe_method_ignore_errors')
except oslo_messaging.exceptions.MessageDeliveryFailure:
@@ -271,7 +276,7 @@
cfg.CONF(sys.argv[1:])
try:
- transport = oslo_messaging.get_transport(cfg.CONF)
+ transport = oslo_messaging.get_rpc_transport(cfg.CONF)
except Exception as ex:
message = getattr(ex, "message", str(ex))
sys.stderr.write("Message bus driver load error: %s" % message)
diff --git a/charts/neutron/templates/bin/_neutron-openvswitch-agent-init-netoffload.sh.tpl b/charts/neutron/templates/bin/_neutron-openvswitch-agent-init-netoffload.sh.tpl
new file mode 100644
index 0000000..ea84eee
--- /dev/null
+++ b/charts/neutron/templates/bin/_neutron-openvswitch-agent-init-netoffload.sh.tpl
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+# ASAP2
+for cfg in $(cat /tmp/netoffload | jq -r '(.asap2 // [])[] | @base64'); do
+ _jq() {
+ echo ${cfg} | base64 --decode | jq -r ${1}
+ }
+
+ DEVICE=$(_jq '.dev')
+ VFS=$(_jq '.vfs')
+
+ offloadctl enable asap2 ${DEVICE} --vfs ${VFS}
+done
diff --git a/charts/neutron/templates/bin/_neutron-openvswitch-agent-liveness.sh.tpl b/charts/neutron/templates/bin/_neutron-openvswitch-agent-liveness.sh.tpl
new file mode 100644
index 0000000..fc154ca
--- /dev/null
+++ b/charts/neutron/templates/bin/_neutron-openvswitch-agent-liveness.sh.tpl
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -e
+
+/tmp/neutron-openvswitch-agent-readiness.sh
+
+python \
+ /tmp/health-probe.py \
+ --config-file \
+ /etc/neutron/neutron.conf \
+ --config-file \
+ /etc/neutron/plugins/ml2/openvswitch_agent.ini \
+ --agent-queue-name \
+ q-agent-notifier-tunnel-update \
+{{- if .Values.pod.use_fqdn.neutron_agent }}
+ --use-fqdn \
+{{- end }}
+ --liveness-probe
diff --git a/charts/neutron/templates/configmap-bin.yaml b/charts/neutron/templates/configmap-bin.yaml
index a701c52..5d87faa 100644
--- a/charts/neutron/templates/configmap-bin.yaml
+++ b/charts/neutron/templates/configmap-bin.yaml
@@ -65,8 +65,12 @@
{{ tuple "bin/_neutron-openvswitch-agent-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
neutron-openvswitch-agent-init-modules.sh: |
{{ tuple "bin/_neutron-openvswitch-agent-init-modules.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+ neutron-openvswitch-agent-init-netoffload.sh: |
+{{ tuple "bin/_neutron-openvswitch-agent-init-netoffload.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
neutron-openvswitch-agent-readiness.sh: |
{{ tuple "bin/_neutron-openvswitch-agent-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+ neutron-openvswitch-agent-liveness.sh: |
+{{ tuple "bin/_neutron-openvswitch-agent-liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
neutron-sriov-agent.sh: |
{{ tuple "bin/_neutron-sriov-agent.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
neutron-sriov-agent-init.sh: |
diff --git a/charts/neutron/templates/configmap-etc.yaml b/charts/neutron/templates/configmap-etc.yaml
index 45176b9..ffc535b 100644
--- a/charts/neutron/templates/configmap-etc.yaml
+++ b/charts/neutron/templates/configmap-etc.yaml
@@ -310,6 +310,7 @@
neutron_sudoers: {{ $envAll.Values.conf.neutron_sudoers | b64enc }}
rootwrap.conf: {{ $envAll.Values.conf.rootwrap | b64enc }}
auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }}
+ netoffload: {{ toJson $envAll.Values.conf.netoffload | b64enc }}
dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }}
update_dpdk_bond_config: {{ $envAll.Values.conf.ovs_dpdk.update_dpdk_bond_config | toString | b64enc }}
{{- if ( has "ovn" .Values.network.backend ) }}
diff --git a/charts/neutron/templates/daemonset-ovs-agent.yaml b/charts/neutron/templates/daemonset-ovs-agent.yaml
index 59e33f0..3b4f1e8 100644
--- a/charts/neutron/templates/daemonset-ovs-agent.yaml
+++ b/charts/neutron/templates/daemonset-ovs-agent.yaml
@@ -20,18 +20,7 @@
{{- define "ovsAgentLivenessProbeTemplate" }}
exec:
command:
- - python
- - /tmp/health-probe.py
- - --config-file
- - /etc/neutron/neutron.conf
- - --config-file
- - /etc/neutron/plugins/ml2/openvswitch_agent.ini
- - --agent-queue-name
- - q-agent-notifier-tunnel-update
- - --liveness-probe
-{{- if .Values.pod.use_fqdn.neutron_agent }}
- - --use-fqdn
-{{- end }}
+ - /tmp/neutron-openvswitch-agent-liveness.sh
{{- end }}
{{- define "neutron.ovs_agent.daemonset" }}
@@ -104,6 +93,24 @@
- name: pci-devices
mountPath: /sys/bus/pci/devices
{{- end }}
+ - name: netoffload
+{{ tuple $envAll "netoffload" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "neutron_ovs_agent" "container" "netoffload" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+ command:
+ - /tmp/neutron-openvswitch-agent-init-netoffload.sh
+ volumeMounts:
+ - name: pod-tmp
+ mountPath: /tmp
+ - name: neutron-bin
+ mountPath: /tmp/neutron-openvswitch-agent-init-netoffload.sh
+ subPath: neutron-openvswitch-agent-init-netoffload.sh
+ readOnly: true
+ - name: neutron-etc
+ mountPath: /tmp/netoffload
+ subPath: netoffload
+ readOnly: true
+ - name: run
+ mountPath: /run
- name: neutron-ovs-agent-init
{{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -213,6 +220,10 @@
subPath: neutron-openvswitch-agent-readiness.sh
readOnly: true
- name: neutron-bin
+ mountPath: /tmp/neutron-openvswitch-agent-liveness.sh
+ subPath: neutron-openvswitch-agent-liveness.sh
+ readOnly: true
+ - name: neutron-bin
mountPath: /tmp/health-probe.py
subPath: health-probe.py
readOnly: true
diff --git a/charts/neutron/values.yaml b/charts/neutron/values.yaml
index fa311b4..9966a6f 100644
--- a/charts/neutron/values.yaml
+++ b/charts/neutron/values.yaml
@@ -30,6 +30,7 @@
ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
ks_service: docker.io/openstackhelm/heat:stein-ubuntu_bionic
ks_endpoints: docker.io/openstackhelm/heat:stein-ubuntu_bionic
+ netoffload: ghcr.io/vexxhost/netoffload:v1.0.1
neutron_server: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_dhcp: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
neutron_metadata: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
@@ -540,6 +541,10 @@
- SYS_CHROOT
runAsUser: 0
readOnlyRootFilesystem: true
+ netoffload:
+ privileged: true
+ runAsUser: 0
+ readOnlyRootFilesystem: true
neutron_ovs_agent_init:
privileged: true
runAsUser: 0
@@ -1977,6 +1982,12 @@
auto_bridge_add:
br-ex: null
+ # Network off-loading configuration
+ netoffload:
+ asap2:
+ # - dev: enp97s0f0
+ # vfs: 16
+
# configuration of OVS DPDK bridges and NICs
# this is a separate section and not part of the auto_bridge_add section
# because additional parameters are needed
diff --git a/docs/user/networking.md b/docs/user/networking.md
new file mode 100644
index 0000000..24b1b5e
--- /dev/null
+++ b/docs/user/networking.md
@@ -0,0 +1,49 @@
+# Networking
+
+## Hardware Acceleration
+
+### Mellanox Accelerated Switching And Packet Processing (ASAP2)
+
+Mellanox ASAP2 is a technology that enables the offloading of the Open vSwitch
+datapath to the NIC. This offloading is done by the NIC's firmware, and is
+transparent to the host.
+
+Atmosphere uses the [`netoffload`](https://github.com/vexxhost/netoffload)
+project which takes care of validating and preparing the host for SR-IOV.
+
+It is recommended to follow the BIOS, Kernel & NIC configuration steps documented
+within the [`netoffload`](https://github.com/vexxhost/netoffload#bios-configuration)
+project before getting started.
+
+#### Open vSwitch configuration
+
+In order to enable hardware off-loading in Open vSwitch, you must make sure that
+you deploy with the following configuration:
+
+```yaml
+openvswitch_helm_values:
+ conf:
+ ovs_hw_offload:
+ enabled: true
+```
+
+#### Neutron configuration
+
+In order to enable hardware off-loading in Neutron, you can simply deploy it
+with the following configuration and it will use
+[`netoffload`](https://github.com/vexxhost/netoffload) to automatically
+configure ASAP2.
+
+!!! note
+
+ If you see an Init error when deploying Neutron, you may need to look at the
+ logs of the `netoffload` container to see what went wrong.
+
+```yaml
+neutron_helm_values:
+ conf:
+ netoffload:
+ asap2:
+ - dev: enp97s0f0
+ vfs: 16
+```
diff --git a/hack/sync-charts.sh b/hack/sync-charts.sh
index 328d5f4..1a360f2 100755
--- a/hack/sync-charts.sh
+++ b/hack/sync-charts.sh
@@ -132,15 +132,16 @@
curl -sL https://github.com/rancher/local-path-provisioner/archive/refs/tags/v${LOCAL_PATH_PROVISIONER_VERSION}.tar.gz \
| tar -xz -C ${ATMOSPHERE}/charts --strip-components=3 local-path-provisioner-${LOCAL_PATH_PROVISIONER_VERSION}/deploy/chart/
-NEUTRON_VERSION=0.3.8
+NEUTRON_VERSION=0.3.13
curl -sL https://tarballs.opendev.org/openstack/openstack-helm/neutron-${NEUTRON_VERSION}.tgz \
| tar -xz -C ${ATMOSPHERE}/charts
-curl 'https://review.opendev.org/changes/openstack%2Fopenstack-helm~872436/revisions/2/patch?download' \
+curl 'https://review.opendev.org/changes/openstack%2Fopenstack-helm~883168/revisions/11/patch?download' \
| base64 --decode \
| filterdiff -p1 -x 'releasenotes/*' \
| filterdiff -p2 -x 'Chart.yaml' \
+ | filterdiff -p1 -i 'neutron/*' \
| patch -p2 -d ${ATMOSPHERE}/charts/neutron
-curl 'https://review.opendev.org/changes/openstack%2Fopenstack-helm~883168/revisions/11/patch?download' \
+curl 'https://review.opendev.org/changes/openstack%2Fopenstack-helm~887354/revisions/2/patch?download' \
| base64 --decode \
| filterdiff -p1 -x 'releasenotes/*' \
| filterdiff -p2 -x 'Chart.yaml' \
diff --git a/roles/defaults/vars/main.yml b/roles/defaults/vars/main.yml
index 6ad2394..a2d22e6 100644
--- a/roles/defaults/vars/main.yml
+++ b/roles/defaults/vars/main.yml
@@ -108,6 +108,7 @@
manila_scheduler: quay.io/vexxhost/manila@sha256:78a258f0a632b0778d91767c4ec7b6f84e6ef1de7cf1e51e726374caa295b8f6 # image-source: quay.io/vexxhost/manila:zed
manila_share: quay.io/vexxhost/manila@sha256:78a258f0a632b0778d91767c4ec7b6f84e6ef1de7cf1e51e726374caa295b8f6 # image-source: quay.io/vexxhost/manila:zed
memcached: docker.io/library/memcached:1.6.17
+ netoffload: ghcr.io/vexxhost/netoffload:v1.0.1
neutron_bagpipe_bgp: quay.io/vexxhost/neutron@sha256:1f7e7d9d3db9100b55243b33bd215aebec89939b8249411153fc4035db62ba93 # image-source: quay.io/vexxhost/neutron:zed
neutron_coredns: docker.io/coredns/coredns:1.9.3
neutron_db_sync: quay.io/vexxhost/neutron@sha256:1f7e7d9d3db9100b55243b33bd215aebec89939b8249411153fc4035db62ba93 # image-source: quay.io/vexxhost/neutron:zed